AuroraMiddleware/qt5base-lts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

QNetworkReply: Decompress when reading

Browse Source 
Rather than when the data is received. Source compatibility is
achieved through double-decompressing the data. This lets us know
how many bytes are available just as before but without having the
uncompressed data left in memory.

Fixes: QTBUG-83269
Change-Id: I352bd09581614c582e4628243e2a0e895ba4946b
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>
This commit is contained in:
Mårten Nordheim 2020-05-13 11:04:28 +02:00
parent f9b867216b
commit 6f25051536
16 changed files with 1670 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/network/access/qhttp2protocolhandler.cpp
View File

@ -1233,13 +1233,8 @@ void QHttp2ProtocolHandler::updateStream(Stream &stream, const HPack::HttpHeader
if (QHttpNetworkReply::isHttpRedirect(statusCode) && redirectUrl.isValid())
httpReply->setRedirectUrl(redirectUrl);
if (httpReplyPrivate->isCompressed() && httpRequest.d->autoDecompress) {
if (httpReplyPrivate->isCompressed() && httpRequest.d->autoDecompress)
httpReplyPrivate->removeAutoDecompressHeader();
httpReplyPrivate->decompressHelper.setEncoding(
httpReplyPrivate->headerField("content-encoding"));
httpReplyPrivate->decompressHelper.setMinimumArchiveBombSize(
httpReplyPrivate->request.minimumArchiveBombSize());
}
if (QHttpNetworkReply::isHttpRedirect(statusCode)) {
// Note: This status code can trigger uploadByteDevice->reset() in
@ -1276,27 +1271,12 @@ void QHttp2ProtocolHandler::updateStream(Stream &stream, const Frame &frame,
if (const auto length = frame.dataSize()) {
const char *data = reinterpret_cast<const char *>(frame.dataBegin());
auto &httpRequest = stream.request();
auto replyPrivate = httpReply->d_func();
replyPrivate->totalProgress += length;
const QByteArray wrapped(data, length);
if (httpRequest.d->autoDecompress && replyPrivate->isCompressed()) {
Q_ASSERT(replyPrivate->decompressHelper.isValid());
replyPrivate->decompressHelper.feed(wrapped);
while (replyPrivate->decompressHelper.hasData()) {
QByteArray output(4 * 1024, Qt::Uninitialized);
qint64 read = replyPrivate->decompressHelper.read(output.data(), output.size());
if (read > 0) {
output.resize(read);
replyPrivate->responseData.append(std::move(output));
}
}
} else {
replyPrivate->responseData.append(wrapped);
}
replyPrivate->responseData.append(wrapped);
if (replyPrivate->shouldEmitSignals()) {
if (connectionType == Qt::DirectConnection) {

1
src/network/access/qhttpnetworkconnection.cpp
View File

@ -48,6 +48,7 @@
#include <qnetworkproxy.h>
#include <qauthenticator.h>
#include <qcoreapplication.h>
#include <private/qdecompresshelper_p.h>
#include <qbuffer.h>
#include <qpair.h>

38
src/network/access/qhttpnetworkreply.cpp
View File

@ -46,6 +46,8 @@
# include <QtNetwork/qsslconfiguration.h>
#endif
#include <private/qdecompresshelper_p.h>
QT_BEGIN_NAMESPACE
QHttpNetworkReply::QHttpNetworkReply(const QUrl &url, QObject *parent)
@ -348,7 +350,6 @@ void QHttpNetworkReplyPrivate::clearHttpLayerInformation()
currentChunkRead = 0;
lastChunkRead = false;
connectionCloseEnabled = true;
decompressHelper.clear();
fields.clear();
}
@ -554,11 +555,6 @@ qint64 QHttpNetworkReplyPrivate::readHeader(QAbstractSocket *socket)
headerField("proxy-connection").toLower().contains("close")) ||
(majorVersion == 1 && minorVersion == 0 &&
(connectionHeaderField.isEmpty() && !headerField("proxy-connection").toLower().contains("keep-alive")));
if (autoDecompress && isCompressed()) {
if (!decompressHelper.setEncoding(headerField("content-encoding")))
return -1; // Either the encoding was unsupported or the decoder could not be set up
decompressHelper.setMinimumArchiveBombSize(request.minimumArchiveBombSize());
}
}
return bytes;
}
@ -660,42 +656,18 @@ qint64 QHttpNetworkReplyPrivate::readBody(QAbstractSocket *socket, QByteDataBuff
{
qint64 bytes = 0;
// for compressed data we'll allocate a temporary one that we then decompress
QByteDataBuffer *tempOutDataBuffer = (autoDecompress ? new QByteDataBuffer : out);
if (isChunked()) {
// chunked transfer encoding (rfc 2616, sec 3.6)
bytes += readReplyBodyChunked(socket, tempOutDataBuffer);
bytes += readReplyBodyChunked(socket, out);
} else if (bodyLength > 0) {
// we have a Content-Length
bytes += readReplyBodyRaw(socket, tempOutDataBuffer, bodyLength - contentRead);
bytes += readReplyBodyRaw(socket, out, bodyLength - contentRead);
if (contentRead + bytes == bodyLength)
state = AllDoneState;
} else {
// no content length. just read what's possible
bytes += readReplyBodyRaw(socket, tempOutDataBuffer, socket->bytesAvailable());
bytes += readReplyBodyRaw(socket, out, socket->bytesAvailable());
}
// This is true if there is compressed encoding and we're supposed to use it.
if (autoDecompress) {
QScopedPointer holder(tempOutDataBuffer);
if (!decompressHelper.isValid())
return -1;
decompressHelper.feed(std::move(*tempOutDataBuffer));
while (decompressHelper.hasData()) {
QByteArray output(4 * 1024, Qt::Uninitialized);
qint64 read = decompressHelper.read(output.data(), output.size());
if (read < 0) {
return -1;
} else if (read > 0) {
output.resize(read);
out->append(std::move(output));
}
}
}
contentRead += bytes;
return bytes;
}

2
src/network/access/qhttpnetworkreply_p.h
View File

@ -274,8 +274,6 @@ public:
char* userProvidedDownloadBuffer;
QUrl redirectUrl;
QDecompressHelper decompressHelper;
};

13
src/network/access/qhttpnetworkrequest.cpp
View File

@ -57,7 +57,6 @@ QHttpNetworkRequestPrivate::QHttpNetworkRequestPrivate(const QHttpNetworkRequest
customVerb(other.customVerb),
priority(other.priority),
uploadByteDevice(other.uploadByteDevice),
minimumArchiveBombSize(other.minimumArchiveBombSize),
autoDecompress(other.autoDecompress),
pipeliningAllowed(other.pipeliningAllowed),
http2Allowed(other.http2Allowed),
@ -94,7 +93,7 @@ bool QHttpNetworkRequestPrivate::operator==(const QHttpNetworkRequestPrivate &ot
&& (redirectPolicy == other.redirectPolicy)
&& (peerVerifyName == other.peerVerifyName)
&& (needResendWithCredentials == other.needResendWithCredentials)
&& (minimumArchiveBombSize == other.minimumArchiveBombSize);
;
}
QByteArray QHttpNetworkRequest::methodName() const
@ -406,15 +405,5 @@ void QHttpNetworkRequest::setPeerVerifyName(const QString &peerName)
d->peerVerifyName = peerName;
}
qint64 QHttpNetworkRequest::minimumArchiveBombSize() const
{
return d->minimumArchiveBombSize;
}
void QHttpNetworkRequest::setMinimumArchiveBombSize(qint64 threshold)
{
d->minimumArchiveBombSize = threshold;
}
QT_END_NAMESPACE

4
src/network/access/qhttpnetworkrequest_p.h
View File

@ -150,9 +150,6 @@ public:
QString peerVerifyName() const;
void setPeerVerifyName(const QString &peerName);
qint64 minimumArchiveBombSize() const;
void setMinimumArchiveBombSize(qint64 threshold);
private:
QSharedDataPointer<QHttpNetworkRequestPrivate> d;
friend class QHttpNetworkRequestPrivate;
@ -178,7 +175,6 @@ public:
QByteArray customVerb;
QHttpNetworkRequest::Priority priority;
mutable QNonContiguousByteDevice* uploadByteDevice;
qint64 minimumArchiveBombSize = 0;
bool autoDecompress;
bool pipeliningAllowed;
bool http2Allowed;

3
src/network/access/qhttpprotocolhandler.cpp
View File

@ -177,8 +177,7 @@ void QHttpProtocolHandler::_q_receiveReply()
m_connection->d_func()->emitReplyError(m_socket, m_reply, QNetworkReply::RemoteHostClosedError);
break;
}
} else if (!replyPrivate->isChunked() && !replyPrivate->autoDecompress
&& replyPrivate->bodyLength > 0) {
} else if (!replyPrivate->isChunked() && replyPrivate->bodyLength > 0) {
// bulk files like images should fulfill these properties and
// we can therefore save on memory copying
qint64 haveRead = replyPrivate->readBodyFast(m_socket, &replyPrivate->responseData);

5
src/network/access/qhttpthreaddelegate.cpp
View File

@ -545,6 +545,7 @@ void QHttpThreadDelegate::synchronousFinishedSlot()
incomingErrorCode = statusCodeFromHttp(httpReply->statusCode(), httpRequest.url());
}
isCompressed = httpReply->isCompressed();
synchronousDownloadData = httpReply->readAll();
QMetaObject::invokeMethod(httpReply, "deleteLater", Qt::QueuedConnection);
@ -634,6 +635,7 @@ void QHttpThreadDelegate::headerChangedSlot()
incomingContentLength = httpReply->contentLength();
removedContentLength = httpReply->removedContentLength();
isHttp2Used = httpReply->isHttp2Used();
isCompressed = httpReply->isCompressed();
emit downloadMetaData(incomingHeaders,
incomingStatusCode,
@ -642,7 +644,8 @@ void QHttpThreadDelegate::headerChangedSlot()
downloadBuffer,
incomingContentLength,
removedContentLength,
isHttp2Used);
isHttp2Used,
isCompressed);
}
void QHttpThreadDelegate::synchronousHeaderChangedSlot()

4
src/network/access/qhttpthreaddelegate_p.h
View File

@ -119,6 +119,8 @@ public:
QString incomingErrorDetail;
QHttp2Configuration http2Parameters;
bool isCompressed;
protected:
// The zerocopy download buffer, if used:
QSharedPointer<char> downloadBuffer;
@ -142,7 +144,7 @@ signals:
void preSharedKeyAuthenticationRequired(QSslPreSharedKeyAuthenticator *);
#endif
void downloadMetaData(const QList<QPair<QByteArray,QByteArray> > &, int, const QString &, bool,
QSharedPointer<char>, qint64, qint64, bool);
QSharedPointer<char>, qint64, qint64, bool, bool);
void downloadProgress(qint64, qint64);
void downloadData(const QByteArray &);
void error(QNetworkReply::NetworkError, const QString &);

174
src/network/access/qnetworkreplyhttpimpl.cpp
View File

@ -305,6 +305,13 @@ qint64 QNetworkReplyHttpImpl::bytesAvailable() const
return QNetworkReply::bytesAvailable() + d->downloadBufferCurrentSize - d->downloadBufferReadPosition;
}
if (d->decompressHelper.isValid()) {
if (d->decompressHelper.isCountingBytes())
return QNetworkReply::bytesAvailable() + d->decompressHelper.uncompressedSize();
if (d->decompressHelper.hasData())
return QNetworkReply::bytesAvailable() + 1;
}
// normal buffer
return QNetworkReply::bytesAvailable();
}
@ -345,6 +352,30 @@ qint64 QNetworkReplyHttpImpl::readData(char* data, qint64 maxlen)
}
if (d->decompressHelper.isValid() && (d->decompressHelper.hasData() || !isFinished())) {
if (maxlen == 0 || !d->decompressHelper.hasData())
return 0;
const qint64 bytesRead = d->decompressHelper.read(data, maxlen);
if (!d->decompressHelper.isValid()) {
// error occurred, error copied from QHttpNetworkConnectionPrivate::errorDetail
d->error(QNetworkReplyImpl::NetworkError::ProtocolFailure,
QCoreApplication::translate("QHttp", "Data corrupted"));
}
if (d->cacheSaveDevice) {
// Need to write to the cache now that we have the data
d->cacheSaveDevice->write(data, bytesRead);
// ... and if we've read everything then the cache can be closed.
if (isFinished() && !d->decompressHelper.hasData())
d->completeCacheSave();
}
// In case of buffer size restriction we need to emit that it has been emptied
qint64 wasBuffered = d->bytesBuffered;
d->bytesBuffered = 0;
if (readBufferSize())
emit readBufferFreed(wasBuffered);
return bytesRead;
}
// normal buffer
if (d->state == d->Finished || d->state == d->Aborted)
return -1;
@ -774,7 +805,6 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq
if (request.attribute(QNetworkRequest::EmitAllUploadProgressSignalsAttribute).toBool())
emitAllUploadProgressSignals = true;
httpRequest.setMinimumArchiveBombSize(newHttpRequest.minimumArchiveBombSize());
httpRequest.setPeerVerifyName(newHttpRequest.peerVerifyName());
// Create the HTTP thread delegate
@ -839,14 +869,8 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq
QObject::connect(delegate, SIGNAL(downloadFinished()),
q, SLOT(replyFinished()),
Qt::QueuedConnection);
QObject::connect(delegate, SIGNAL(downloadMetaData(QList<QPair<QByteArray,QByteArray> >,
int, QString, bool,
QSharedPointer<char>, qint64, qint64,
bool)),
q, SLOT(replyDownloadMetaData(QList<QPair<QByteArray,QByteArray> >,
int, QString, bool,
QSharedPointer<char>, qint64, qint64, bool)),
Qt::QueuedConnection);
connect(delegate, &QHttpThreadDelegate::downloadMetaData, this,
&QNetworkReplyHttpImplPrivate::replyDownloadMetaData, Qt::QueuedConnection);
QObject::connect(delegate, SIGNAL(downloadProgress(qint64,qint64)),
q, SLOT(replyDownloadProgressSlot(qint64,qint64)),
Qt::QueuedConnection);
@ -946,30 +970,20 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq
if (synchronous) {
emit q->startHttpRequestSynchronously(); // This one is BlockingQueuedConnection, so it will return when all work is done
if (delegate->incomingErrorCode != QNetworkReply::NoError) {
replyDownloadMetaData
(delegate->incomingHeaders,
delegate->incomingStatusCode,
delegate->incomingReasonPhrase,
delegate->isPipeliningUsed,
QSharedPointer<char>(),
delegate->incomingContentLength,
delegate->removedContentLength,
delegate->isHttp2Used);
replyDownloadData(delegate->synchronousDownloadData);
replyDownloadMetaData
(delegate->incomingHeaders,
delegate->incomingStatusCode,
delegate->incomingReasonPhrase,
delegate->isPipeliningUsed,
QSharedPointer<char>(),
delegate->incomingContentLength,
delegate->removedContentLength,
delegate->isHttp2Used,
delegate->isCompressed);
replyDownloadData(delegate->synchronousDownloadData);
if (delegate->incomingErrorCode != QNetworkReply::NoError)
httpError(delegate->incomingErrorCode, delegate->incomingErrorDetail);
} else {
replyDownloadMetaData
(delegate->incomingHeaders,
delegate->incomingStatusCode,
delegate->incomingReasonPhrase,
delegate->isPipeliningUsed,
QSharedPointer<char>(),
delegate->incomingContentLength,
delegate->removedContentLength,
delegate->isHttp2Used);
replyDownloadData(delegate->synchronousDownloadData);
}
thread->quit();
thread->wait(QDeadlineTimer(5000));
@ -1040,23 +1054,72 @@ void QNetworkReplyHttpImplPrivate::replyDownloadData(QByteArray d)
if (!q->isOpen())
return;
// cache this, we need it later and it's invalidated when dealing with compressed data
auto dataSize = d.size();
// Grab this to compare later (only relevant for compressed data) in case none of the data
// will be propagated to the user
const qint64 previousBytesDownloaded = bytesDownloaded;
if (cacheEnabled && isCachingAllowed() && !cacheSaveDevice)
initCacheSaveDevice();
if (decompressHelper.isValid()) {
qint64 uncompressedBefore = -1;
if (decompressHelper.isCountingBytes())
uncompressedBefore = decompressHelper.uncompressedSize();
decompressHelper.feed(std::move(d));
if (!decompressHelper.isValid()) {
// error occurred, error copied from QHttpNetworkConnectionPrivate::errorDetail
error(QNetworkReplyImpl::NetworkError::ProtocolFailure,
QCoreApplication::translate("QHttp", "Data corrupted"));
return;
}
if (!isHttpRedirectResponse()) {
if (decompressHelper.isCountingBytes())
bytesDownloaded += (decompressHelper.uncompressedSize() - uncompressedBefore);
setupTransferTimeout();
}
if (synchronous) {
d = QByteArray();
const qsizetype increments = 16 * 1024;
qint64 bytesRead = 0;
while (decompressHelper.hasData()) {
quint64 nextSize = quint64(d.size()) + quint64(increments);
if (nextSize > quint64(std::numeric_limits<QByteArray::size_type>::max())) {
error(QNetworkReplyImpl::NetworkError::ProtocolFailure,
QCoreApplication::translate("QHttp", "Data corrupted"));
return;
}
d.resize(nextSize);
bytesRead += decompressHelper.read(d.data() + bytesRead, increments);
}
d.resize(bytesRead);
// we're synchronous so we're not calling this function again; reset the decompressHelper
decompressHelper.clear();
}
}
// This is going to look a little strange. When downloading data while a
// HTTP redirect is happening (and enabled), we write the redirect
// response to the cache. However, we do not append it to our internal
// buffer as that will contain the response data only for the final
// response
if (cacheSaveDevice)
// Note: For compressed data this is done in readData()
if (cacheSaveDevice && !decompressHelper.isValid()) {
cacheSaveDevice->write(d);
}
if (!isHttpRedirectResponse()) {
// if decompressHelper is valid then we have compressed data, and this is handled above
if (!decompressHelper.isValid() && !isHttpRedirectResponse()) {
buffer.append(d);
bytesDownloaded += d.size();
bytesDownloaded += dataSize;
setupTransferTimeout();
}
bytesBuffered += d.size();
bytesBuffered += dataSize;
int pendingSignals = pendingDownloadDataEmissions->fetchAndSubAcquire(1) - 1;
if (pendingSignals > 0) {
@ -1070,17 +1133,25 @@ void QNetworkReplyHttpImplPrivate::replyDownloadData(QByteArray d)
if (isHttpRedirectResponse())
return;
// This can occur when downloading compressed data as some of the data may be the content
// encoding's header. Don't emit anything for this.
if (previousBytesDownloaded == bytesDownloaded) {
if (readBufferMaxSize)
emit q->readBufferFreed(dataSize);
return;
}
QVariant totalSize = cookedHeaders.value(QNetworkRequest::ContentLengthHeader);
emit q->readyRead();
// emit readyRead before downloadProgress incase this will cause events to be
// processed and we get into a recursive call (as in QProgressDialog).
if (downloadProgressSignalChoke.elapsed() >= progressSignalInterval) {
if (downloadProgressSignalChoke.elapsed() >= progressSignalInterval
&& (!decompressHelper.isValid() || decompressHelper.isCountingBytes())) {
downloadProgressSignalChoke.restart();
emit q->downloadProgress(bytesDownloaded,
totalSize.isNull() ? Q_INT64_C(-1) : totalSize.toLongLong());
}
}
void QNetworkReplyHttpImplPrivate::replyFinished()
@ -1207,6 +1278,7 @@ void QNetworkReplyHttpImplPrivate::followRedirect()
Q_Q(QNetworkReplyHttpImpl);
Q_ASSERT(managerPrivate);
decompressHelper.clear();
rawHeaders.clear();
cookedHeaders.clear();
@ -1242,7 +1314,7 @@ void QNetworkReplyHttpImplPrivate::replyDownloadMetaData(const QList<QPair<QByte
QSharedPointer<char> db,
qint64 contentLength,
qint64 removedContentLength,
bool h2Used)
bool h2Used, bool isCompressed)
{
Q_Q(QNetworkReplyHttpImpl);
Q_UNUSED(contentLength);
@ -1283,6 +1355,20 @@ void QNetworkReplyHttpImplPrivate::replyDownloadMetaData(const QList<QPair<QByte
if (it->first.toLower() == "location")
value.clear();
if (isCompressed && !decompressHelper.isValid()
&& it->first.compare("content-encoding", Qt::CaseInsensitive) == 0) {
if (!synchronous) // with synchronous all the data is expected to be handled at once
decompressHelper.setCountingBytesEnabled(true);
if (!decompressHelper.setEncoding(it->second)) {
// error occurred, error copied from QHttpNetworkConnectionPrivate::errorDetail
error(QNetworkReplyImpl::NetworkError::ProtocolFailure,
QCoreApplication::translate("QHttp", "Data corrupted"));
}
decompressHelper.setMinimumArchiveBombSize(request.minimumArchiveBombSize());
}
if (!value.isEmpty()) {
// Why are we appending values for headers which are already
// present?
@ -2001,9 +2087,12 @@ void QNetworkReplyHttpImplPrivate::finished()
QVariant totalSize = cookedHeaders.value(QNetworkRequest::ContentLengthHeader);
// if we don't know the total size of or we received everything save the cache
if (totalSize.isNull() || totalSize == -1 || bytesDownloaded == totalSize)
// if we don't know the total size of or we received everything save the cache.
// If the data is compressed then this is done in readData()
if ((totalSize.isNull() || totalSize == -1 || bytesDownloaded == totalSize)
&& !decompressHelper.isValid()) {
completeCacheSave();
}
// We check for errorCode too as in case of SSL handshake failure, we still
// get the HTTP redirect status code (301, 303 etc)
@ -2041,6 +2130,9 @@ void QNetworkReplyHttpImplPrivate::error(QNetworkReplyImpl::NetworkError code, c
return;
}
if (decompressHelper.isValid())
decompressHelper.clear(); // Just get rid of any data that might be stored
errorCode = code;
q->setErrorString(errorMessage);

9
src/network/access/qnetworkreplyhttpimpl_p.h
View File

@ -73,6 +73,8 @@
Q_MOC_INCLUDE(<QtNetwork/QAuthenticator>)
#include <private/qdecompresshelper_p.h>
QT_REQUIRE_CONFIG(http);
QT_BEGIN_NAMESPACE
@ -108,9 +110,6 @@ public:
// From reply
Q_PRIVATE_SLOT(d_func(), void replyDownloadData(QByteArray))
Q_PRIVATE_SLOT(d_func(), void replyFinished())
Q_PRIVATE_SLOT(d_func(), void replyDownloadMetaData(QList<QPair<QByteArray,QByteArray> >,
int, QString, bool, QSharedPointer<char>,
qint64, qint64, bool))
Q_PRIVATE_SLOT(d_func(), void replyDownloadProgressSlot(qint64,qint64))
Q_PRIVATE_SLOT(d_func(), void httpAuthenticationRequired(const QHttpNetworkRequest &, QAuthenticator *))
Q_PRIVATE_SLOT(d_func(), void httpError(QNetworkReply::NetworkError, const QString &))
@ -258,6 +257,8 @@ public:
QNetworkRequest redirectRequest;
QDecompressHelper decompressHelper;
bool loadFromCacheIfAllowed(QHttpNetworkRequest &httpRequest);
void invalidateCache();
bool sendCacheContents(const QNetworkCacheMetaData &metaData);
@ -274,7 +275,7 @@ public:
void replyDownloadData(QByteArray);
void replyFinished();
void replyDownloadMetaData(const QList<QPair<QByteArray,QByteArray> > &, int, const QString &,
bool, QSharedPointer<char>, qint64, qint64, bool);
bool, QSharedPointer<char>, qint64, qint64, bool, bool);
void replyDownloadProgressSlot(qint64,qint64);
void httpAuthenticationRequired(const QHttpNetworkRequest &request, QAuthenticator *auth);
void httpError(QNetworkReply::NetworkError error, const QString &errorString);

BIN
tests/auto/network/access/qnetworkreply/4G.br Normal file
View File

Binary file not shown.

1235
tests/auto/network/access/qnetworkreply/data/gzip.rcc.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

116
tests/auto/network/access/qnetworkreply/data/zstandard.rcc.cpp Normal file
View File

@ -0,0 +1,116 @@
/****************************************************************************
** Resource object code
**
** Created by: The Resource Compiler for Qt version 6.0.0
**
** WARNING! All changes made in this file will be lost!
*****************************************************************************/
static const unsigned char qt_resource_data[] = {
// D:/projects/qt/dev/src/qtbase/tests/auto/network/access/decompresshelper/4G.zst
0x0,0x0,0x1,0x75,
0x0,
0x2,0x3,0x93,0x78,0xda,0xed,0xd4,0x21,0xe,0x83,0x40,0x10,0x86,0xd1,0x29,0x98,
0x26,0x98,0x3d,0x46,0x1d,0x1a,0x8f,0xec,0x29,0x50,0xdc,0x84,0x13,0xe1,0x2b,0x90,
0x1c,0x89,0xcd,0x32,0xe9,0x25,0x2a,0xfa,0x26,0x79,0xc9,0xe8,0x5f,0x7c,0xaf,0x7d,
0xac,0xc7,0x1a,0x79,0x8f,0xf4,0x8e,0x78,0xe6,0x73,0xb5,0xa9,0x74,0x5d,0x94,0x0,
0xfe,0xcf,0xfc,0xed,0x41,0x6d,0xe7,0x50,0xcc,0x1,0x32,0x60,0xe,0x90,0x1,0x73,
0x80,0xc,0x98,0x4,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,
0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,
0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,
0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,
0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,
0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,
0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,
0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,
0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,
0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,
0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,
0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,
0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,
0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,
0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,
0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,
0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,
0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x80,0x5f,0xe8,0xd3,0xf2,0x69,0xdb,0xd,
0xcd,0x15,0x90,0xe9,
};
static const unsigned char qt_resource_name[] = {
// 4G.zst
0x0,0x6,
0x3,0x8a,0x61,0xa4,
0x0,0x34,
0x0,0x47,0x0,0x2e,0x0,0x7a,0x0,0x73,0x0,0x74,
};
static const unsigned char qt_resource_struct[] = {
// :
0x0,0x0,0x0,0x0,0x0,0x2,0x0,0x0,0x0,0x1,0x0,0x0,0x0,0x1,
0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,
// :/4G.zst
0x0,0x0,0x0,0x0,0x0,0x1,0x0,0x0,0x0,0x1,0x0,0x0,0x0,0x0,
0x0,0x0,0x1,0x72,0x1c,0x8d,0x7,0xac,
};
#ifdef QT_NAMESPACE
# define QT_RCC_PREPEND_NAMESPACE(name) ::QT_NAMESPACE::name
# define QT_RCC_MANGLE_NAMESPACE0(x) x
# define QT_RCC_MANGLE_NAMESPACE1(a, b) a##_##b
# define QT_RCC_MANGLE_NAMESPACE2(a, b) QT_RCC_MANGLE_NAMESPACE1(a,b)
# define QT_RCC_MANGLE_NAMESPACE(name) QT_RCC_MANGLE_NAMESPACE2( \
QT_RCC_MANGLE_NAMESPACE0(name), QT_RCC_MANGLE_NAMESPACE0(QT_NAMESPACE))
#else
# define QT_RCC_PREPEND_NAMESPACE(name) name
# define QT_RCC_MANGLE_NAMESPACE(name) name
#endif
#ifdef QT_NAMESPACE
namespace QT_NAMESPACE {
#endif
bool qRegisterResourceData(int, const unsigned char *, const unsigned char *, const unsigned char *);
bool qUnregisterResourceData(int, const unsigned char *, const unsigned char *, const unsigned char *);
#if defined(__ELF__) || defined(__APPLE__)
static inline unsigned char qResourceFeatureZlib()
{
extern const unsigned char qt_resourceFeatureZlib;
return qt_resourceFeatureZlib;
}
#else
unsigned char qResourceFeatureZlib();
#endif
#ifdef QT_NAMESPACE
}
#endif
int QT_RCC_MANGLE_NAMESPACE(qInitResources_zstandard)();
int QT_RCC_MANGLE_NAMESPACE(qInitResources_zstandard)()
{
int version = 3;
QT_RCC_PREPEND_NAMESPACE(qRegisterResourceData)
(version, qt_resource_struct, qt_resource_name, qt_resource_data);
return 1;
}
int QT_RCC_MANGLE_NAMESPACE(qCleanupResources_zstandard)();
int QT_RCC_MANGLE_NAMESPACE(qCleanupResources_zstandard)()
{
int version = 3;
version += QT_RCC_PREPEND_NAMESPACE(qResourceFeatureZlib());
QT_RCC_PREPEND_NAMESPACE(qUnregisterResourceData)
(version, qt_resource_struct, qt_resource_name, qt_resource_data);
return 1;
}
namespace {
struct initializer {
initializer() { QT_RCC_MANGLE_NAMESPACE(qInitResources_zstandard)(); }
~initializer() { QT_RCC_MANGLE_NAMESPACE(qCleanupResources_zstandard)(); }
} dummy;
}

2
tests/auto/network/access/qnetworkreply/test/CMakeLists.txt
View File

@ -21,6 +21,8 @@ qt_internal_add_test(tst_qnetworkreply
OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/../" # special case
SOURCES
../tst_qnetworkreply.cpp
../data/gzip.rcc.cpp
../data/zstandard.rcc.cpp
PUBLIC_LIBRARIES
Qt::CorePrivate
Qt::NetworkPrivate

165
tests/auto/network/access/qnetworkreply/tst_qnetworkreply.cpp
View File

@ -509,6 +509,12 @@ private Q_SLOTS:
void contentEncoding_data();
void contentEncoding();
void contentEncodingBigPayload_data();
void contentEncodingBigPayload();
void cacheWithContentEncoding_data();
void cacheWithContentEncoding();
void downloadProgressWithContentEncoding_data();
void downloadProgressWithContentEncoding();
// NOTE: This test must be last!
void parentingRepliesToTheApp();
@ -871,9 +877,15 @@ public:
{
}
QIODevice *data(const QUrl &) override
QIODevice *data(const QUrl &url) override
{
return 0;
QIODevice *device = nullptr;
auto it = m_buffers.constFind(url);
if (it != m_buffers.cend()) {
device = *it;
device->seek(0);
}
return device;
}
bool remove(const QUrl &url) override
@ -900,7 +912,6 @@ public:
{
QUrl url = buffer->property("url").toUrl();
m_insertedUrls << url;
delete m_buffers.take(url);
}
void clear() override { m_insertedUrls.clear(); }
@ -1588,6 +1599,12 @@ void tst_QNetworkReply::initTestCase()
if (QSslSocket::activeBackend() == QStringLiteral("openssl"))
flukeCertTlsError = QSslError::SelfSignedCertificate;
#endif
// For content encoding tests
Q_INIT_RESOURCE(gzip);
#if defined(QT_BUILD_INTERNAL) && QT_CONFIG(zstd)
Q_INIT_RESOURCE(zstandard);
#endif
}
void tst_QNetworkReply::cleanupTestCase()
@ -1596,6 +1613,12 @@ void tst_QNetworkReply::cleanupTestCase()
if (!wronlyFileName.isNull())
QFile::remove(wronlyFileName);
#endif
// For content encoding tests
#if defined(QT_BUILD_INTERNAL) && QT_CONFIG(zstd)
Q_CLEANUP_RESOURCE(zstandard);
#endif
Q_CLEANUP_RESOURCE(gzip);
}
void tst_QNetworkReply::cleanupTestData()
@ -9413,6 +9436,142 @@ void tst_QNetworkReply::contentEncoding()
QCOMPARE(reply->readAll(), expected);
}
void tst_QNetworkReply::contentEncodingBigPayload_data()
{
QTest::addColumn<QByteArray>("encoding");
QTest::addColumn<QString>("path");
QTest::addColumn<qint64>("expectedSize");
qint64 fourGiB = 4ll * 1024ll * 1024ll * 1024ll;
QTest::addRow("gzip-4GB") << QByteArray("gzip") << (":/4G.gz") << fourGiB;
#if QT_CONFIG(brotli)
QTest::addRow("brotli-4GB") << QByteArray("br") << (testDataDir + "./4G.br") << fourGiB;
#endif
#if defined(QT_BUILD_INTERNAL) && QT_CONFIG(zstd)
QTest::addRow("zstd-4GB") << QByteArray("zstd") << (":/4G.zst") << fourGiB;
#else
qDebug("Note: ZStandard testdata is only available for developer builds.");
#endif
}
void tst_QNetworkReply::contentEncodingBigPayload()
{
QFETCH(QString, path);
QFile compressedFile(path);
QVERIFY(compressedFile.open(QIODevice::ReadOnly));
QByteArray body = compressedFile.readAll();
QFETCH(QByteArray, encoding);
QString header("HTTP/1.0 200 OK\r\nContent-Encoding: %1\r\nContent-Length: %2\r\n\r\n");
header = header.arg(encoding, QString::number(body.size()));
MiniHttpServer server(header.toLatin1() + body);
QNetworkRequest request(QUrl("http://localhost:" + QString::number(server.serverPort())));
// QDecompressHelper will abort the download if the compressed to decompressed size ratio
// differs too much, so we override it
request.setMinimumArchiveBombSize(-1);
QNetworkReplyPtr reply(manager.get(request));
QTRY_VERIFY2_WITH_TIMEOUT(reply->isFinished(), qPrintable(reply->errorString()), 15000);
QCOMPARE(reply->error(), QNetworkReply::NoError);
QFETCH(qint64, expectedSize);
QCOMPARE(reply->bytesAvailable(), expectedSize);
QByteArray output(512 * 1024 * 1024, Qt::Uninitialized);
qint64 total = 0;
while (reply->bytesAvailable()) {
qint64 read = reply->read(output.data(), output.size());
QVERIFY(read != -1);
total += read;
static const auto isZero = [](char c) { return c == '\0'; };
bool allZero = std::all_of(output.cbegin(), output.cbegin() + read, isZero);
QVERIFY(allZero);
}
QCOMPARE(total, expectedSize);
}
void tst_QNetworkReply::cacheWithContentEncoding_data()
{
contentEncoding_data();
}
void tst_QNetworkReply::cacheWithContentEncoding()
{
QFETCH(QByteArray, encoding);
QFETCH(QByteArray, body);
QFETCH(QByteArray, expected);
QString header("HTTP/1.0 200 OK\r\nContent-Encoding: %1\r\nContent-Length: %2\r\n\r\n");
header = header.arg(encoding, QString::number(body.size()));
MiniHttpServer server(header.toLatin1() + body);
MySpyMemoryCache *cache = new MySpyMemoryCache(this);
manager.setCache(cache);
auto unsetCache = qScopeGuard([this](){ manager.setCache(nullptr); });
QUrl url("http://localhost:" + QString::number(server.serverPort()));
QNetworkRequest request(url);
QNetworkReplyPtr reply(manager.get(request));
QVERIFY2(waitForFinish(reply) == Success, msgWaitForFinished(reply));
QCOMPARE(reply->error(), QNetworkReply::NoError);
QByteArray output = reply->readAll();
QIODevice *device = cache->data(url);
QVERIFY(device);
QByteArray fromCache = device->readAll();
QCOMPARE(output, expected);
QCOMPARE(fromCache, expected);
}
void tst_QNetworkReply::downloadProgressWithContentEncoding_data()
{
contentEncoding_data();
}
void tst_QNetworkReply::downloadProgressWithContentEncoding()
{
QFETCH(QByteArray, encoding);
QFETCH(QByteArray, body);
QFETCH(QByteArray, expected);
QString header("HTTP/1.0 200 OK\r\nContent-Encoding: %1\r\nContent-Length: %2\r\n\r\n");
header = header.arg(encoding, QString::number(body.size()));
MiniHttpServer server(header.toLatin1() + body);
QUrl url("http://localhost:" + QString::number(server.serverPort()));
QNetworkRequest request(url);
QNetworkReplyPtr reply(manager.get(request));
// Limit the amount of bytes we read so we get more than one downloadProgress emission
reply->setReadBufferSize(5);
qint64 bytesReceived = -1;
connect(reply.data(), &QNetworkReply::downloadProgress, this,
[reply = reply.data(), &expected, &bytesReceived](qint64 recv, qint64 total) {
qint64 previous = bytesReceived;
bytesReceived = recv;
if (bytesReceived > expected.size()) {
qWarning("bytesReceived greater than expected size!");
reply->abort();
}
if (bytesReceived < previous) {
qWarning("bytesReceived shrank!");
reply->abort();
}
});
SlowReader reader(reply.data());
QVERIFY2(waitForFinish(reply) == Success, msgWaitForFinished(reply));
QCOMPARE(reply->error(), QNetworkReply::NoError);
QCOMPARE(bytesReceived, expected.size());
}
// NOTE: This test must be last testcase in tst_qnetworkreply!
void tst_QNetworkReply::parentingRepliesToTheApp()
{