AuroraMiddleware/qt5base-lts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

QNAM HTTP: Re-write compression code

Browse Source 
This eliminates some code (header parsing) that can be done by
zlib already.
Add support for 'deflate' encoding.
Also do less memory copying while uncompressing.

Change-Id: I94de21e3c58b904dd91d004c375ed8cbea56cb0b
Task-Number: QTBUG-13191
Reviewed-on: http://codereview.qt.nokia.com/1314
Reviewed-by: Qt Sanity Bot <qt_sanity_bot@ovi.com>
Reviewed-by: Martin Petersson <Martin.Petersson@nokia.com>
Reviewed-by: Peter Hartmann <peter.hartmann@nokia.com>
Reviewed-by: Markus Goetz
This commit is contained in:
Markus Goetz 2011-07-06 16:08:59 +02:00 committed by Qt by Nokia
parent 363c710bc4
commit 2b5dcfcee1
6 changed files with 100 additions and 249 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/network/access/qhttpnetworkconnection.cpp
View File

@ -271,7 +271,7 @@ void QHttpNetworkConnectionPrivate::prepareRequest(HttpMessagePair &messagePair)
value = request.headerField("accept-encoding");
if (value.isEmpty()) {
#ifndef QT_NO_COMPRESS
request.setHeaderField("Accept-Encoding", "gzip");
request.setHeaderField("Accept-Encoding", "gzip, deflate");
request.d->autoDecompress = true;
#else
// if zlib is not available set this to false always

3
src/network/access/qhttpnetworkconnection_p.h
View File

@ -202,9 +202,6 @@ public:
QString errorDetail(QNetworkReply::NetworkError errorCode, QAbstractSocket *socket,
const QString &extraDetail = QString());
#ifndef QT_NO_COMPRESS
bool expand(QAbstractSocket *socket, QHttpNetworkReply *reply, bool dataComplete);
#endif
void removeReply(QHttpNetworkReply *reply);
QString hostName;

82
src/network/access/qhttpnetworkconnectionchannel.cpp
View File

@ -403,7 +403,7 @@ void QHttpNetworkConnectionChannel::_q_receiveReply()
bytes += headerBytes;
// If headers were parsed successfully now it is the ReadingDataState
if (replyPrivate->state == QHttpNetworkReplyPrivate::ReadingDataState) {
if (replyPrivate->isGzipped() && replyPrivate->autoDecompress) {
if (replyPrivate->isCompressed() && replyPrivate->autoDecompress) {
// remove the Content-Length from header
replyPrivate->removeAutoDecompressHeader();
} else {
@ -475,30 +475,18 @@ void QHttpNetworkConnectionChannel::_q_receiveReply()
{
// use the traditional slower reading (for compressed encoding, chunked encoding,
// no content-length etc)
QByteDataBuffer byteDatas;
qint64 haveRead = replyPrivate->readBody(socket, &byteDatas);
if (haveRead) {
qint64 haveRead = replyPrivate->readBody(socket, &replyPrivate->responseData);
if (haveRead > 0) {
bytes += haveRead;
if (replyPrivate->autoDecompress)
replyPrivate->appendCompressedReplyData(byteDatas);
else
replyPrivate->appendUncompressedReplyData(byteDatas);
if (!replyPrivate->autoDecompress) {
replyPrivate->totalProgress += bytes;
if (replyPrivate->shouldEmitSignals()) {
// important: At the point of this readyRead(), the byteDatas list must be empty,
// else implicit sharing will trigger memcpy when the user is reading data!
emit reply->readyRead();
emit reply->dataReadProgress(replyPrivate->totalProgress, replyPrivate->bodyLength);
}
replyPrivate->totalProgress += haveRead;
if (replyPrivate->shouldEmitSignals()) {
emit reply->readyRead();
emit reply->dataReadProgress(replyPrivate->totalProgress, replyPrivate->bodyLength);
}
#ifndef QT_NO_COMPRESS
else if (!expand(false)) { // expand a chunk if possible
// If expand() failed we can just return, it had already called connection->emitReplyError()
return;
}
#endif
} else if (haveRead == -1) {
// Some error occured
connection->d_func()->emitReplyError(socket, reply, QNetworkReply::ProtocolFailure);
break;
}
}
// still in ReadingDataState? This function will be called again by the socket's readyRead
@ -638,57 +626,9 @@ bool QHttpNetworkConnectionChannel::ensureConnection()
return true;
}
#ifndef QT_NO_COMPRESS
bool QHttpNetworkConnectionChannel::expand(bool dataComplete)
{
Q_ASSERT(socket);
Q_ASSERT(reply);
qint64 total = reply->d_func()->compressedData.size();
if (total >= CHUNK || dataComplete) {
// uncompress the data
QByteArray content, inflated;
content = reply->d_func()->compressedData;
reply->d_func()->compressedData.clear();
int ret = Z_OK;
if (content.size())
ret = reply->d_func()->gunzipBodyPartially(content, inflated);
int retCheck = (dataComplete) ? Z_STREAM_END : Z_OK;
if (ret >= retCheck) {
if (inflated.size()) {
reply->d_func()->totalProgress += inflated.size();
reply->d_func()->appendUncompressedReplyData(inflated);
if (reply->d_func()->shouldEmitSignals()) {
// important: At the point of this readyRead(), inflated must be cleared,
// else implicit sharing will trigger memcpy when the user is reading data!
emit reply->readyRead();
emit reply->dataReadProgress(reply->d_func()->totalProgress, 0);
}
}
} else {
connection->d_func()->emitReplyError(socket, reply, QNetworkReply::ProtocolFailure);
return false;
}
}
return true;
}
#endif
void QHttpNetworkConnectionChannel::allDone()
{
Q_ASSERT(reply);
#ifndef QT_NO_COMPRESS
// expand the whole data.
if (reply->d_func()->expectContent() && reply->d_func()->autoDecompress && !reply->d_func()->streamEnd) {
bool expandResult = expand(true);
// If expand() failed we can just return, it had already called connection->emitReplyError()
if (!expandResult)
return;
}
#endif
if (!reply) {
qWarning() << "QHttpNetworkConnectionChannel::allDone() called without reply. Please report at http://bugreports.qt.nokia.com/";

1
src/network/access/qhttpnetworkconnectionchannel_p.h
View File

@ -149,7 +149,6 @@ public:
bool ensureConnection();
bool expand(bool dataComplete);
void allDone(); // reply header + body have been read
void handleStatus(); // called from allDone()

235
src/network/access/qhttpnetworkreply.cpp
View File

@ -65,6 +65,11 @@ QHttpNetworkReply::~QHttpNetworkReply()
if (d->connection) {
d->connection->d_func()->removeReply(this);
}
#ifndef QT_NO_COMPRESS
if (d->autoDecompress && d->isCompressed())
inflateEnd(&d->inflateStrm);
#endif
}
QUrl QHttpNetworkReply::url() const
@ -252,7 +257,7 @@ QHttpNetworkReplyPrivate::QHttpNetworkReplyPrivate(const QUrl &newUrl)
chunkedTransferEncoding(false),
connectionCloseEnabled(true),
forceConnectionCloseEnabled(false),
currentChunkSize(0), currentChunkRead(0), connection(0), initInflate(false),
currentChunkSize(0), currentChunkRead(0), connection(0),
autoDecompress(false), responseData(), requestIsPrepared(false)
,pipeliningUsed(false), downstreamLimited(false)
,userProvidedDownloadBuffer(0)
@ -274,11 +279,9 @@ void QHttpNetworkReplyPrivate::clearHttpLayerInformation()
currentChunkRead = 0;
connectionCloseEnabled = true;
#ifndef QT_NO_COMPRESS
if (initInflate)
if (autoDecompress)
inflateEnd(&inflateStrm);
#endif
initInflate = false;
streamEnd = false;
fields.clear();
}
@ -297,10 +300,10 @@ qint64 QHttpNetworkReplyPrivate::bytesAvailable() const
return (state != ReadingDataState ? 0 : fragment.size());
}
bool QHttpNetworkReplyPrivate::isGzipped()
bool QHttpNetworkReplyPrivate::isCompressed()
{
QByteArray encoding = headerField("content-encoding");
return qstricmp(encoding.constData(), "gzip") == 0;
return qstricmp(encoding.constData(), "gzip") == 0 || qstricmp(encoding.constData(), "deflate") == 0;
}
void QHttpNetworkReplyPrivate::removeAutoDecompressHeader()
@ -358,120 +361,6 @@ QAuthenticatorPrivate::Method QHttpNetworkReplyPrivate::authenticationMethod(boo
return method;
}
#ifndef QT_NO_COMPRESS
bool QHttpNetworkReplyPrivate::gzipCheckHeader(QByteArray &content, int &pos)
{
int method = 0; // method byte
int flags = 0; // flags byte
bool ret = false;
// Assure two bytes in the buffer so we can peek ahead -- handle case
// where first byte of header is at the end of the buffer after the last
// gzip segment
pos = -1;
QByteArray &body = content;
int maxPos = body.size()-1;
if (maxPos < 1) {
return ret;
}
// Peek ahead to check the gzip magic header
if (body[0] != char(gz_magic[0]) ||
body[1] != char(gz_magic[1])) {
return ret;
}
pos += 2;
// Check the rest of the gzip header
if (++pos <= maxPos)
method = body[pos];
if (pos++ <= maxPos)
flags = body[pos];
if (method != Z_DEFLATED || (flags & RESERVED) != 0) {
return ret;
}
// Discard time, xflags and OS code:
pos += 6;
if (pos > maxPos)
return ret;
if ((flags & EXTRA_FIELD) && ((pos+2) <= maxPos)) { // skip the extra field
unsigned len = (unsigned)body[++pos];
len += ((unsigned)body[++pos])<<8;
pos += len;
if (pos > maxPos)
return ret;
}
if ((flags & ORIG_NAME) != 0) { // skip the original file name
while(++pos <= maxPos && body[pos]) {}
}
if ((flags & COMMENT) != 0) { // skip the .gz file comment
while(++pos <= maxPos && body[pos]) {}
}
if ((flags & HEAD_CRC) != 0) { // skip the header crc
pos += 2;
if (pos > maxPos)
return ret;
}
ret = (pos < maxPos); // return failed, if no more bytes left
return ret;
}
int QHttpNetworkReplyPrivate::gunzipBodyPartially(QByteArray &compressed, QByteArray &inflated)
{
int ret = Z_DATA_ERROR;
unsigned have;
unsigned char out[CHUNK];
int pos = -1;
if (!initInflate) {
// check the header
if (!gzipCheckHeader(compressed, pos))
return ret;
// allocate inflate state
inflateStrm.zalloc = Z_NULL;
inflateStrm.zfree = Z_NULL;
inflateStrm.opaque = Z_NULL;
inflateStrm.avail_in = 0;
inflateStrm.next_in = Z_NULL;
ret = inflateInit2(&inflateStrm, -MAX_WBITS);
if (ret != Z_OK)
return ret;
initInflate = true;
streamEnd = false;
}
//remove the header.
compressed.remove(0, pos+1);
// expand until deflate stream ends
inflateStrm.next_in = (unsigned char *)compressed.data();
inflateStrm.avail_in = compressed.size();
do {
inflateStrm.avail_out = sizeof(out);
inflateStrm.next_out = out;
ret = inflate(&inflateStrm, Z_NO_FLUSH);
switch (ret) {
case Z_NEED_DICT:
ret = Z_DATA_ERROR;
// and fall through
case Z_DATA_ERROR:
case Z_MEM_ERROR:
inflateEnd(&inflateStrm);
initInflate = false;
return ret;
}
have = sizeof(out) - inflateStrm.avail_out;
inflated.append(QByteArray((const char *)out, have));
} while (inflateStrm.avail_out == 0);
// clean up and return
if (ret <= Z_ERRNO || ret == Z_STREAM_END) {
inflateEnd(&inflateStrm);
initInflate = false;
}
streamEnd = (ret == Z_STREAM_END);
return ret;
}
#endif
qint64 QHttpNetworkReplyPrivate::readStatus(QAbstractSocket *socket)
{
if (fragment.isEmpty()) {
@ -616,6 +505,24 @@ qint64 QHttpNetworkReplyPrivate::readHeader(QAbstractSocket *socket)
connectionCloseEnabled = (connectionHeaderField.toLower().contains("close") ||
headerField("proxy-connection").toLower().contains("close")) ||
(majorVersion == 1 && minorVersion == 0 && connectionHeaderField.isEmpty());
#ifndef QT_NO_COMPRESS
if (autoDecompress && isCompressed()) {
// allocate inflate state
inflateStrm.zalloc = Z_NULL;
inflateStrm.zfree = Z_NULL;
inflateStrm.opaque = Z_NULL;
inflateStrm.avail_in = 0;
inflateStrm.next_in = Z_NULL;
// "windowBits can also be greater than 15 for optional gzip decoding.
// Add 32 to windowBits to enable zlib and gzip decoding with automatic header detection"
// http://www.zlib.net/manual.html
int ret = inflateInit2(&inflateStrm, MAX_WBITS+32);
if (ret != Z_OK)
return -1;
}
#endif
}
return bytes;
}
@ -712,22 +619,74 @@ qint64 QHttpNetworkReplyPrivate::readBodyFast(QAbstractSocket *socket, QByteData
qint64 QHttpNetworkReplyPrivate::readBody(QAbstractSocket *socket, QByteDataBuffer *out)
{
qint64 bytes = 0;
#ifndef QT_NO_COMPRESS
// for gzip we'll allocate a temporary one that we then decompress
QByteDataBuffer *tempOutDataBuffer = (autoDecompress ? new QByteDataBuffer : out);
#else
QByteDataBuffer *tempOutDataBuffer = out;
#endif
if (isChunked()) {
// chunked transfer encoding (rfc 2616, sec 3.6)
bytes += readReplyBodyChunked(socket, out);
bytes += readReplyBodyChunked(socket, tempOutDataBuffer);
} else if (bodyLength > 0) {
// we have a Content-Length
bytes += readReplyBodyRaw(socket, out, bodyLength - contentRead);
bytes += readReplyBodyRaw(socket, tempOutDataBuffer, bodyLength - contentRead);
if (contentRead + bytes == bodyLength)
state = AllDoneState;
} else {
// no content length. just read what's possible
bytes += readReplyBodyRaw(socket, out, socket->bytesAvailable());
bytes += readReplyBodyRaw(socket, tempOutDataBuffer, socket->bytesAvailable());
}
#ifndef QT_NO_COMPRESS
// This is true if there is compressed encoding and we're supposed to use it.
if (autoDecompress) {
qint64 uncompressRet = uncompressBodyData(tempOutDataBuffer, out);
delete tempOutDataBuffer;
if (uncompressRet < 0)
return -1;
}
#endif
contentRead += bytes;
return bytes;
}
#ifndef QT_NO_COMPRESS
qint64 QHttpNetworkReplyPrivate::uncompressBodyData(QByteDataBuffer *in, QByteDataBuffer *out)
{
for (int i = 0; i < in->bufferCount(); i++) {
QByteArray &bIn = (*in)[i];
inflateStrm.avail_in = bIn.size();
inflateStrm.next_in = reinterpret_cast<Bytef*>(bIn.data());
do {
QByteArray bOut;
// make a wild guess about the uncompressed size.
bOut.reserve(inflateStrm.avail_in * 3 + 512);
inflateStrm.avail_out = bOut.capacity();
inflateStrm.next_out = reinterpret_cast<Bytef*>(bOut.data());
int ret = inflate(&inflateStrm, Z_NO_FLUSH);
switch (ret) {
case Z_NEED_DICT:
case Z_DATA_ERROR:
case Z_MEM_ERROR:
return -1;
}
bOut.resize(bOut.capacity() - inflateStrm.avail_out);
out->append(bOut);
} while (inflateStrm.avail_in > 0);
}
return out->byteAmount();
}
#endif
qint64 QHttpNetworkReplyPrivate::readReplyBodyRaw(QAbstractSocket *socket, QByteDataBuffer *out, qint64 size)
{
// FIXME get rid of this function and just use readBodyFast and give it socket->bytesAvailable()
@ -841,36 +800,6 @@ qint64 QHttpNetworkReplyPrivate::getChunkSize(QAbstractSocket *socket, qint64 *c
return bytes;
}
void QHttpNetworkReplyPrivate::appendUncompressedReplyData(QByteArray &qba)
{
responseData.append(qba);
// clear the original! helps with implicit sharing and
// avoiding memcpy when the user is reading the data
qba.clear();
}
void QHttpNetworkReplyPrivate::appendUncompressedReplyData(QByteDataBuffer &data)
{
responseData.append(data);
// clear the original! helps with implicit sharing and
// avoiding memcpy when the user is reading the data
data.clear();
}
void QHttpNetworkReplyPrivate::appendCompressedReplyData(QByteDataBuffer &data)
{
// Work in progress: Later we will directly use a list of QByteArray or a QRingBuffer
// instead of one QByteArray.
for(int i = 0; i < data.bufferCount(); i++) {
QByteArray &byteData = data[i];
compressedData.append(byteData.constData(), byteData.size());
}
data.clear();
}
bool QHttpNetworkReplyPrivate::shouldEmitSignals()
{
// for 401 & 407 don't emit the data signals. Content along with these

26
src/network/access/qhttpnetworkreply_p.h
View File

@ -56,15 +56,7 @@
#ifndef QT_NO_HTTP
#ifndef QT_NO_COMPRESS
# include <zlib.h>
static const unsigned char gz_magic[2] = {0x1f, 0x8b}; // gzip magic header
// gzip flag byte
#define HEAD_CRC 0x02 // bit 1 set: header CRC present
#define EXTRA_FIELD 0x04 // bit 2 set: extra field present
#define ORIG_NAME 0x08 // bit 3 set: original file name present
#define COMMENT 0x10 // bit 4 set: file comment present
#define RESERVED 0xE0 // bits 5..7: reserved
#define CHUNK 16384
#include <zlib.h>
#endif
#include <QtNetwork/qtcpsocket.h>
@ -192,10 +184,6 @@ public:
qint64 readReplyBodyChunked(QAbstractSocket *in, QByteDataBuffer *out);
qint64 getChunkSize(QAbstractSocket *in, qint64 *chunkSize);
void appendUncompressedReplyData(QByteArray &qba);
void appendUncompressedReplyData(QByteDataBuffer &data);
void appendCompressedReplyData(QByteDataBuffer &data);
bool shouldEmitSignals();
bool expectContent();
void eraseData();
@ -203,11 +191,8 @@ public:
qint64 bytesAvailable() const;
bool isChunked();
bool isConnectionCloseEnabled();
bool isGzipped();
#ifndef QT_NO_COMPRESS
bool gzipCheckHeader(QByteArray &content, int &pos);
int gunzipBodyPartially(QByteArray &compressed, QByteArray &inflated);
#endif
bool isCompressed();
void removeAutoDecompressHeader();
enum ReplyState {
@ -236,11 +221,12 @@ public:
qint64 currentChunkRead;
QPointer<QHttpNetworkConnection> connection;
QPointer<QHttpNetworkConnectionChannel> connectionChannel;
bool initInflate;
bool streamEnd;
#ifndef QT_NO_COMPRESS
z_stream inflateStrm;
qint64 uncompressBodyData(QByteDataBuffer *in, QByteDataBuffer *out);
#endif
bool autoDecompress;
QByteDataBuffer responseData; // uncompressed body