AuroraMiddleware/qt5base-lts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

QHttpHeaderParser: Allow larger fields but restrict total size

Browse Source 
Our limit of 8k for a single header field was too small for certain
services which returned Location values or WWW-Authenticate challenges
longer than 8k.

Instead, taking inspiration from Chromium, we have a limit of 250k for
the full header itself. And increasing our field limit to 100k, which
would occupy almost half of what the total header allows.

Also took the opportunity to make it adjustable from the outside so we
can set more strict limits in other components.

Fixes: QTBUG-104132
Pick-to: 6.3 6.4
Change-Id: Ibbe139445e79baaef30829cfbc9a59f884e96293
Reviewed-by: Ievgenii Meshcheriakov <ievgenii.meshcheriakov@qt.io>
This commit is contained in:
Lars Schmertmann 2022-06-13 08:55:07 +02:00 committed by Mårten Nordheim
parent e8a782fb2c
commit e3ea1d02e6
6 changed files with 183 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/network/access/qhttpheaderparser.cpp
View File

@ -1,4 +1,4 @@
// Copyright (C) 2021 The Qt Company Ltd.
// Copyright (C) 2022 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "qhttpheaderparser_p.h"
@ -7,12 +7,6 @@
QT_BEGIN_NAMESPACE
// both constants are taken from the default settings of Apache
// see: http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestfieldsize and
// http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestfields
static const int MAX_HEADER_FIELD_SIZE = 8 * 1024;
static const int MAX_HEADER_FIELDS = 100;
QHttpHeaderParser::QHttpHeaderParser()
: statusCode(100) // Required by tst_QHttpNetworkConnection::ignoresslerror(failure)
, majorVersion(0)
@ -55,19 +49,22 @@ bool QHttpHeaderParser::parseHeaders(QByteArrayView header)
while (int tail = header.endsWith("\n\r\n") ? 2 : header.endsWith("\n\n") ? 1 : 0)
header.chop(tail);
if (header.size() - (header.endsWith("\r\n") ? 2 : 1) > maxTotalSize)
return false;
QList<QPair<QByteArray, QByteArray>> result;
while (header.size()) {
const int colon = header.indexOf(':');
if (colon == -1) // if no colon check if empty headers
return result.size() == 0 && (header == "\n" || header == "\r\n");
if (result.size() >= MAX_HEADER_FIELDS)
if (result.size() >= maxFieldCount)
return false;
QByteArrayView name = header.first(colon);
if (!fieldNameCheck(name))
return false;
header = header.sliced(colon + 1);
QByteArray value;
int valueSpace = MAX_HEADER_FIELD_SIZE - name.size() - 1;
qsizetype valueSpace = maxFieldSize - name.size() - 1;
do {
const int endLine = header.indexOf('\n');
Q_ASSERT(endLine != -1);
@ -84,7 +81,7 @@ bool QHttpHeaderParser::parseHeaders(QByteArrayView header)
}
header = header.sliced(endLine + 1);
} while (hSpaceStart(header));
Q_ASSERT(name.size() + 1 + value.size() <= MAX_HEADER_FIELD_SIZE);
Q_ASSERT(name.size() + 1 + value.size() <= maxFieldSize);
result.append(qMakePair(name.toByteArray(), value));
}

34
src/network/access/qhttpheaderparser_p.h
View File

@ -1,4 +1,4 @@
// Copyright (C) 2021 The Qt Company Ltd.
// Copyright (C) 2022 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#ifndef QHTTPHEADERPARSER_H
@ -24,6 +24,25 @@
QT_BEGIN_NAMESPACE
namespace HeaderConstants {
// We previously used 8K, which is common on server side, but it turned out to
// not be enough for various uses. Historically Firefox used 10K as the limit of
// a single field, but some Location headers and Authorization challenges can
// get even longer. Other browsers, such as Chrome, instead have a limit on the
// total size of all the headers (as well as extra limits on some of the
// individual fields). We'll use 100K as our default limit, which would be a ridiculously large
// header, with the possibility to override it where we need to.
static constexpr int MAX_HEADER_FIELD_SIZE = 100 * 1024;
// Taken from http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestfields
static constexpr int MAX_HEADER_FIELDS = 100;
// Chromium has a limit on the total size of the header set to 256KB,
// which is a reasonable default for QNetworkAccessManager.
// https://stackoverflow.com/a/3436155
static constexpr int MAX_TOTAL_HEADER_SIZE = 256 * 1024;
}
class Q_NETWORK_PRIVATE_EXPORT QHttpHeaderParser
{
public:
@ -54,12 +73,25 @@ public:
void removeHeaderField(const QByteArray &name);
void clearHeaders();
void setMaxHeaderFieldSize(qsizetype size) { maxFieldSize = size; }
qsizetype maxHeaderFieldSize() const { return maxFieldSize; }
void setMaxTotalHeaderSize(qsizetype size) { maxTotalSize = size; }
qsizetype maxTotalHeaderSize() const { return maxTotalSize; }
void setMaxHeaderFields(qsizetype count) { maxFieldCount = count; }
qsizetype maxHeaderFields() const { return maxFieldCount; }
private:
QList<QPair<QByteArray, QByteArray> > fields;
QString reasonPhrase;
int statusCode;
int majorVersion;
int minorVersion;
qsizetype maxFieldSize = HeaderConstants::MAX_HEADER_FIELD_SIZE;
qsizetype maxTotalSize = HeaderConstants::MAX_TOTAL_HEADER_SIZE;
qsizetype maxFieldCount = HeaderConstants::MAX_HEADER_FIELDS;
};

1
tests/auto/network/access/CMakeLists.txt
View File

@ -9,6 +9,7 @@ add_subdirectory(qnetworkreply)
add_subdirectory(qnetworkcachemetadata)
add_subdirectory(qabstractnetworkcache)
if(QT_FEATURE_private_tests)
add_subdirectory(qhttpheaderparser)
add_subdirectory(qhttpnetworkconnection)
add_subdirectory(qhttpnetworkreply)
add_subdirectory(hpack)

11
tests/auto/network/access/qhttpheaderparser/CMakeLists.txt Normal file
View File

@ -0,0 +1,11 @@
if(NOT QT_FEATURE_private_tests)
return()
endif()
qt_internal_add_test(tst_qhttpheaderparser
SOURCES
tst_qhttpheaderparser.cpp
LIBRARIES
Qt::NetworkPrivate
)

94
tests/auto/network/access/qhttpheaderparser/tst_qhttpheaderparser.cpp Normal file
View File

@ -0,0 +1,94 @@
// Copyright (C) 2022 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
#include <QtTest/qtest.h>
#include <QObject>
#include <QtNetwork/private/qhttpheaderparser_p.h>
class tst_QHttpHeaderParser : public QObject
{
Q_OBJECT
private Q_SLOTS:
void constructor();
void limitsSetters();
void adjustableLimits_data();
void adjustableLimits();
// general parsing tests can be found in tst_QHttpNetworkReply
};
void tst_QHttpHeaderParser::constructor()
{
QHttpHeaderParser parser;
QCOMPARE(parser.getStatusCode(), 100);
QCOMPARE(parser.getMajorVersion(), 0);
QCOMPARE(parser.getMinorVersion(), 0);
QCOMPARE(parser.getReasonPhrase(), QByteArray());
QCOMPARE(parser.combinedHeaderValue("Location"), QByteArray());
QCOMPARE(parser.maxHeaderFields(), HeaderConstants::MAX_HEADER_FIELDS);
QCOMPARE(parser.maxHeaderFieldSize(), HeaderConstants::MAX_HEADER_FIELD_SIZE);
QCOMPARE(parser.maxTotalHeaderSize(), HeaderConstants::MAX_TOTAL_HEADER_SIZE);
}
void tst_QHttpHeaderParser::limitsSetters()
{
QHttpHeaderParser parser;
parser.setMaxHeaderFields(10);
QCOMPARE(parser.maxHeaderFields(), 10);
parser.setMaxHeaderFieldSize(10);
QCOMPARE(parser.maxHeaderFieldSize(), 10);
parser.setMaxTotalHeaderSize(10);
QCOMPARE(parser.maxTotalHeaderSize(), 10);
}
void tst_QHttpHeaderParser::adjustableLimits_data()
{
QTest::addColumn<qsizetype>("maxFieldCount");
QTest::addColumn<qsizetype>("maxFieldSize");
QTest::addColumn<qsizetype>("maxTotalSize");
QTest::addColumn<QByteArray>("headers");
QTest::addColumn<bool>("success");
// We pretend -1 means to not set a new limit.
QTest::newRow("maxFieldCount-pass") << qsizetype(10) << qsizetype(-1) << qsizetype(-1)
<< QByteArray("Location: hi\r\n\r\n") << true;
QTest::newRow("maxFieldCount-fail") << qsizetype(1) << qsizetype(-1) << qsizetype(-1)
<< QByteArray("Location: hi\r\nCookie: a\r\n\r\n") << false;
QTest::newRow("maxFieldSize-pass") << qsizetype(-1) << qsizetype(50) << qsizetype(-1)
<< QByteArray("Location: hi\r\n\r\n") << true;
constexpr char cookieHeader[] = "Cookie: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
static_assert(sizeof(cookieHeader) - 1 == 51);
QByteArray fullHeader = QByteArray("Location: hi\r\n") + cookieHeader;
QTest::newRow("maxFieldSize-fail") << qsizetype(-1) << qsizetype(50) << qsizetype(-1)
<< (fullHeader + "\r\n\r\n") << false;
QTest::newRow("maxTotalSize-pass") << qsizetype(-1) << qsizetype(-1) << qsizetype(50)
<< QByteArray("Location: hi\r\n\r\n") << true;
QTest::newRow("maxTotalSize-fail") << qsizetype(-1) << qsizetype(-1) << qsizetype(10)
<< QByteArray("Location: hi\r\n\r\n") << false;
}
void tst_QHttpHeaderParser::adjustableLimits()
{
QFETCH(qsizetype, maxFieldCount);
QFETCH(qsizetype, maxFieldSize);
QFETCH(qsizetype, maxTotalSize);
QFETCH(QByteArray, headers);
QFETCH(bool, success);
QHttpHeaderParser parser;
if (maxFieldCount != qsizetype(-1))
parser.setMaxHeaderFields(maxFieldCount);
if (maxFieldSize != qsizetype(-1))
parser.setMaxHeaderFieldSize(maxFieldSize);
if (maxTotalSize != qsizetype(-1))
parser.setMaxTotalHeaderSize(maxTotalSize);
QCOMPARE(parser.parseHeaders(headers), success);
}
QTEST_MAIN(tst_QHttpHeaderParser)
#include "tst_qhttpheaderparser.moc"

53
tests/auto/network/access/qhttpnetworkreply/tst_qhttpnetworkreply.cpp
View File

@ -1,10 +1,11 @@
// Copyright (C) 2016 The Qt Company Ltd.
// Copyright (C) 2022 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
#include <QTest>
#include <QtCore/QBuffer>
#include <QtCore/QByteArray>
#include <QtCore/QStringBuilder>
#include "private/qhttpnetworkconnection_p.h"
@ -83,12 +84,6 @@ void tst_QHttpNetworkReply::parseHeader()
}
}
// both constants are taken from the default settings of Apache
// see: http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestfieldsize and
// http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestfields
const int MAX_HEADER_FIELD_SIZE = 8 * 1024;
const int MAX_HEADER_FIELDS = 100;
void tst_QHttpNetworkReply::parseHeaderVerification_data()
{
QTest::addColumn<QByteArray>("headers");
@ -106,36 +101,62 @@ void tst_QHttpNetworkReply::parseHeaderVerification_data()
QTest::newRow("missing-colon-3")
<< QByteArray("Content-Encoding: gzip\r\nContent-Length\r\n") << false;
QTest::newRow("header-field-too-long")
<< (QByteArray("Content-Type: ") + QByteArray(MAX_HEADER_FIELD_SIZE, 'a')
+ QByteArray("\r\n"))
<< (QByteArray("Content-Type: ")
+ QByteArray(HeaderConstants::MAX_HEADER_FIELD_SIZE, 'a') + QByteArray("\r\n"))
<< false;
QByteArray name = "Content-Type: ";
QTest::newRow("max-header-field-size")
<< (name + QByteArray(MAX_HEADER_FIELD_SIZE - name.size(), 'a') + QByteArray("\r\n"))
<< (name + QByteArray(HeaderConstants::MAX_HEADER_FIELD_SIZE - name.size(), 'a')
+ QByteArray("\r\n"))
<< true;
QByteArray tooManyHeaders = QByteArray("Content-Type: text/html; charset=utf-8\r\n")
.repeated(MAX_HEADER_FIELDS + 1);
.repeated(HeaderConstants::MAX_HEADER_FIELDS + 1);
QTest::newRow("too-many-headers") << tooManyHeaders << false;
QByteArray maxHeaders =
QByteArray("Content-Type: text/html; charset=utf-8\r\n").repeated(MAX_HEADER_FIELDS);
QByteArray maxHeaders = QByteArray("Content-Type: text/html; charset=utf-8\r\n")
.repeated(HeaderConstants::MAX_HEADER_FIELDS);
QTest::newRow("max-headers") << maxHeaders << true;
QByteArray firstValue(MAX_HEADER_FIELD_SIZE / 2, 'a');
QByteArray firstValue(HeaderConstants::MAX_HEADER_FIELD_SIZE / 2, 'a');
constexpr int obsFold = 1;
QTest::newRow("max-continuation-size")
<< (name + firstValue + QByteArray("\r\n ")
+ QByteArray(MAX_HEADER_FIELD_SIZE - name.size() - firstValue.size() - obsFold, 'b')
+ QByteArray(HeaderConstants::MAX_HEADER_FIELD_SIZE - name.size()
- firstValue.size() - obsFold,
'b')
+ QByteArray("\r\n"))
<< true;
QTest::newRow("too-long-continuation-size")
<< (name + firstValue + QByteArray("\r\n ")
+ QByteArray(MAX_HEADER_FIELD_SIZE - name.size() - firstValue.size() - obsFold + 1,
+ QByteArray(HeaderConstants::MAX_HEADER_FIELD_SIZE - name.size()
- firstValue.size() - obsFold + 1,
'b')
+ QByteArray("\r\n"))
<< false;
auto appendLongHeaderElement = [](QByteArray &result, QByteArrayView name) {
const qsizetype size = result.size();
result += name;
result += ": ";
result.resize(size + HeaderConstants::MAX_HEADER_FIELD_SIZE, 'a');
};
QByteArray longHeader;
constexpr qsizetype TrailerLength = sizeof("\r\n\r\n") - 1; // we ignore the trailing newlines
longHeader.reserve(HeaderConstants::MAX_TOTAL_HEADER_SIZE + TrailerLength + 1);
appendLongHeaderElement(longHeader, "Location");
longHeader += "\r\n";
appendLongHeaderElement(longHeader, "WWW-Authenticate");
longHeader += "\r\nProxy-Authenticate: ";
longHeader.resize(HeaderConstants::MAX_TOTAL_HEADER_SIZE, 'a');
longHeader += "\r\n\r\n";
// Test with headers which are just large enough to fit our MAX_TOTAL_HEADER_SIZE limit:
QTest::newRow("total-header-close-to-max-size") << longHeader << true;
// Now add another character to make the total header size exceed the limit:
longHeader.insert(HeaderConstants::MAX_TOTAL_HEADER_SIZE - TrailerLength, 'a');
QTest::newRow("total-header-too-large") << longHeader << false;
}
void tst_QHttpNetworkReply::parseHeaderVerification()