CBOR support: prevent overflowing QByteArray's max allocation

QByteArray doesn't like it.

Apply the same protection to QString, which we know uses the same
backend but uses elements twice as big. That means it can contain
slightly more than half as many elements, but exact half will suffice
for our needs.

Change-Id: Iaa63461109844e978376fffd15f9d4c7a9137856
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>
This commit is contained in:
Thiago Macieira 2020-03-06 13:38:17 -08:00
parent 0b4ae8e682
commit 783d574b93
9 changed files with 294 additions and 51 deletions

View File

@ -338,7 +338,7 @@ void addValidationColumns()
QTest::addColumn<CborError>("expectedError");
}
void addValidationData()
void addValidationData(size_t minInvalid = ~size_t(0))
{
// illegal numbers are future extension points
QTest::newRow("illegal-number-in-unsigned-1") << raw("\x81\x1c") << 0 << CborErrorIllegalNumber;
@ -488,26 +488,35 @@ void addValidationData()
QTest::newRow("map-break-after-value-tag2") << raw("\x81\xbf\0\xd8\x20\xff") << 0 << CborErrorUnexpectedBreak;
// check for pointer additions wrapping over the limit of the address space
CborError tooLargeOn32bit = (sizeof(void *) == 4) ? CborErrorDataTooLarge : CborErrorUnexpectedEOF;
auto wraparoundError = [minInvalid](uint64_t encodedSize) {
if (encodedSize > minInvalid)
return CborErrorDataTooLarge;
return CborErrorUnexpectedEOF;
};
constexpr uint64_t FourGB = UINT32_MAX + UINT64_C(1);
// on 32-bit systems, this is a -1
QTest::newRow("bytearray-wraparound1") << raw("\x81\x5a\xff\xff\xff\xff") << 0 << CborErrorUnexpectedEOF;
QTest::newRow("string-wraparound1") << raw("\x81\x7a\xff\xff\xff\xff") << 0 << CborErrorUnexpectedEOF;
QTest::newRow("bytearray-wraparound1") << raw("\x81\x5a\xff\xff\xff\xff") << 0 << wraparoundError(UINT32_MAX);
QTest::newRow("string-wraparound1") << raw("\x81\x7a\xff\xff\xff\xff") << 0 << wraparoundError(UINT32_MAX);
// on 32-bit systems, a 4GB addition could be dropped
QTest::newRow("bytearray-wraparound2") << raw("\x81\x5b\0\0\0\1\0\0\0\0") << 0 << tooLargeOn32bit;
QTest::newRow("string-wraparound2") << raw("\x81\x7b\0\0\0\1\0\0\0\0") << 0 << tooLargeOn32bit;
QTest::newRow("bytearray-wraparound2") << raw("\x81\x5b\0\0\0\1\0\0\0\0") << 0 << wraparoundError(FourGB);
QTest::newRow("string-wraparound2") << raw("\x81\x7b\0\0\0\1\0\0\0\0") << 0 << wraparoundError(FourGB);
// on 64-bit systems, this could be a -1
QTest::newRow("bytearray-wraparound3") << raw("\x81\x5b\xff\xff\xff\xff\xff\xff\xff\xff") << 0 << tooLargeOn32bit;
QTest::newRow("string-wraparound3") << raw("\x81\x7b\xff\xff\xff\xff\xff\xff\xff\xff") << 0 << tooLargeOn32bit;
QTest::newRow("bytearray-wraparound3") << raw("\x81\x5b\xff\xff\xff\xff\xff\xff\xff\xff") << 0
<< wraparoundError(UINT64_MAX);
QTest::newRow("string-wraparound3") << raw("\x81\x7b\xff\xff\xff\xff\xff\xff\xff\xff") << 0
<< wraparoundError(UINT64_MAX);
// ditto on chunks
QTest::newRow("bytearray-chunk-wraparound1") << raw("\x81\x5f\x5a\xff\xff\xff\xff") << 0 << CborErrorUnexpectedEOF;
QTest::newRow("string-chunk-wraparound1") << raw("\x81\x7f\x7a\xff\xff\xff\xff") << 0 << CborErrorUnexpectedEOF;
QTest::newRow("bytearray-chunk-wraparound1") << raw("\x81\x5f\x5a\xff\xff\xff\xff") << 0 << wraparoundError(UINT32_MAX);
QTest::newRow("string-chunk-wraparound1") << raw("\x81\x7f\x7a\xff\xff\xff\xff") << 0 << wraparoundError(UINT32_MAX);
// on 32-bit systems, a 4GB addition could be dropped
QTest::newRow("bytearray-chunk-wraparound2") << raw("\x81\x5f\x5b\0\0\0\1\0\0\0\0") << 0 << tooLargeOn32bit;
QTest::newRow("string-chunk-wraparound2") << raw("\x81\x7f\x7b\0\0\0\1\0\0\0\0") << 0 << tooLargeOn32bit;
QTest::newRow("bytearray-chunk-wraparound2") << raw("\x81\x5f\x5b\0\0\0\1\0\0\0\0") << 0 << wraparoundError(FourGB);
QTest::newRow("string-chunk-wraparound2") << raw("\x81\x7f\x7b\0\0\0\1\0\0\0\0") << 0 << wraparoundError(FourGB);
// on 64-bit systems, this could be a -1
QTest::newRow("bytearray-chunk-wraparound3") << raw("\x81\x5f\x5b\xff\xff\xff\xff\xff\xff\xff\xff") << 0 << tooLargeOn32bit;
QTest::newRow("string-chunk-wraparound3") << raw("\x81\x7f\x7b\xff\xff\xff\xff\xff\xff\xff\xff") << 0 << tooLargeOn32bit;
QTest::newRow("bytearray-chunk-wraparound3") << raw("\x81\x5f\x5b\xff\xff\xff\xff\xff\xff\xff\xff") << 0
<< wraparoundError(UINT64_MAX);
QTest::newRow("string-chunk-wraparound3") << raw("\x81\x7f\x7b\xff\xff\xff\xff\xff\xff\xff\xff") << 0
<< wraparoundError(UINT64_MAX);
QTest::newRow("eof-after-array") << raw("\x81") << 0 << CborErrorUnexpectedEOF;
QTest::newRow("eof-after-array2") << raw("\x81\x78\x20") << 0 << CborErrorUnexpectedEOF;

View File

@ -1,6 +1,6 @@
/****************************************************************************
**
** Copyright (C) 2018 Intel Corporation.
** Copyright (C) 2020 Intel Corporation.
** Contact: https://www.qt.io/licensing/
**
** This file is part of the QtCore module of the Qt Toolkit.
@ -39,6 +39,7 @@
#include "qcborstream.h"
#include <private/qbytearray_p.h>
#include <private/qnumeric_p.h>
#include <private/qutfcodec_p.h>
#include <qbuffer.h>
@ -2282,6 +2283,10 @@ bool QCborStreamReader::next(int maxRecursion)
} else if (isString() || isByteArray()) {
auto r = _readByteArray_helper();
while (r.status == Ok) {
if (isString() && r.data.size() > MaxStringSize) {
d->handleError(CborErrorDataTooLarge);
break;
}
if (isString() && !QUtf8::isValidUtf8(r.data, r.data.size()).isValidUtf8) {
d->handleError(CborErrorInvalidUtf8TextString);
break;
@ -2564,15 +2569,23 @@ QCborStreamReader::StringResult<QString> QCborStreamReader::_readString_helper()
result.status = r.status;
if (r.status == Ok) {
QTextCodec::ConverterState cs;
result.data = QUtf8::convertToUnicode(r.data, r.data.size(), &cs);
if (cs.invalidChars == 0 && cs.remainingChars == 0)
return result;
// See QUtf8::convertToUnicode() a detailed explanation of why this
// conversion uses the same number of words or less.
CborError err = CborNoError;
if (r.data.size() > MaxStringSize) {
err = CborErrorDataTooLarge;
} else {
QTextCodec::ConverterState cs;
result.data = QUtf8::convertToUnicode(r.data, r.data.size(), &cs);
if (cs.invalidChars != 0 || cs.remainingChars != 0)
err = CborErrorInvalidUtf8TextString;
}
d->handleError(CborErrorInvalidUtf8TextString);
result.data.clear();
result.status = Error;
return result;
if (err) {
d->handleError(err);
result.data.clear();
result.status = Error;
}
}
return result;
}
@ -2600,6 +2613,10 @@ QCborStreamReader::StringResult<QByteArray> QCborStreamReader::_readByteArray_he
qsizetype len = _currentStringChunkSize();
if (len < 0)
return result;
if (len > MaxByteArraySize) {
d->handleError(CborErrorDataTooLarge);
return result;
}
result.data.resize(len);
auto r = readStringChunk(result.data.data(), len);

View File

@ -1,6 +1,6 @@
/****************************************************************************
**
** Copyright (C) 2018 Intel Corporation.
** Copyright (C) 2020 Intel Corporation.
** Contact: https://www.qt.io/licensing/
**
** This file is part of the QtCore module of the Qt Toolkit.
@ -46,6 +46,7 @@
#include <qendian.h>
#include <qlocale.h>
#include <private/qbytearray_p.h>
#include <private/qnumeric_p.h>
#include <private/qsimd_p.h>
@ -1534,6 +1535,8 @@ void QCborContainerPrivate::decodeStringFromCbor(QCborStreamReader &reader)
// and calculate the final size
if (add_overflow(offset, increment, &newSize))
return -1;
if (newSize > MaxByteArraySize)
return -1;
// since usedData <= data.size(), this can't overflow
usedData += increment;
@ -1608,13 +1611,6 @@ void QCborContainerPrivate::decodeStringFromCbor(QCborStreamReader &reader)
setErrorInReader(reader, { QCborError::DataTooLarge });
}
if (r.status == QCborStreamReader::Error) {
// There can only be errors if there was data to be read.
Q_ASSERT(e.flags & Element::HasByteData);
data.truncate(e.value);
return;
}
// update size
if (e.flags & Element::HasByteData) {
auto b = new (dataPtr() + e.value) ByteData;
@ -1626,6 +1622,21 @@ void QCborContainerPrivate::decodeStringFromCbor(QCborStreamReader &reader)
Q_ASSERT(e.type == QCborValue::String);
e.flags |= Element::StringIsAscii;
}
// check that this UTF-8 text string can be loaded onto a QString
if (e.type == QCborValue::String) {
if (Q_UNLIKELY(b->len > MaxStringSize)) {
setErrorInReader(reader, { QCborError::DataTooLarge });
r.status = QCborStreamReader::Error;
}
}
}
if (r.status == QCborStreamReader::Error) {
// There can only be errors if there was data to be read.
Q_ASSERT(e.flags & Element::HasByteData);
data.truncate(e.value);
return;
}
elements.append(e);

View File

@ -56,10 +56,9 @@
QT_BEGIN_NAMESPACE
enum {
// Define as enum to force inlining. Don't expose MaxAllocSize in a public header.
MaxByteArraySize = MaxAllocSize - sizeof(std::remove_pointer<QByteArray::DataPtr>::type)
};
// -1 because of the terminating NUL
constexpr qsizetype MaxByteArraySize = MaxAllocSize - sizeof(std::remove_pointer<QByteArray::DataPtr>::type) - 1;
constexpr qsizetype MaxStringSize = (MaxAllocSize - sizeof(std::remove_pointer<QByteArray::DataPtr>::type)) / 2 - 1;
QT_END_NAMESPACE

View File

@ -0,0 +1,134 @@
/****************************************************************************
**
** Copyright (C) 2020 Intel Corporation.
** Contact: https://www.qt.io/licensing/
**
** This file is part of the QtCore module of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 3 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL3 included in the
** packaging of this file. Please review the following information to
** ensure the GNU Lesser General Public License version 3 requirements
** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 2.0 or (at your option) the GNU General
** Public license version 3 or any later version approved by the KDE Free
** Qt Foundation. The licenses are as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-2.0.html and
** https://www.gnu.org/licenses/gpl-3.0.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
#include <QtTest/QtTest>
#include <cbor.h>
namespace {
// A QIODevice that supplies a fixed header followed by a large sequence of
// null bytes up until a pre-determined size.
class LargeIODevice final : public QIODevice
{
public:
qint64 realSize;
QByteArray start;
LargeIODevice(const QByteArray &start, qint64 size, QObject *parent = nullptr)
: QIODevice(parent), realSize(size), start(start)
{}
qint64 size() const override { return realSize; }
bool isSequential() const override { return false; }
protected:
qint64 readData(char *data, qint64 maxlen) override;
qint64 writeData(const char *, qint64) override { return -1; }
};
};
qint64 LargeIODevice::readData(char *data, qint64 maxlen)
{
qint64 p = pos();
if (maxlen > realSize - p)
maxlen = realSize - p;
memset(data, '\0', maxlen);
qint64 fromstart = start.size() - p;
if (fromstart > maxlen)
fromstart = maxlen;
else if (fromstart < 0)
fromstart = 0;
if (fromstart)
memcpy(data, start.constData() + p, fromstart);
return maxlen;
}
void addValidationLargeData(qsizetype minInvalid, qsizetype maxInvalid)
{
char toolong[2 + sizeof(qsizetype)] = { char(0x81) };
for (qsizetype v = maxInvalid; v >= minInvalid; --v) {
// 0x5a for 32-bit, 0x5b for 64-bit
toolong[1] = sizeof(v) > 4 ? 0x5b : 0x5a;
qToBigEndian(v, toolong + 2);
QTest::addRow("bytearray-too-big-for-qbytearray-%llx", v)
<< QByteArray(toolong, sizeof(toolong)) << 0 << CborErrorDataTooLarge;
toolong[1] |= 0x20;
// QCborStreamReader::readString copies to a QByteArray first
QTest::addRow("string-too-big-for-qbytearray-%llx", v)
<< QByteArray(toolong, sizeof(toolong)) << 0 << CborErrorDataTooLarge;
}
}
void addValidationHugeDevice(qsizetype byteArrayInvalid, qsizetype stringInvalid)
{
qRegisterMetaType<QSharedPointer<QIODevice>>();
QTest::addColumn<QSharedPointer<QIODevice>>("device");
QTest::addColumn<CborError>("expectedError");
char buf[1 + sizeof(quint64)];
auto device = [&buf](QCborStreamReader::Type t, quint64 size) {
buf[0] = quint8(t) | 0x1b;
qToBigEndian(size, buf + 1);
size += sizeof(buf);
QSharedPointer<QIODevice> p =
QSharedPointer<LargeIODevice>::create(QByteArray(buf, sizeof(buf)), size);
return p;
};
// do the exact limits
QTest::newRow("bytearray-just-too-big")
<< device(QCborStreamReader::ByteArray, byteArrayInvalid) << CborErrorDataTooLarge;
QTest::newRow("string-just-too-big")
<< device(QCborStreamReader::String, stringInvalid) << CborErrorDataTooLarge;
auto addSize = [=](const char *sizename, qint64 size) {
if (byteArrayInvalid < size)
QTest::addRow("bytearray-%s", sizename)
<< device(QCborStreamReader::ByteArray, size) << CborErrorDataTooLarge;
if (stringInvalid < size)
QTest::addRow("string-%s", sizename)
<< device(QCborStreamReader::String, size) << CborErrorDataTooLarge;
};
addSize("1GB", quint64(1) << 30);
addSize("2GB", quint64(1) << 31);
addSize("4GB", quint64(1) << 32);
addSize("max", std::numeric_limits<qint64>::max() - sizeof(buf));
}

View File

@ -1,4 +1,4 @@
QT = core testlib
QT = core-private testlib
TARGET = tst_qcborstreamreader
CONFIG += testcase
SOURCES += \

View File

@ -1,6 +1,6 @@
/****************************************************************************
**
** Copyright (C) 2018 Intel Corporation.
** Copyright (C) 2020 Intel Corporation.
** Contact: https://www.qt.io/licensing/
**
** This file is part of the QtCore module of the Qt Toolkit.
@ -40,6 +40,8 @@
#include <QtCore/qcborstream.h>
#include <QtTest>
#include <QtCore/private/qbytearray_p.h>
class tst_QCborStreamReader : public QObject
{
Q_OBJECT
@ -73,6 +75,8 @@ private Q_SLOTS:
void next();
void validation_data();
void validation();
void hugeDeviceValidation_data();
void hugeDeviceValidation();
void recursionLimit_data();
void recursionLimit();
@ -902,16 +906,26 @@ void tst_QCborStreamReader::next()
QVERIFY(doit("\xbf\x9f\1\xff\x9f" + data + "\xff\xff"));
}
#include "../cborlargedatavalidation.cpp"
void tst_QCborStreamReader::validation_data()
{
// Add QCborStreamReader-specific limitations due to use of QByteArray and
// QString, which are allocated by QArrayData::allocate().
const qsizetype MaxInvalid = std::numeric_limits<QByteArray::size_type>::max();
const qsizetype MinInvalid = MaxByteArraySize + 1;
addValidationColumns();
addValidationData();
addValidationData(MinInvalid);
addValidationLargeData(MinInvalid, MaxInvalid);
}
void tst_QCborStreamReader::validation()
{
QFETCH_GLOBAL(bool, useDevice);
QFETCH(QByteArray, data);
QFETCH(CborError, expectedError);
QCborError error = { QCborError::Code(expectedError) };
QBuffer buffer(&data);
QCborStreamReader reader(data);
@ -920,12 +934,39 @@ void tst_QCborStreamReader::validation()
reader.setDevice(&buffer);
}
parse(reader, data);
QVERIFY(reader.lastError() != QCborError::NoError);
QCOMPARE(reader.lastError(), error);
// next() should fail
reader.reset();
QVERIFY(!reader.next());
QVERIFY(reader.lastError() != QCborError::NoError);
QCOMPARE(reader.lastError(), error);
}
void tst_QCborStreamReader::hugeDeviceValidation_data()
{
addValidationHugeDevice(MaxByteArraySize + 1, MaxStringSize + 1);
}
void tst_QCborStreamReader::hugeDeviceValidation()
{
QFETCH_GLOBAL(bool, useDevice);
if (!useDevice)
return;
QFETCH(QSharedPointer<QIODevice>, device);
QFETCH(CborError, expectedError);
QCborError error = { QCborError::Code(expectedError) };
device->open(QIODevice::ReadOnly | QIODevice::Unbuffered);
QCborStreamReader reader(device.data());
QVERIFY(parseOne(reader).isEmpty());
QCOMPARE(reader.lastError(), error);
// next() should fail
reader.reset();
QVERIFY(!reader.next());
QCOMPARE(reader.lastError(), error);
}
static const int Recursions = 3;

View File

@ -1,4 +1,4 @@
QT = core testlib
QT = core-private testlib
TARGET = tst_qcborvalue
CONFIG += testcase
SOURCES += \

View File

@ -1,6 +1,6 @@
/****************************************************************************
**
** Copyright (C) 2018 Intel Corporation.
** Copyright (C) 2020 Intel Corporation.
** Contact: https://www.qt.io/licensing/
**
** This file is part of the QtCore module of the Qt Toolkit.
@ -40,6 +40,8 @@
#include <QtCore/qcborvalue.h>
#include <QtTest>
#include <QtCore/private/qbytearray_p.h>
Q_DECLARE_METATYPE(QCborKnownTags)
Q_DECLARE_METATYPE(QCborValue)
Q_DECLARE_METATYPE(QCborValue::EncodingOptions)
@ -102,6 +104,8 @@ private slots:
void fromCborStreamReaderIODevice();
void validation_data();
void validation();
void hugeDeviceValidation_data();
void hugeDeviceValidation();
void recursionLimit_data();
void recursionLimit();
void toDiagnosticNotation_data();
@ -1689,10 +1693,17 @@ void tst_QCborValue::fromCborStreamReaderIODevice()
fromCbor_common(doCheck);
}
#include "../cborlargedatavalidation.cpp"
void tst_QCborValue::validation_data()
{
// Add QCborStreamReader-specific limitations due to use of QByteArray and
// QString, which are allocated by QArrayData::allocate().
const qsizetype MaxInvalid = std::numeric_limits<QByteArray::size_type>::max();
const qsizetype MinInvalid = MaxByteArraySize + 1;
addValidationColumns();
addValidationData();
addValidationData(MinInvalid);
addValidationLargeData(MinInvalid, MaxInvalid);
// These tests say we have arrays and maps with very large item counts.
// They are meant to ensure we don't pre-allocate a lot of memory
@ -1700,28 +1711,49 @@ void tst_QCborValue::validation_data()
// elements in the stream is only 2, so we should get an unexpected EOF
// error. QCborValue internally uses 16 bytes per element, so we get to
// 2 GB at 2^27 elements.
QTest::addRow("very-large-array-no-overflow") << raw("\x9a\x07\xff\xff\xff" "\0\0");
QTest::addRow("very-large-array-overflow1") << raw("\x9a\x40\0\0\0" "\0\0");
QTest::addRow("very-large-array-no-overflow") << raw("\x9a\x07\xff\xff\xff" "\0\0") << 0 << CborErrorUnexpectedEOF;
QTest::addRow("very-large-array-overflow1") << raw("\x9a\x40\0\0\0" "\0\0") << 0 << CborErrorUnexpectedEOF;
// this makes sure we don't accidentally clip to 32-bit: sending 2^32+2 elements
QTest::addRow("very-large-array-overflow2") << raw("\x9b\0\0\0\1""\0\0\0\2" "\0\0");
QTest::addRow("very-large-array-overflow2") << raw("\x9b\0\0\0\1""\0\0\0\2" "\0\0") << 0 << CborErrorDataTooLarge;
}
void tst_QCborValue::validation()
{
QFETCH(QByteArray, data);
QFETCH(CborError, expectedError);
QCborError error = { QCborError::Code(expectedError) };
QCborParserError error;
QCborValue decoded = QCborValue::fromCbor(data, &error);
QVERIFY(error.error != QCborError{});
QCborParserError parserError;
QCborValue decoded = QCborValue::fromCbor(data, &parserError);
QCOMPARE(parserError.error, error);
if (data.startsWith('\x81')) {
// decode without the array prefix
decoded = QCborValue::fromCbor(data.mid(1), &error);
QVERIFY(error.error != QCborError{});
char *ptr = const_cast<char *>(data.constData());
QByteArray mid = QByteArray::fromRawData(ptr + 1, data.size() - 1);
decoded = QCborValue::fromCbor(mid, &parserError);
QCOMPARE(parserError.error, error);
}
}
void tst_QCborValue::hugeDeviceValidation_data()
{
addValidationHugeDevice(MaxByteArraySize + 1, MaxStringSize + 1);
}
void tst_QCborValue::hugeDeviceValidation()
{
QFETCH(QSharedPointer<QIODevice>, device);
QFETCH(CborError, expectedError);
QCborError error = { QCborError::Code(expectedError) };
device->open(QIODevice::ReadOnly | QIODevice::Unbuffered);
QCborStreamReader reader(device.data());
QCborValue decoded = QCborValue::fromCbor(reader);
QCOMPARE(reader.lastError(), error);
}
void tst_QCborValue::recursionLimit_data()
{
constexpr int RecursionAttempts = 4096;