From e1d3687d64a19d27448b3f8247505daa99261ea1 Mon Sep 17 00:00:00 2001 From: Lars Knoll Date: Fri, 15 Feb 2013 10:44:54 +0100 Subject: [PATCH] Fix crashes when creating large documents Compact an object in regular intervals when inserting data into it, to avoid the object becoming huge. Compact an object/array before inserting into another array or object. Check that the document doesn't get so big it's overflowing the internal data structures. Task-number: QTBUG-29288 Change-Id: Id39d80dac1e7d5a11f40819f41b4b336bce16947 Reviewed-by: Thiago Macieira --- src/corelib/json/qjson.cpp | 11 ++++++++++- src/corelib/json/qjson_p.h | 5 ++++- src/corelib/json/qjsonarray.cpp | 24 ++++++++++++++++-------- src/corelib/json/qjsondocument.h | 3 ++- src/corelib/json/qjsonobject.cpp | 16 +++++++++++----- src/corelib/json/qjsonparser.cpp | 21 +++++++++++++++++++++ 6 files changed, 64 insertions(+), 16 deletions(-) diff --git a/src/corelib/json/qjson.cpp b/src/corelib/json/qjson.cpp index ed6ef74e3c..8215aeefc0 100644 --- a/src/corelib/json/qjson.cpp +++ b/src/corelib/json/qjson.cpp @@ -149,6 +149,10 @@ bool Data::valid() const int Base::reserveSpace(uint dataSize, int posInTable, uint numItems, bool replace) { Q_ASSERT(posInTable >= 0 && posInTable <= (int)length); + if (size + dataSize >= Value::MaxSize) { + qWarning("QJson: Document too large to store in data structure %d %d %d", (uint)size, dataSize, Value::MaxSize); + return 0; + } offset off = tableOffset; // move table to new position @@ -334,7 +338,7 @@ bool Value::isValid(const Base *b) const /*! \internal */ -int Value::requiredStorage(const QJsonValue &v, bool *compressed) +int Value::requiredStorage(QJsonValue &v, bool *compressed) { *compressed = false; switch (v.t) { @@ -351,6 +355,11 @@ int Value::requiredStorage(const QJsonValue &v, bool *compressed) } case QJsonValue::Array: case QJsonValue::Object: + if (v.d && v.d->compactionCounter) { + v.detach(); + v.d->compact(); + v.base = static_cast(v.d->header->root()); + } return v.base ? v.base->size : sizeof(QJsonPrivate::Base); case QJsonValue::Undefined: case QJsonValue::Null: diff --git a/src/corelib/json/qjson_p.h b/src/corelib/json/qjson_p.h index 81439a00ce..06885ad972 100644 --- a/src/corelib/json/qjson_p.h +++ b/src/corelib/json/qjson_p.h @@ -543,6 +543,9 @@ public: class Value { public: + enum { + MaxSize = (1<<27) - 1 + }; union { uint _dummy; qle_bitfield<0, 3> type; @@ -564,7 +567,7 @@ public: bool isValid(const Base *b) const; - static int requiredStorage(const QJsonValue &v, bool *compressed); + static int requiredStorage(QJsonValue &v, bool *compressed); static uint valueToStore(const QJsonValue &v, uint offset); static void copyData(const QJsonValue &v, char *dest, bool compressed); }; diff --git a/src/corelib/json/qjsonarray.cpp b/src/corelib/json/qjsonarray.cpp index 5f1c38a752..fb8d2e83ff 100644 --- a/src/corelib/json/qjsonarray.cpp +++ b/src/corelib/json/qjsonarray.cpp @@ -391,9 +391,10 @@ QJsonValue QJsonArray::takeAt(int i) void QJsonArray::insert(int i, const QJsonValue &value) { Q_ASSERT (i >= 0 && i <= (a ? (int)a->length : 0)); + QJsonValue val = value; bool compressed; - int valueSize = QJsonPrivate::Value::requiredStorage(value, &compressed); + int valueSize = QJsonPrivate::Value::requiredStorage(val, &compressed); detach(valueSize + sizeof(QJsonPrivate::Value)); @@ -401,13 +402,16 @@ void QJsonArray::insert(int i, const QJsonValue &value) a->tableOffset = sizeof(QJsonPrivate::Array); int valueOffset = a->reserveSpace(valueSize, i, 1, false); + if (!valueOffset) + return; + QJsonPrivate::Value &v = (*a)[i]; - v.type = (value.t == QJsonValue::Undefined ? QJsonValue::Null : value.t); + v.type = (val.t == QJsonValue::Undefined ? QJsonValue::Null : val.t); v.latinOrIntValue = compressed; v.latinKey = false; - v.value = QJsonPrivate::Value::valueToStore(value, valueOffset); + v.value = QJsonPrivate::Value::valueToStore(val, valueOffset); if (valueSize) - QJsonPrivate::Value::copyData(value, (char *)a + valueOffset, compressed); + QJsonPrivate::Value::copyData(val, (char *)a + valueOffset, compressed); } /*! @@ -437,9 +441,10 @@ void QJsonArray::insert(int i, const QJsonValue &value) void QJsonArray::replace(int i, const QJsonValue &value) { Q_ASSERT (a && i >= 0 && i < (int)(a->length)); + QJsonValue val = value; bool compressed; - int valueSize = QJsonPrivate::Value::requiredStorage(value, &compressed); + int valueSize = QJsonPrivate::Value::requiredStorage(val, &compressed); detach(valueSize); @@ -447,13 +452,16 @@ void QJsonArray::replace(int i, const QJsonValue &value) a->tableOffset = sizeof(QJsonPrivate::Array); int valueOffset = a->reserveSpace(valueSize, i, 1, true); + if (!valueOffset) + return; + QJsonPrivate::Value &v = (*a)[i]; - v.type = (value.t == QJsonValue::Undefined ? QJsonValue::Null : value.t); + v.type = (val.t == QJsonValue::Undefined ? QJsonValue::Null : val.t); v.latinOrIntValue = compressed; v.latinKey = false; - v.value = QJsonPrivate::Value::valueToStore(value, valueOffset); + v.value = QJsonPrivate::Value::valueToStore(val, valueOffset); if (valueSize) - QJsonPrivate::Value::copyData(value, (char *)a + valueOffset, compressed); + QJsonPrivate::Value::copyData(val, (char *)a + valueOffset, compressed); ++d->compactionCounter; if (d->compactionCounter > 32u && d->compactionCounter >= unsigned(a->length) / 2u) diff --git a/src/corelib/json/qjsondocument.h b/src/corelib/json/qjsondocument.h index 4d4f3885dc..0354262e2c 100644 --- a/src/corelib/json/qjsondocument.h +++ b/src/corelib/json/qjsondocument.h @@ -67,7 +67,8 @@ struct Q_CORE_EXPORT QJsonParseError IllegalUTF8String, UnterminatedString, MissingObject, - DeepNesting + DeepNesting, + DocumentTooLarge }; QString errorString() const; diff --git a/src/corelib/json/qjsonobject.cpp b/src/corelib/json/qjsonobject.cpp index 55c736afce..2be9d8891d 100644 --- a/src/corelib/json/qjsonobject.cpp +++ b/src/corelib/json/qjsonobject.cpp @@ -317,9 +317,10 @@ QJsonObject::iterator QJsonObject::insert(const QString &key, const QJsonValue & remove(key); return end(); } + QJsonValue val = value; bool latinOrIntValue; - int valueSize = QJsonPrivate::Value::requiredStorage(value, &latinOrIntValue); + int valueSize = QJsonPrivate::Value::requiredStorage(val, &latinOrIntValue); bool latinKey = QJsonPrivate::useCompressed(key); int valueOffset = sizeof(QJsonPrivate::Entry) + QJsonPrivate::qStringSize(key, latinKey); @@ -335,16 +336,21 @@ QJsonObject::iterator QJsonObject::insert(const QString &key, const QJsonValue & if (keyExists) ++d->compactionCounter; - o->reserveSpace(requiredSize, pos, 1, keyExists); + uint off = o->reserveSpace(requiredSize, pos, 1, keyExists); + if (!off) + return end(); QJsonPrivate::Entry *e = o->entryAt(pos); - e->value.type = value.t; + e->value.type = val.t; e->value.latinKey = latinKey; e->value.latinOrIntValue = latinOrIntValue; - e->value.value = QJsonPrivate::Value::valueToStore(value, (char *)e - (char *)o + valueOffset); + e->value.value = QJsonPrivate::Value::valueToStore(val, (char *)e - (char *)o + valueOffset); QJsonPrivate::copyString((char *)(e + 1), key, latinKey); if (valueSize) - QJsonPrivate::Value::copyData(value, (char *)e + valueOffset, latinOrIntValue); + QJsonPrivate::Value::copyData(val, (char *)e + valueOffset, latinOrIntValue); + + if (d->compactionCounter > 32u && d->compactionCounter >= unsigned(o->length) / 2u) + compact(); return iterator(this, pos); } diff --git a/src/corelib/json/qjsonparser.cpp b/src/corelib/json/qjsonparser.cpp index e569cbf435..7989d18901 100644 --- a/src/corelib/json/qjsonparser.cpp +++ b/src/corelib/json/qjsonparser.cpp @@ -76,6 +76,7 @@ QT_BEGIN_NAMESPACE #define JSONERR_UTERM_STR QT_TRANSLATE_NOOP("QJsonParseError", "unterminated string") #define JSONERR_MISS_OBJ QT_TRANSLATE_NOOP("QJsonParseError", "object is missing after a comma") #define JSONERR_DEEP_NEST QT_TRANSLATE_NOOP("QJsonParseError", "too deeply nested document") +#define JSONERR_DOC_LARGE QT_TRANSLATE_NOOP("QJsonParseError", "too large document") /*! \class QJsonParseError @@ -105,6 +106,7 @@ QT_BEGIN_NAMESPACE \value UnterminatedString A string wasn't terminated with a quote \value MissingObject An object was expected but couldn't be found \value DeepNesting The JSON document is too deeply nested for the parser to parse it + \value DocumentTooLarge The JSON document is too large for the parser to parse it */ /*! @@ -173,6 +175,9 @@ QString QJsonParseError::errorString() const case DeepNesting: sz = JSONERR_DEEP_NEST; break; + case DocumentTooLarge: + sz = JSONERR_DOC_LARGE; + break; } #ifndef QT_BOOTSTRAPPED return QCoreApplication::translate("QJsonParseError", sz); @@ -579,6 +584,10 @@ bool Parser::parseValue(QJsonPrivate::Value *val, int baseOffset) return false; case Quote: { val->type = QJsonValue::String; + if (current - baseOffset >= Value::MaxSize) { + lastError = QJsonParseError::DocumentTooLarge; + return false; + } val->value = current - baseOffset; bool latin1; if (!parseString(&latin1)) @@ -590,6 +599,10 @@ bool Parser::parseValue(QJsonPrivate::Value *val, int baseOffset) } case BeginArray: val->type = QJsonValue::Array; + if (current - baseOffset >= Value::MaxSize) { + lastError = QJsonParseError::DocumentTooLarge; + return false; + } val->value = current - baseOffset; if (!parseArray()) return false; @@ -598,6 +611,10 @@ bool Parser::parseValue(QJsonPrivate::Value *val, int baseOffset) return true; case BeginObject: val->type = QJsonValue::Object; + if (current - baseOffset >= Value::MaxSize) { + lastError = QJsonParseError::DocumentTooLarge; + return false; + } val->value = current - baseOffset; if (!parseObject()) return false; @@ -707,6 +724,10 @@ bool Parser::parseNumber(QJsonPrivate::Value *val, int baseOffset) int pos = reserveSpace(sizeof(double)); *(quint64 *)(data + pos) = qToLittleEndian(ui); + if (current - baseOffset >= Value::MaxSize) { + lastError = QJsonParseError::DocumentTooLarge; + return false; + } val->value = pos - baseOffset; val->latinOrIntValue = false;