AuroraMiddleware/qt5base-lts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix 64-bit integer support in QtJSON

Browse Source 
Fixes parsing writing and pass-through of integers with
higher precision than double can handle.

Note this adds extra precision compared to JavaScript, but the
JSON files read and written this way are still valid, and the extra
precision in reading and writing this way is used by many JSON
libraries.

Fixes: QTBUG-28560
Change-Id: I30b2415c928d1c34c8cb4e4c6218602095e7e8aa
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>
This commit is contained in:
Allan Sandfeld Jensen 2019-10-21 10:26:45 +02:00
parent e99b0016e8
commit edec095cf8
8 changed files with 169 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/corelib/global/qnumeric_p.h
View File

@ -202,7 +202,7 @@ namespace {
This function works for v containing infinities, but not NaN. It's the
caller's responsibility to exclude that possibility before calling it.
*/
template <typename T> static inline bool convertDoubleTo(double v, T *value)
template <typename T> static inline bool convertDoubleTo(double v, T *value, bool allow_precision_upgrade = true)
{
Q_STATIC_ASSERT(std::numeric_limits<T>::is_integer);
@ -227,6 +227,10 @@ template <typename T> static inline bool convertDoubleTo(double v, T *value)
supremum = -2.0 * std::numeric_limits<ST>::min(); // -2 * (-2^63) = 2^64, exact (for T = quint64)
v = fabs(v);
}
if (std::is_integral<T>::value && sizeof(T) > 4 && !allow_precision_upgrade) {
if (v > double(Q_INT64_C(1)<<53) || v < double(-((Q_INT64_C(1)<<53) + 1)))
return false;
}
*value = std::numeric_limits<T>::max();
if (v >= supremum)

22
src/corelib/serialization/qjsoncbor.cpp
View File

@ -262,8 +262,7 @@ QJsonValue qt_convertToJson(QCborContainerPrivate *d, qsizetype idx)
const auto &e = d->elements.at(idx);
switch (e.type) {
case QCborValue::Integer:
return QJsonPrivate::Value::fromTrustedCbor(e.value);
return QJsonValue(e.value);
case QCborValue::ByteArray:
case QCborValue::String:
case QCborValue::SimpleType:
@ -370,7 +369,7 @@ QJsonValue QCborValue::toJsonValue() const
return false;
case Integer:
return QJsonPrivate::Value::fromTrustedCbor(n);
return QJsonPrivate::Value::fromTrustedCbor(*this);
case True:
return true;
@ -615,11 +614,9 @@ QCborValue QCborValue::fromJsonValue(const QJsonValue &v)
case QJsonValue::Bool:
return v.toBool();
case QJsonValue::Double: {
qint64 i;
const double dbl = v.toDouble();
if (convertDoubleTo(dbl, &i))
return i;
return dbl;
if (v.t == Integer)
return v.toInteger();
return v.toDouble();
}
case QJsonValue::String:
return v.toString();
@ -667,9 +664,7 @@ static void appendVariant(QCborContainerPrivate *d, const QVariant &variant)
\row \li \c bool \li Bool
\row \li \c std::nullptr_t \li Null
\row \li \c short, \c ushort, \c int, \c uint, \l qint64 \li Integer
\row \li \l quint64 \li Integer, but they are cast to \c qint64 first so
values higher than 2\sup{63}-1 (\c INT64_MAX) will
be wrapped to negative
\row \li \l quint64 \li Integer, or Double if outside the range of qint64
\row \li \c float, \c double \li Double
\row \li \l QByteArray \li ByteArray
\row \li \l QDateTime \li DateTime
@ -713,9 +708,12 @@ QCborValue QCborValue::fromVariant(const QVariant &variant)
case QMetaType::UShort:
case QVariant::Int:
case QVariant::LongLong:
case QVariant::ULongLong:
case QVariant::UInt:
return variant.toLongLong();
case QVariant::ULongLong:
if (variant.toULongLong() <= static_cast<uint64_t>(std::numeric_limits<qint64>::max()))
return variant.toLongLong();
Q_FALLTHROUGH();
case QMetaType::Float:
case QVariant::Double:
return variant.toDouble();

5
src/corelib/serialization/qjsonparser.cpp
View File

@ -709,10 +709,11 @@ bool Parser::parseNumber()
// frac = decimal-point 1*DIGIT
if (json < end && *json == '.') {
isInt = false;
++json;
while (json < end && *json >= '0' && *json <= '9')
while (json < end && *json >= '0' && *json <= '9') {
isInt = isInt && *json == '0';
++json;
}
}
// exp = e [ minus / plus ] 1*DIGIT

92
src/corelib/serialization/qjsonvalue.cpp
View File

@ -154,7 +154,9 @@ QJsonValue::QJsonValue(bool b)
QJsonValue::QJsonValue(double v)
: d(nullptr)
{
if (convertDoubleTo(v, &n)) {
// Convert to integer if the number is an integer and changing wouldn't
// introduce additional digit precision not present in the double.
if (convertDoubleTo<qint64>(v, &n, false /* allow_precision_upgrade */)) {
t = QCborValue::Integer;
} else {
memcpy(&n, &v, sizeof(n));
@ -449,12 +451,18 @@ QJsonValue QJsonValue::fromVariant(const QVariant &variant)
return QJsonValue(Null);
case QVariant::Bool:
return QJsonValue(variant.toBool());
case QMetaType::Short:
case QMetaType::UShort:
case QVariant::Int:
case QVariant::UInt:
case QVariant::LongLong:
return QJsonValue(variant.toLongLong());
case QVariant::ULongLong:
if (variant.toULongLong() <= static_cast<uint64_t>(std::numeric_limits<qint64>::max()))
return QJsonValue(variant.toLongLong());
Q_FALLTHROUGH();
case QMetaType::Float:
case QVariant::Double:
case QVariant::LongLong:
case QVariant::ULongLong:
case QVariant::UInt:
return QJsonValue(variant.toDouble());
case QVariant::String:
return QJsonValue(variant.toString());
@ -504,7 +512,7 @@ QJsonValue QJsonValue::fromVariant(const QVariant &variant)
\value Null QMetaType::Nullptr
\value Bool QMetaType::Bool
\value Double QMetaType::Double
\value Double QMetaType::Double or QMetaType::LongLong
\value String QString
\value Array QVariantList
\value Object QVariantMap
@ -520,6 +528,7 @@ QVariant QJsonValue::toVariant() const
case QCborValue::False:
return false;
case QCborValue::Integer:
return toInteger();
case QCborValue::Double:
return toDouble();
case QCborValue::String:
@ -548,7 +557,8 @@ QVariant QJsonValue::toVariant() const
\value Null A Null value
\value Bool A boolean value. Use toBool() to convert to a bool.
\value Double A double. Use toDouble() to convert to a double.
\value Double A number value. Use toDouble() to convert to a double,
or toInteger() to convert to a qint64.
\value String A string. Use toString() to convert to a QString.
\value Array An array. Use toArray() to convert to a QJsonArray.
\value Object An object. Use toObject() to convert to a QJsonObject.
@ -613,18 +623,43 @@ int QJsonValue::toInt(int defaultValue) const
{
switch (t) {
case QCborValue::Double: {
const double dbl = toDouble();
int dblInt;
convertDoubleTo<int>(dbl, &dblInt);
return dbl == dblInt ? dblInt : defaultValue;
if (convertDoubleTo<int>(toDouble(), &dblInt))
return dblInt;
break;
}
case QCborValue::Integer:
return (n <= qint64(std::numeric_limits<int>::max())
&& n >= qint64(std::numeric_limits<int>::min()))
? n : defaultValue;
if (qint64(int(n)) == n)
return int(n);
break;
default:
return defaultValue;
break;
}
return defaultValue;
}
/*!
\since 6.0
Converts the value to an integer and returns it.
If type() is not Double or the value is not a whole number
representable as qint64, the \a defaultValue will be returned.
*/
qint64 QJsonValue::toInteger(qint64 defaultValue) const
{
switch (t) {
case QCborValue::Integer:
return n;
case QCborValue::Double: {
qint64 dblInt;
if (convertDoubleTo<qint64>(toDouble(), &dblInt))
return dblInt;
break;
}
default:
break;
}
return defaultValue;
}
/*!
@ -641,7 +676,7 @@ double QJsonValue::toDouble(double defaultValue) const
return d;
}
case QCborValue::Integer:
return n;
return double(n);
default:
return defaultValue;
}
@ -787,8 +822,13 @@ const QJsonValue QJsonValue::operator[](int i) const
*/
bool QJsonValue::operator==(const QJsonValue &other) const
{
if (t != other.t)
if (t != other.t) {
if (isDouble() && other.isDouble()) {
// One value Cbor integer, one Cbor double, should interact as doubles.
return toDouble() == other.toDouble();
}
return false;
}
switch (t) {
case QCborValue::Undefined:
@ -929,32 +969,38 @@ uint qHash(const QJsonValue &value, uint seed)
QDebug operator<<(QDebug dbg, const QJsonValue &o)
{
QDebugStateSaver saver(dbg);
switch (o.type()) {
case QJsonValue::Undefined:
switch (o.t) {
case QCborValue::Undefined:
dbg << "QJsonValue(undefined)";
break;
case QJsonValue::Null:
case QCborValue::Null:
dbg << "QJsonValue(null)";
break;
case QJsonValue::Bool:
case QCborValue::True:
case QCborValue::False:
dbg.nospace() << "QJsonValue(bool, " << o.toBool() << ')';
break;
case QJsonValue::Double:
case QCborValue::Integer:
dbg.nospace() << "QJsonValue(double, " << o.toInteger() << ')';
break;
case QCborValue::Double:
dbg.nospace() << "QJsonValue(double, " << o.toDouble() << ')';
break;
case QJsonValue::String:
case QCborValue::String:
dbg.nospace() << "QJsonValue(string, " << o.toString() << ')';
break;
case QJsonValue::Array:
case QCborValue::Array:
dbg.nospace() << "QJsonValue(array, ";
dbg << o.toArray();
dbg << ')';
break;
case QJsonValue::Object:
case QCborValue::Map:
dbg.nospace() << "QJsonValue(object, ";
dbg << o.toObject();
dbg << ')';
break;
default:
Q_UNREACHABLE();
}
return dbg;
}

1
src/corelib/serialization/qjsonvalue.h
View File

@ -112,6 +112,7 @@ public:
bool toBool(bool defaultValue = false) const;
int toInt(int defaultValue = 0) const;
qint64 toInteger(qint64 defaultValue = 0) const;
double toDouble(double defaultValue = 0) const;
QString toString() const;
QString toString(const QString &defaultValue) const;

11
src/corelib/serialization/qjsonwriter.cpp
View File

@ -138,15 +138,14 @@ static void valueToJson(const QCborValue &v, QByteArray &json, int indent, bool
json += "false";
break;
case QCborValue::Integer:
json += QByteArray::number(v.toInteger());
break;
case QCborValue::Double: {
const double d = v.toDouble();
if (qIsFinite(d)) {
quint64 absInt;
json += QByteArray::number(d, convertDoubleTo(std::abs(d), &absInt) ? 'f' : 'g',
QLocale::FloatingPointShortest);
} else {
if (qIsFinite(d))
json += QByteArray::number(d, 'g', QLocale::FloatingPointShortest);
else
json += "null"; // +INF || -INF || NaN (see RFC4627#section2.4)
}
break;
}
case QCborValue::String:

78
tests/auto/corelib/serialization/json/tst_qtjson.cpp
View File

@ -52,6 +52,8 @@ private Q_SLOTS:
void testNumbers_3();
void testNumbers_4();
void testNumberComparisons();
void testObjectSimple();
void testObjectSmallKeys();
void testArraySimple();
@ -218,26 +220,32 @@ void tst_QtJson::testNumbers()
{
int numbers[] = {
0,
-1,
1,
2,
-1,
-2,
(1<<25),
(1<<26),
(1<<27),
(1<<28),
-(1<<25),
-(1<<26),
-(1<<27),
-(1<<28),
(1<<26) - 1,
(1<<27) - 1,
(1<<28) - 1,
(1<<29) - 1,
-((1<<26) - 1),
-((1<<27) - 1),
-((1<<28) - 1)
-((1<<28) - 1),
-((1<<29) - 1)
};
int n = sizeof(numbers)/sizeof(int);
QJsonArray array;
for (int i = 0; i < n; ++i)
array.append((double)numbers[i]);
array.append(numbers[i]);
QByteArray serialized = QJsonDocument(array).toJson();
QJsonDocument json = QJsonDocument::fromJson(serialized);
@ -246,8 +254,10 @@ void tst_QtJson::testNumbers()
QCOMPARE(array.size(), array2.size());
for (int i = 0; i < array.size(); ++i) {
QCOMPARE(array.at(i).type(), QJsonValue::Double);
QCOMPARE(array.at(i).toInt(), numbers[i]);
QCOMPARE(array.at(i).toDouble(), (double)numbers[i]);
QCOMPARE(array2.at(i).type(), QJsonValue::Double);
QCOMPARE(array2.at(i).toInt(), numbers[i]);
QCOMPARE(array2.at(i).toDouble(), (double)numbers[i]);
}
}
@ -255,8 +265,10 @@ void tst_QtJson::testNumbers()
{
qint64 numbers[] = {
0,
-1,
1,
2,
-1,
-2,
(1ll<<54),
(1ll<<55),
(1ll<<56),
@ -266,15 +278,21 @@ void tst_QtJson::testNumbers()
(1ll<<54) - 1,
(1ll<<55) - 1,
(1ll<<56) - 1,
(1ll<<57) - 1,
(1ll<<58) - 1,
(1ll<<59) + 1001,
-((1ll<<54) - 1),
-((1ll<<55) - 1),
-((1ll<<56) - 1)
-((1ll<<56) - 1),
-((1ll<<57) - 1),
-((1ll<<58) - 1),
-((1ll<<59) + 1001),
};
int n = sizeof(numbers)/sizeof(qint64);
QJsonArray array;
for (int i = 0; i < n; ++i)
array.append((double)numbers[i]);
array.append(QJsonValue(numbers[i]));
QByteArray serialized = QJsonDocument(array).toJson();
QJsonDocument json = QJsonDocument::fromJson(serialized);
@ -283,8 +301,10 @@ void tst_QtJson::testNumbers()
QCOMPARE(array.size(), array2.size());
for (int i = 0; i < array.size(); ++i) {
QCOMPARE(array.at(i).type(), QJsonValue::Double);
QCOMPARE(array.at(i).toInteger(), numbers[i]);
QCOMPARE(array.at(i).toDouble(), (double)numbers[i]);
QCOMPARE(array2.at(i).type(), QJsonValue::Double);
QCOMPARE(array2.at(i).toInteger(), numbers[i]);
QCOMPARE(array2.at(i).toDouble(), (double)numbers[i]);
}
}
@ -422,6 +442,46 @@ void tst_QtJson::testNumbers_4()
" -18446744073709552000\n"
"]\n";
QCOMPARE(json, expected);
QJsonArray array2;
array2 << QJsonValue(Q_INT64_C(+1000000000000000));
array2 << QJsonValue(Q_INT64_C(-1000000000000000));
array2 << QJsonValue(Q_INT64_C(+9007199254740992));
array2 << QJsonValue(Q_INT64_C(-9007199254740992));
array2 << QJsonValue(Q_INT64_C(+9223372036854775807));
array2 << QJsonValue(Q_INT64_C(-9223372036854775807));
const QByteArray json2(QJsonDocument(array2).toJson());
const QByteArray expected2 =
"[\n"
" 1000000000000000,\n"
" -1000000000000000,\n"
" 9007199254740992,\n"
" -9007199254740992,\n"
" 9223372036854775807,\n"
" -9223372036854775807\n"
"]\n";
QCOMPARE(json2, expected2);
}
void tst_QtJson::testNumberComparisons()
{
// QJsonValues created using doubles only have double precision
QJsonValue llMinDbl(-9223372036854775807.0);
QJsonValue llMinPlus1Dbl(-9223372036854775806.0);
QCOMPARE(llMinDbl == llMinPlus1Dbl, -9223372036854775807.0 == -9223372036854775806.0); // true
// QJsonValues created using qint64 have full qint64 precision
QJsonValue llMin(Q_INT64_C(-9223372036854775807));
QJsonValue llMinPlus1(Q_INT64_C(-9223372036854775806));
QCOMPARE(llMin == llMinPlus1, Q_INT64_C(-9223372036854775807) == Q_INT64_C(-9223372036854775806)); // false
// The different storage formats should be able to compare as their C++ versions (all true)
QCOMPARE(llMin == llMinDbl, Q_INT64_C(-9223372036854775807) == -9223372036854775807.0);
QCOMPARE(llMinDbl == llMin, -9223372036854775807.0 == Q_INT64_C(-9223372036854775807));
QCOMPARE(llMinPlus1 == llMinPlus1Dbl, Q_INT64_C(-9223372036854775806) == -9223372036854775806.0);
QCOMPARE(llMinPlus1Dbl == llMinPlus1, -9223372036854775806.0 == Q_INT64_C(-9223372036854775806));
QCOMPARE(llMinPlus1 == llMinDbl, Q_INT64_C(-9223372036854775806) == -9223372036854775807.0);
QCOMPARE(llMinPlus1Dbl == llMin, -9223372036854775806.0 == Q_INT64_C(-9223372036854775807));
}
void tst_QtJson::testObjectSimple()
@ -1160,8 +1220,8 @@ void tst_QtJson::fromVariant_data()
bool boolValue = true;
int intValue = -1;
uint uintValue = 1;
long long longlongValue = -2;
unsigned long long ulonglongValue = 2;
qlonglong longlongValue = -2;
qulonglong ulonglongValue = 2;
float floatValue = 3.3f;
double doubleValue = 4.4;
QString stringValue("str");
@ -1207,7 +1267,7 @@ void tst_QtJson::fromVariant_data()
QTest::newRow("nullptr") << QVariant::fromValue(nullptr) << QJsonValue(QJsonValue::Null);
QTest::newRow("bool") << QVariant(boolValue) << QJsonValue(boolValue);
QTest::newRow("int") << QVariant(intValue) << QJsonValue(intValue);
QTest::newRow("uint") << QVariant(uintValue) << QJsonValue(static_cast<double>(uintValue));
QTest::newRow("uint") << QVariant(uintValue) << QJsonValue(static_cast<qint64>(uintValue));
QTest::newRow("longlong") << QVariant(longlongValue) << QJsonValue(longlongValue);
QTest::newRow("ulonglong") << QVariant(ulonglongValue) << QJsonValue(static_cast<double>(ulonglongValue));
QTest::newRow("float") << QVariant(floatValue) << QJsonValue(floatValue);

15
tests/auto/corelib/serialization/qcborvalue_json/tst_qcborvalue_json.cpp
View File

@ -97,9 +97,12 @@ void tst_QCborValue_Json::toVariant_data()
add(1, 1, 1);
add(-1, -1, -1);
add(0., 0., 0.);
add(2., 2., 2.);
add(1.25, 1.25, 1.25);
add(-1.25, -1.25, -1.25);
add("Hello", "Hello", "Hello");
add(std::numeric_limits<qint64>::max(), std::numeric_limits<qint64>::max(), std::numeric_limits<qint64>::max());
add(std::numeric_limits<qint64>::min(), std::numeric_limits<qint64>::min(), std::numeric_limits<qint64>::min());
// converts to string in JSON:
add(QByteArray("Hello"), QByteArray("Hello"), "SGVsbG8");
@ -123,14 +126,6 @@ void tst_QCborValue_Json::toVariant_data()
<< QVariant(qQNaN())
<< QJsonValue();
// large integral values lose precision in JSON
QTest::newRow("Integer:max") << QCborValue(std::numeric_limits<qint64>::max())
<< QVariant(std::numeric_limits<qint64>::max())
<< QJsonValue(std::numeric_limits<qint64>::max());
QTest::newRow("Integer:min") << QCborValue(std::numeric_limits<qint64>::min())
<< QVariant(std::numeric_limits<qint64>::min())
<< QJsonValue(std::numeric_limits<qint64>::min());
// empty arrays and maps
add(QCborArray(), QVariantList(), QJsonArray());
add(QCborMap(), QVariantMap(), QJsonObject());
@ -257,6 +252,10 @@ void tst_QCborValue_Json::fromJson_data()
QTest::newRow("0") << QCborValue(0) << QJsonValue(0.);
QTest::newRow("1") << QCborValue(1) << QJsonValue(1);
QTest::newRow("1.5") << QCborValue(1.5) << QJsonValue(1.5);
QTest::newRow("Integer:max") << QCborValue(std::numeric_limits<qint64>::max())
<< QJsonValue(std::numeric_limits<qint64>::max());
QTest::newRow("Integer:min") << QCborValue(std::numeric_limits<qint64>::min())
<< QJsonValue(std::numeric_limits<qint64>::min());
QTest::newRow("string") << QCborValue("Hello") << QJsonValue("Hello");
QTest::newRow("array") << QCborValue(QCborValue::Array) << QJsonValue(QJsonValue::Array);
QTest::newRow("map") << QCborValue(QCborValue::Map) << QJsonValue(QJsonValue::Object);