From 44fe284f9def19a6f3957ebfc4dd6d34140cdcc6 Mon Sep 17 00:00:00 2001 From: chenguoping Date: Wed, 13 May 2020 20:29:10 +0800 Subject: [PATCH 1/8] Enhace to_cbor() to support +/-Infinity, NaN, and single-precision float --- .../nlohmann/detail/output/binary_writer.hpp | 30 +++++++++++++++-- include/nlohmann/json.hpp | 2 +- single_include/nlohmann/json.hpp | 32 +++++++++++++++++-- 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index f1f901ee1..4b0d141d2 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -6,6 +6,7 @@ #include // memcpy #include // numeric_limits #include // string +#include // isnan, isinf #include #include @@ -177,8 +178,33 @@ class binary_writer case value_t::number_float: { - oa->write_character(get_cbor_float_prefix(j.m_value.number_float)); - write_number(j.m_value.number_float); + if (std::isnan(j.m_value.number_float)) + { + // NaN is 0xf97e00 in CBOR + oa->write_character(to_char_type(0xF9)); + oa->write_character(to_char_type(0x7E)); + oa->write_character(to_char_type(0x00)); + } + else if (std::isinf(j.m_value.number_float)) + { + // Infinity is 0xf97c00, -Infinity is 0xf9fc00 + oa->write_character(to_char_type(0xf9)); + oa->write_character(j.m_value.number_float > 0 ? to_char_type(0x7C) : to_char_type(0xFC)); + oa->write_character(to_char_type(0x00)); + } + else + { + if (static_cast(static_cast(j.m_value.number_float)) == j.m_value.number_float) + { + oa->write_character(get_cbor_float_prefix(static_cast(j.m_value.number_float))); + write_number(static_cast(j.m_value.number_float)); + } + else + { + oa->write_character(get_cbor_float_prefix(j.m_value.number_float)); + write_number(j.m_value.number_float); + } + } break; } diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 9b26dd704..1a06b8d31 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -7059,7 +7059,7 @@ class basic_json - break (0xFF) @param[in] j JSON value to serialize - @return MessagePack serialization as byte vector + @return CBOR serialization as byte vector @complexity Linear in the size of the JSON value @a j. diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index abec8d3f1..30c353dae 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -11828,6 +11828,7 @@ class json_ref #include // memcpy #include // numeric_limits #include // string +#include // isnan, isinf // #include @@ -12126,8 +12127,33 @@ class binary_writer case value_t::number_float: { - oa->write_character(get_cbor_float_prefix(j.m_value.number_float)); - write_number(j.m_value.number_float); + if (std::isnan(j.m_value.number_float)) + { + // NaN is 0xf97e00 in CBOR + oa->write_character(to_char_type(0xF9)); + oa->write_character(to_char_type(0x7E)); + oa->write_character(to_char_type(0x00)); + } + else if (std::isinf(j.m_value.number_float)) + { + // Infinity is 0xf97c00, -Infinity is 0xf9fc00 + oa->write_character(to_char_type(0xf9)); + oa->write_character(j.m_value.number_float > 0 ? to_char_type(0x7C) : to_char_type(0xFC)); + oa->write_character(to_char_type(0x00)); + } + else + { + if (static_cast(static_cast(j.m_value.number_float)) == j.m_value.number_float) + { + oa->write_character(get_cbor_float_prefix(static_cast(j.m_value.number_float))); + write_number(static_cast(j.m_value.number_float)); + } + else + { + oa->write_character(get_cbor_float_prefix(j.m_value.number_float)); + write_number(j.m_value.number_float); + } + } break; } @@ -22507,7 +22533,7 @@ class basic_json - break (0xFF) @param[in] j JSON value to serialize - @return MessagePack serialization as byte vector + @return CBOR serialization as byte vector @complexity Linear in the size of the JSON value @a j. From 47c6570470eb7657a7e65373a93d9f408179e8d1 Mon Sep 17 00:00:00 2001 From: chenguoping Date: Wed, 13 May 2020 20:30:49 +0800 Subject: [PATCH 2/8] Add some test cases about to_cbor() --- test/src/unit-cbor.cpp | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/test/src/unit-cbor.cpp b/test/src/unit-cbor.cpp index 710e7af81..40fe109ac 100644 --- a/test/src/unit-cbor.cpp +++ b/test/src/unit-cbor.cpp @@ -130,6 +130,24 @@ TEST_CASE("CBOR") CHECK(result.empty()); } + SECTION("NaN") + { + // NaN value + json j = std::numeric_limits::quiet_NaN(); + std::vector expected = {0xf9, 0x7e, 0x00}; + const auto result = json::to_cbor(j); + CHECK(result == expected); + } + + SECTION("Infinity") + { + // Infinity value + json j = std::numeric_limits::infinity(); + std::vector expected = {0xf9, 0x7c, 0x00}; + const auto result = json::to_cbor(j); + CHECK(result == expected); + } + SECTION("null") { json j = nullptr; @@ -835,6 +853,20 @@ TEST_CASE("CBOR") CHECK(json::from_cbor(result, true, false) == j); } + SECTION("0.5") + { + double v = 0.5; + json j = v; + // its double-precision float binary value is + // {0xfb, 0x3f, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} + // but to save memory, we can store it as single-precision float. + std::vector expected = {0xfa, 0x3f, 0x00, 0x00, 0x00}; + const auto result = json::to_cbor(j); + CHECK(result == expected); + // roundtrip + CHECK(json::from_cbor(result) == j); + CHECK(json::from_cbor(result) == v); + } } SECTION("half-precision float (edge cases)") @@ -936,7 +968,7 @@ TEST_CASE("CBOR") SECTION("NaN") { - json j = json::from_cbor(std::vector({0xf9, 0x7c, 0x01})); + json j = json::from_cbor(std::vector({0xf9, 0x7e, 0x00})); json::number_float_t d = j; CHECK(std::isnan(d)); CHECK(j.dump() == "null"); From 8f5b5c7469b955b8829a43f1d374007500826971 Mon Sep 17 00:00:00 2001 From: chenguoping Date: Wed, 13 May 2020 20:31:33 +0800 Subject: [PATCH 3/8] use json_test_data version 2.0.0 --- cmake/download_test_data.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/download_test_data.cmake b/cmake/download_test_data.cmake index c6dc135b7..aa0d53208 100644 --- a/cmake/download_test_data.cmake +++ b/cmake/download_test_data.cmake @@ -1,7 +1,7 @@ find_package(Git) set(JSON_TEST_DATA_URL https://github.com/nlohmann/json_test_data) -set(JSON_TEST_DATA_VERSION 1.0.0) +set(JSON_TEST_DATA_VERSION 2.0.0) # target to download test data add_custom_target(download_test_data From e175150f5b3aae9eacc4bcdc01b0e0642a7534f7 Mon Sep 17 00:00:00 2001 From: chenguoping Date: Thu, 14 May 2020 20:54:47 +0800 Subject: [PATCH 4/8] fix UBSAN --- include/nlohmann/detail/output/binary_writer.hpp | 3 ++- single_include/nlohmann/json.hpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 4b0d141d2..c507134ce 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -194,7 +194,8 @@ class binary_writer } else { - if (static_cast(static_cast(j.m_value.number_float)) == j.m_value.number_float) + if (j.m_value.number_float < std::numeric_limits::max() and + static_cast(static_cast(j.m_value.number_float)) == j.m_value.number_float) { oa->write_character(get_cbor_float_prefix(static_cast(j.m_value.number_float))); write_number(static_cast(j.m_value.number_float)); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 30c353dae..6c339c846 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -12143,7 +12143,8 @@ class binary_writer } else { - if (static_cast(static_cast(j.m_value.number_float)) == j.m_value.number_float) + if (j.m_value.number_float < std::numeric_limits::max() and + static_cast(static_cast(j.m_value.number_float)) == j.m_value.number_float) { oa->write_character(get_cbor_float_prefix(static_cast(j.m_value.number_float))); write_number(static_cast(j.m_value.number_float)); From bcf4f3ce9a4ab1f31829bfa9446cbeaaeb644e42 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Thu, 14 May 2020 19:06:48 +0200 Subject: [PATCH 5/8] :memo: add warning for items() on temporary objects #2040 --- include/nlohmann/json.hpp | 5 +++++ single_include/nlohmann/json.hpp | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 6801326af..b72ea58d4 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -4946,6 +4946,11 @@ class basic_json element as string (see example). For primitive types (e.g., numbers), `key()` returns an empty string. + @warning Using `items()` on temporary objects is dangerous. Make sure the + object's lifetime exeeds the iteration. See + for more + information. + @return iteration proxy object wrapping @a ref with an interface to use in range-based for loops diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 12945016a..c21a4308d 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -20437,6 +20437,11 @@ class basic_json element as string (see example). For primitive types (e.g., numbers), `key()` returns an empty string. + @warning Using `items()` on temporary objects is dangerous. Make sure the + object's lifetime exeeds the iteration. See + for more + information. + @return iteration proxy object wrapping @a ref with an interface to use in range-based for loops From 779a0ec7df1a08f3efeed79d39039f78f0ca8cc1 Mon Sep 17 00:00:00 2001 From: chenguoping Date: Fri, 15 May 2020 17:35:43 +0800 Subject: [PATCH 6/8] update --- .../nlohmann/detail/output/binary_writer.hpp | 3 +- single_include/nlohmann/json.hpp | 3 +- test/src/unit-cbor.cpp | 58 ++++++++++++++++++- 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index c507134ce..b8b5a5724 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -194,7 +194,8 @@ class binary_writer } else { - if (j.m_value.number_float < std::numeric_limits::max() and + if (j.m_value.number_float > -std::numeric_limits::min() and + j.m_value.number_float < std::numeric_limits::max() and static_cast(static_cast(j.m_value.number_float)) == j.m_value.number_float) { oa->write_character(get_cbor_float_prefix(static_cast(j.m_value.number_float))); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 6c339c846..23ae81052 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -12143,7 +12143,8 @@ class binary_writer } else { - if (j.m_value.number_float < std::numeric_limits::max() and + if (j.m_value.number_float > -std::numeric_limits::min() and + j.m_value.number_float < std::numeric_limits::max() and static_cast(static_cast(j.m_value.number_float)) == j.m_value.number_float) { oa->write_character(get_cbor_float_prefix(static_cast(j.m_value.number_float))); diff --git a/test/src/unit-cbor.cpp b/test/src/unit-cbor.cpp index 40fe109ac..531f56d77 100644 --- a/test/src/unit-cbor.cpp +++ b/test/src/unit-cbor.cpp @@ -834,7 +834,7 @@ TEST_CASE("CBOR") } } - SECTION("float") + SECTION("double-precision float") { SECTION("3.1415925") { @@ -853,6 +853,10 @@ TEST_CASE("CBOR") CHECK(json::from_cbor(result, true, false) == j); } + } + + SECTION("single-precision float") + { SECTION("0.5") { double v = 0.5; @@ -867,6 +871,58 @@ TEST_CASE("CBOR") CHECK(json::from_cbor(result) == j); CHECK(json::from_cbor(result) == v); } + SECTION("0.0") + { + double v = 0.0; + json j = v; + // its double-precision binary value is: + // {0xfb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} + std::vector expected = {0xfa, 0x00, 0x00, 0x00, 0x00}; + const auto result = json::to_cbor(j); + CHECK(result == expected); + // roundtrip + CHECK(json::from_cbor(result) == j); + CHECK(json::from_cbor(result) == v); + } + SECTION("-0.0") + { + double v = -0.0; + json j = v; + // its double-precision binary value is: + // {0xfb, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} + std::vector expected = {0xfa, 0x80, 0x00, 0x00, 0x00}; + const auto result = json::to_cbor(j); + CHECK(result == expected); + // roundtrip + CHECK(json::from_cbor(result) == j); + CHECK(json::from_cbor(result) == v); + } + SECTION("100.0") + { + double v = 100.0; + json j = v; + // its double-precision binary value is: + // {0xfb, 0x40, 0x59, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} + std::vector expected = {0xfa, 0x42, 0xc8, 0x00, 0x00}; + const auto result = json::to_cbor(j); + CHECK(result == expected); + // roundtrip + CHECK(json::from_cbor(result) == j); + CHECK(json::from_cbor(result) == v); + } + SECTION("200.0") + { + double v = 200.0; + json j = v; + // its double-precision binary value is: + // {0xfb, 0x40, 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} + std::vector expected = {0xfa, 0x43, 0x48, 0x00, 0x00}; + const auto result = json::to_cbor(j); + CHECK(result == expected); + // roundtrip + CHECK(json::from_cbor(result) == j); + CHECK(json::from_cbor(result) == v); + } } SECTION("half-precision float (edge cases)") From 5dd27f1a9f8e9ff480ebbfea5e6fd4de40467dad Mon Sep 17 00:00:00 2001 From: chenguoping Date: Fri, 15 May 2020 21:47:59 +0800 Subject: [PATCH 7/8] compare against max float and min float before converting --- include/nlohmann/detail/output/binary_writer.hpp | 4 ++-- single_include/nlohmann/json.hpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index b8b5a5724..163e88d99 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -194,8 +194,8 @@ class binary_writer } else { - if (j.m_value.number_float > -std::numeric_limits::min() and - j.m_value.number_float < std::numeric_limits::max() and + if (j.m_value.number_float >= std::numeric_limits::lowest() and + j.m_value.number_float <= std::numeric_limits::max() and static_cast(static_cast(j.m_value.number_float)) == j.m_value.number_float) { oa->write_character(get_cbor_float_prefix(static_cast(j.m_value.number_float))); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 23ae81052..b30e45600 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -12143,8 +12143,8 @@ class binary_writer } else { - if (j.m_value.number_float > -std::numeric_limits::min() and - j.m_value.number_float < std::numeric_limits::max() and + if (j.m_value.number_float >= std::numeric_limits::lowest() and + j.m_value.number_float <= std::numeric_limits::max() and static_cast(static_cast(j.m_value.number_float)) == j.m_value.number_float) { oa->write_character(get_cbor_float_prefix(static_cast(j.m_value.number_float))); From ed9c205b5dbeabd21b563ebf8639aa8e39d2d320 Mon Sep 17 00:00:00 2001 From: chenguoping Date: Fri, 15 May 2020 21:48:29 +0800 Subject: [PATCH 8/8] add somes test cases --- test/src/unit-cbor.cpp | 59 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/test/src/unit-cbor.cpp b/test/src/unit-cbor.cpp index 531f56d77..280032587 100644 --- a/test/src/unit-cbor.cpp +++ b/test/src/unit-cbor.cpp @@ -923,6 +923,65 @@ TEST_CASE("CBOR") CHECK(json::from_cbor(result) == j); CHECK(json::from_cbor(result) == v); } + SECTION("3.40282e+38(max float)") + { + float v = std::numeric_limits::max(); + json j = v; + std::vector expected = + { + 0xfa, 0x7f, 0x7f, 0xff, 0xff + }; + const auto result = json::to_cbor(j); + CHECK(result == expected); + // roundtrip + CHECK(json::from_cbor(result) == j); + CHECK(json::from_cbor(result) == v); + } + SECTION("-3.40282e+38(lowest float)") + { + double v = std::numeric_limits::lowest(); + json j = v; + std::vector expected = + { + 0xfa, 0xff, 0x7f, 0xff, 0xff + }; + const auto result = json::to_cbor(j); + CHECK(result == expected); + // roundtrip + CHECK(json::from_cbor(result) == j); + CHECK(json::from_cbor(result) == v); + } + SECTION("1 + 3.40282e+38(more than max float)") + { + double v = std::numeric_limits::max() + 0.1e+34; + json j = v; + std::vector expected = + { + 0xfb, 0x47, 0xf0, 0x00, 0x03, 0x04, 0xdc, 0x64, 0x49 + }; + // double + const auto result = json::to_cbor(j); + CHECK(result == expected); + // roundtrip + CHECK(json::from_cbor(result) == j); + CHECK(json::from_cbor(result) == v); + } + SECTION("-1 - 3.40282e+38(less than lowest float)") + { + double v = std::numeric_limits::lowest() - 1; + json j = v; + std::vector expected = + { + 0xfa, 0xff, 0x7f, 0xff, 0xff + }; + // the same with lowest float + const auto result = json::to_cbor(j); + CHECK(result == expected); + // roundtrip + CHECK(json::from_cbor(result) == j); + CHECK(json::from_cbor(result) == v); + } + } SECTION("half-precision float (edge cases)")