Merge branch 'master' into auto-conversion-macro

.github/workflows/main.yml

@@ -85,6 +85,31 @@ jobs:
       - name: Test
         run: |
             cd build && ctest --output-on-failure
+
+  build-osx:
+    runs-on: macos-10.15
+    strategy:
+      matrix:
+        standard: ['11', '14', '17', '20']
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+      - name: Install
+        run: |
+            brew install boost
+      - name: Configure
+        run: |
+            mkdir build && cd build
+            cmake .. -Dtoml11_BUILD_TEST=ON -DCMAKE_CXX_STANDARD=${{ matrix.standard }}
+      - name: Build
+        run: |
+            cd build && cmake --build .
+      - name: Test
+        run: |
+            cd build && ctest --output-on-failure
+
   build-windows-msvc:
     runs-on: windows-2019
     strategy:
@@ -96,6 +121,13 @@ jobs:
         uses: actions/checkout@v2
         with:
           submodules: true
+      - name: Install
+        run: |
+            (New-Object System.Net.WebClient).DownloadFile("https://github.com/actions/boost-versions/releases/download/1.72.0-20200608.4/boost-1.72.0-win32-msvc14.2-x86_64.tar.gz", "$env:TEMP\\boost.tar.gz")
+            7z.exe x "$env:TEMP\\boost.tar.gz" -o"$env:TEMP\\boostArchive" -y | Out-Null
+            7z.exe x "$env:TEMP\\boostArchive" -o"$env:TEMP\\boost" -y | Out-Null
+            Push-Location -Path "$env:TEMP\\boost"
+            Invoke-Expression .\\setup.ps1
       - uses: ilammy/msvc-dev-cmd@v1
       - name: Configure
         shell: cmd
@@ -103,7 +135,7 @@ jobs:
             file --mime-encoding tests/test_literals.cpp
             mkdir build
             cd build
-            cmake ../ -G "NMake Makefiles" -Dtoml11_BUILD_TEST=ON -DCMAKE_CXX_STANDARD=${{ matrix.standard }} -DBoost_ADDITIONAL_VERSIONS=1.72.0 -DBoost_USE_MULTITHREADED=ON -DBoost_ARCHITECTURE=-x64 -DBoost_NO_BOOST_CMAKE=ON -DBOOST_ROOT=%BOOST_ROOT_1_72_0%
+            cmake ../ -G "NMake Makefiles" -Dtoml11_BUILD_TEST=ON -DCMAKE_CXX_STANDARD=${{ matrix.standard }} -DBoost_NO_BOOST_CMAKE=ON -DBOOST_ROOT="C:\\hostedtoolcache\\windows\\Boost\\1.72.0\\x86_64"
       - name: Build
         working-directory: ./build
         run: |

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.1)
 enable_testing()
 
-project(toml11 VERSION 3.6.0)
+project(toml11 VERSION 3.6.1)
 
 option(toml11_BUILD_TEST "Build toml tests" OFF)
 option(toml11_TEST_WITH_ASAN  "use LLVM address sanitizer" OFF)

README.md

@@ -11,7 +11,7 @@ toml11
 
 toml11 is a C++11 (or later) header-only toml parser/encoder depending only on C++ standard library.
 
-- It is compatible to the latest version of [TOML v1.0.0-rc.2](https://toml.io/en/v1.0.0-rc.2).
+- It is compatible to the latest version of [TOML v1.0.0](https://toml.io/en/v1.0.0).
 - It is one of the most TOML standard compliant libraries, tested with [the language agnostic test suite for TOML parsers by BurntSushi](https://github.com/BurntSushi/toml-test).
 - It shows highly informative error messages. You can see the error messages about invalid files at [CircleCI](https://circleci.com/gh/ToruNiina/toml11).
 - It has configurable container. You can use any random-access containers and key-value maps as backend containers.
@@ -28,6 +28,10 @@ toml11 is a C++11 (or later) header-only toml parser/encoder depending only on C
 
 int main()
 {
+    // ```toml
+    // title = "an example toml file"
+    // nums  = [3, 1, 4, 1, 5]
+    // ```
     auto data = toml::parse("example.toml");
 
     // find a value with the specified type from a table
@@ -37,9 +41,9 @@ int main()
     std::vector<int> nums = toml::find<std::vector<int>>(data, "nums");
 
     // access with STL-like manner
-    if(not data.at("a").contains("b"))
+    if(not data.contains("foo"))
     {
-        data["a"]["b"] = "c";
+        data["foo"] = "bar";
     }
 
     // pass a fallback
@@ -1303,9 +1307,9 @@ struct foo
     double      b;
     std::string c;
 
-    toml::table into_toml() const // you need to mark it const.
+    toml::value into_toml() const // you need to mark it const.
     {
-        return toml::table{{"a", this->a}, {"b", this->b}, {"c", this->c}};
+        return toml::value{{"a", this->a}, {"b", this->b}, {"c", this->c}};
     }
 };
 } // ext
@@ -1332,9 +1336,9 @@ namespace toml
 template<>
 struct into<ext::foo>
 {
-    static toml::table into_toml(const ext::foo& f)
+    static toml::value into_toml(const ext::foo& f)
     {
-        return toml::table{{"a", f.a}, {"b", f.b}, {"c", f.c}};
+        return toml::value{{"a", f.a}, {"b", f.b}, {"c", f.c}};
     }
 };
 } // toml
@@ -1346,6 +1350,27 @@ toml::value v(f);
 Any type that can be converted to `toml::value`, e.g. `int`, `toml::table` and
 `toml::array` are okay to return from `into_toml`.
 
+You can also return a custom `toml::basic_value` from `toml::into`.
+
+```cpp
+namespace toml
+{
+template<>
+struct into<ext::foo>
+{
+    static toml::basic_value<toml::preserve_comments> into_toml(const ext::foo& f)
+    {
+        toml::basic_value<toml::preserve_comments> v{{"a", f.a}, {"b", f.b}, {"c", f.c}};
+        v.comments().push_back(" comment");
+        return v;
+    }
+};
+} // toml
+```
+
+But note that, if this `basic_value` would be assigned into other `toml::value`
+that discards `comments`, the comments would be dropped.
+
 ## Formatting user-defined error messages
 
 When you encounter an error after you read the toml value, you may want to
@@ -1848,6 +1873,12 @@ I appreciate the help of the contributors who introduced the great feature to th
   - Fix include path in README
 - Mohammed Alyousef (@MoAlyousef)
   - Made testing optional in CMake
+- Ivan Shynkarenka (@chronoxor)
+  - Fix compilation error in `<filesystem>` with MinGW
+- Alex Merry (@amerry)
+  - Add missing include files
+- sneakypete81 (@sneakypete81)
+  - Fix typo in error message
 
 
 ## Licensing terms

tests/test_extended_conversions.cpp

@@ -70,9 +70,9 @@ struct from<extlib::foo>
 template<>
 struct into<extlib::foo>
 {
-    static toml::table into_toml(const extlib::foo& f)
+    static toml::value into_toml(const extlib::foo& f)
     {
-        return toml::table{{"a", f.a}, {"b", f.b}};
+        return toml::value{{"a", f.a}, {"b", f.b}};
     }
 };
 

tests/test_serialize_file.cpp

@@ -10,6 +10,7 @@
 #include <map>
 #include <iostream>
 #include <fstream>
+#include <sstream>
 
 template<typename Comment,
          template<typename ...> class Table,
@@ -303,3 +304,60 @@ BOOST_AUTO_TEST_CASE(test_format_key)
         BOOST_TEST("\"special-chars-\\\\-\\\"-\\b-\\f-\\r-\\n-\\t\"" == toml::format_key(key));
     }
 }
+
+// In toml11, an implicitly-defined value does not have any comments.
+// So, in the following file,
+// ```toml
+// # comment
+// [[array-of-tables]]
+// foo = "bar"
+// ```
+// The array named "array-of-tables" does not have the comment, but the first
+// element of the array has. That means that, the above file is equivalent to
+// the following.
+// ```toml
+// array-of-tables = [
+//     # comment
+//     {foo = "bar"},
+// ]
+// ```
+// If the array itself has a comment (value_has_comment_ == true), we should try
+// to make it inline.
+// ```toml
+// # comment about array
+// array-of-tables = [
+//   # comment about table element
+//   {foo = "bar"}
+// ]
+// ```
+// If it is formatted as a multiline table, the two comments becomes
+// indistinguishable.
+// ```toml
+// # comment about array
+// # comment about table element
+// [[array-of-tables]]
+// foo = "bar"
+// ```
+// So we need to try to make it inline, and it force-inlines regardless
+// of the line width limit.
+//     It may fail if the element of a table has comment. In that case,
+// the array-of-tables will be formatted as a multiline table.
+BOOST_AUTO_TEST_CASE(test_distinguish_comment)
+{
+    const std::string str = R"(# comment about array itself
+array_of_table = [
+    # comment about the first element (table)
+    {key = "value"},
+])";
+    std::istringstream iss(str);
+    const auto data = toml::parse<toml::preserve_comments>(iss);
+    const auto serialized = toml::format(data, /*width = */ 0);
+
+    std::istringstream reparse(serialized);
+    const auto parsed = toml::parse<toml::preserve_comments>(reparse);
+
+    BOOST_TEST(parsed.at("array_of_table").comments().size()  == 1u);
+    BOOST_TEST(parsed.at("array_of_table").comments().front() == " comment about array itself");
+    BOOST_TEST(parsed.at("array_of_table").at(0).comments().size()  == 1u);
+    BOOST_TEST(parsed.at("array_of_table").at(0).comments().front() == " comment about the first element (table)");
+}

toml.hpp

@@ -35,7 +35,7 @@
 
 #define TOML11_VERSION_MAJOR 3
 #define TOML11_VERSION_MINOR 6
-#define TOML11_VERSION_PATCH 0
+#define TOML11_VERSION_PATCH 1
 
 #include "toml/parser.hpp"
 #include "toml/literal.hpp"

toml/comments.hpp

@@ -4,6 +4,7 @@
 #define TOML11_COMMENTS_HPP
 #include <initializer_list>
 #include <iterator>
+#include <stdexcept>
 #include <string>
 #include <type_traits>
 #include <utility>

toml/parser.hpp

@@ -13,12 +13,14 @@
 #include "types.hpp"
 #include "value.hpp"
 
+#ifndef TOML11_DISABLE_STD_FILESYSTEM
 #ifdef __cpp_lib_filesystem
 #if __has_include(<filesystem>)
 #define TOML11_HAS_STD_FILESYSTEM
 #include <filesystem>
 #endif // has_include(<string_view>)
 #endif // __cpp_lib_filesystem
+#endif // TOML11_DISABLE_STD_FILESYSTEM
 
 namespace toml
 {
@@ -930,7 +932,7 @@ parse_key(location& loc)
         return ok(std::make_pair(std::vector<key>(1, smpl.unwrap().first),
                                  smpl.unwrap().second));
     }
-    return err(format_underline("toml::parse_key: an invalid key appeaed.",
+    return err(format_underline("toml::parse_key: an invalid key appeared.",
                 {{source_location(loc), "is not a valid key"}}, {
                 "bare keys  : non-empty strings composed only of [A-Za-z0-9_-].",
                 "quoted keys: same as \"basic strings\" or 'literal strings'.",

toml/region.hpp

@@ -344,7 +344,7 @@ struct region final : public region_base
                     {
                         // unwrap the first '#' by std::next.
                         auto str = make_string(std::next(comment_found), iter);
-                        if(str.back() == '\r') {str.pop_back();}
+                        if(!str.empty() && str.back() == '\r') {str.pop_back();}
                         com.push_back(std::move(str));
                     }
                     else
@@ -397,7 +397,7 @@ struct region final : public region_base
                 {
                     // unwrap the first '#' by std::next.
                     auto str = make_string(std::next(comment_found), this->line_end());
-                    if(str.back() == '\r') {str.pop_back();}
+                    if(!str.empty() && str.back() == '\r') {str.pop_back();}
                     com.push_back(std::move(str));
                 }
             }

toml/serializer.hpp

@@ -97,8 +97,10 @@ struct serializer
                const int         float_prec     = std::numeric_limits<toml::floating>::max_digits10,
                const bool        can_be_inlined = false,
                const bool        no_comment     = false,
-               std::vector<toml::key> ks        = {})
+               std::vector<toml::key> ks        = {},
+               const bool     value_has_comment = false)
         : can_be_inlined_(can_be_inlined), no_comment_(no_comment),
+          value_has_comment_(value_has_comment && !no_comment),
           float_prec_(float_prec), width_(w), keys_(std::move(ks))
     {}
     ~serializer() = default;
@@ -120,7 +122,7 @@ struct serializer
         std::snprintf(buf.data(), buf.size(), fmt, this->float_prec_, f);
 
         std::string token(buf.begin(), std::prev(buf.end()));
-        if(token.back() == '.') // 1. => 1.0
+        if(!token.empty() && token.back() == '.') // 1. => 1.0
         {
             token += '0';
         }
@@ -244,88 +246,14 @@ struct serializer
 
     std::string operator()(const array_type& v) const
     {
-        if(!v.empty() && v.front().is_table())// v is an array of tables
-        {
-            // if it's not inlined, we need to add `[[table.key]]`.
-            // but if it can be inlined,
-            // ```
-            // table.key = [
-            //   {...},
-            //   # comment
-            //   {...},
-            // ]
-            // ```
-            if(this->can_be_inlined_)
-            {
-                std::string token;
-                if(!keys_.empty())
-                {
-                    token += format_key(keys_.back());
-                    token += " = ";
-                }
-                bool failed = false;
-                token += "[\n";
-                for(const auto& item : v)
-                {
-                    // if an element of the table has a comment, the table
-                    // cannot be inlined.
-                    if(this->has_comment_inside(item.as_table()))
-                    {
-                        failed = true;
-                        break;
-                    }
-                    if(!no_comment_)
-                    {
-                        for(const auto& c : item.comments())
-                        {
-                            token += '#';
-                            token += c;
-                            token += '\n';
-                        }
-                    }
-
-                    const auto t = this->make_inline_table(item.as_table());
-
-                    if(t.size() + 1 > width_ || // +1 for the last comma {...},
-                       std::find(t.cbegin(), t.cend(), '\n') != t.cend())
-                    {
-                        failed = true;
-                        break;
-                    }
-                    token += t;
-                    token += ",\n";
-                }
-                if(!failed)
-                {
-                    token += "]\n";
-                    return token;
-                }
-                // if failed, serialize them as [[array.of.tables]].
-            }
-
-            std::string token;
-            for(const auto& item : v)
-            {
-                if(!no_comment_)
-                {
-                    for(const auto& c : item.comments())
-                    {
-                        token += '#';
-                        token += c;
-                        token += '\n';
-                    }
-                }
-                token += "[[";
-                token += format_keys(keys_);
-                token += "]]\n";
-                token += this->make_multiline_table(item.as_table());
-            }
-            return token;
-        }
         if(v.empty())
         {
             return std::string("[]");
         }
+        if(this->is_array_of_tables(v))
+        {
+            return make_array_of_tables(v);
+        }
 
         // not an array of tables. normal array.
         // first, try to make it inline if none of the elements have a comment.
@@ -376,7 +304,7 @@ struct serializer
                     token += '\n';
                 }
                 token += toml::visit(*this, item);
-                if(token.back() == '\n') {token.pop_back();}
+                if(!token.empty() && token.back() == '\n') {token.pop_back();}
                 token += ",\n";
                 continue;
             }
@@ -384,7 +312,7 @@ struct serializer
             next_elem += toml::visit(*this, item);
 
             // comma before newline.
-            if(next_elem.back() == '\n') {next_elem.pop_back();}
+            if(!next_elem.empty() && next_elem.back() == '\n') {next_elem.pop_back();}
 
             // if current line does not exceeds the width limit, continue.
             if(current_line.size() + next_elem.size() + 1 < this->width_)
@@ -411,7 +339,10 @@ struct serializer
         }
         if(!current_line.empty())
         {
-            if(current_line.back() != '\n') {current_line += '\n';}
+            if(!current_line.empty() && current_line.back() != '\n')
+            {
+                current_line += '\n';
+            }
             token += current_line;
         }
         token += "]\n";
@@ -557,8 +488,10 @@ struct serializer
         for(const auto& item : v)
         {
             if(is_first) {is_first = false;} else {token += ',';}
-            token += visit(serializer((std::numeric_limits<std::size_t>::max)(),
+            token += visit(serializer(
-                                      this->float_prec_, true), item);
+                (std::numeric_limits<std::size_t>::max)(), this->float_prec_,
+                /* inlined */ true, /*no comment*/ false, /*keys*/ {},
+                /*has_comment*/ !item.comments().empty()), item);
         }
         token += ']';
         return token;
@@ -577,8 +510,10 @@ struct serializer
             if(is_first) {is_first = false;} else {token += ',';}
             token += format_key(kv.first);
             token += '=';
-            token += visit(serializer((std::numeric_limits<std::size_t>::max)(),
+            token += visit(serializer(
-                                      this->float_prec_, true), kv.second);
+                (std::numeric_limits<std::size_t>::max)(), this->float_prec_,
+                /* inlined */ true, /*no comment*/ false, /*keys*/ {},
+                /*has_comment*/ !kv.second.comments().empty()), kv.second);
         }
         token += '}';
         return token;
@@ -588,8 +523,16 @@ struct serializer
     {
         std::string token;
 
-        // print non-table stuff first. because after printing [foo.bar], the
+        // print non-table elements first.
-        // remaining non-table values will be assigned into [foo.bar], not [foo]
+        // ```toml
+        // [foo]         # a table we're writing now here
+        // key = "value" # <- non-table element, "key"
+        // # ...
+        // [foo.bar] # <- table element, "bar"
+        // ```
+        // because after printing [foo.bar], the remaining non-table values will
+        // be assigned into [foo.bar], not [foo]. Those values should be printed
+        // earlier.
         for(const auto& kv : v)
         {
             if(kv.second.is_table() || is_array_of_tables(kv.second))
@@ -597,21 +540,16 @@ struct serializer
                 continue;
             }
 
-            if(!kv.second.comments().empty() && !no_comment_)
+            token += write_comments(kv.second);
-            {
+
-                for(const auto& c : kv.second.comments())
-                {
-                    token += '#';
-                    token += c;
-                    token += '\n';
-                }
-            }
             const auto key_and_sep    = format_key(kv.first) + " = ";
             const auto residual_width = (this->width_ > key_and_sep.size()) ?
                                         this->width_ - key_and_sep.size() : 0;
             token += key_and_sep;
-            token += visit(serializer(residual_width, this->float_prec_, true),
+            token += visit(serializer(residual_width, this->float_prec_,
-                           kv.second);
+                /*can be inlined*/ true, /*no comment*/ false, /*keys*/ {},
+                /*has_comment*/ !kv.second.comments().empty()), kv.second);
+
             if(token.back() != '\n')
             {
                 token += '\n';
@@ -637,45 +575,172 @@ struct serializer
             ks.push_back(kv.first);
 
             auto tmp = visit(serializer(this->width_, this->float_prec_,
-                !multiline_table_printed, this->no_comment_, ks),
+                !multiline_table_printed, this->no_comment_, ks,
-                kv.second);
+                /*has_comment*/ !kv.second.comments().empty()), kv.second);
 
+            // If it is the first time to print a multi-line table, it would be
+            // helpful to separate normal key-value pair and subtables by a
+            // newline.
+            // (this checks if the current key-value pair contains newlines.
+            //  but it is not perfect because multi-line string can also contain
+            //  a newline. in such a case, an empty line will be written) TODO
             if((!multiline_table_printed) &&
                std::find(tmp.cbegin(), tmp.cend(), '\n') != tmp.cend())
             {
                 multiline_table_printed = true;
-            }
+                token += '\n'; // separate key-value pairs and subtables
-            else
-            {
-                // still inline tables only.
-                tmp += '\n';
-            }
 
-            if(!kv.second.comments().empty() && !no_comment_)
+                token += write_comments(kv.second);
+                token += tmp;
+
+                // care about recursive tables (all tables in each level prints
+                // newline and there will be a full of newlines)
+                if(tmp.substr(tmp.size() - 2, 2) != "\n\n" &&
+                   tmp.substr(tmp.size() - 4, 4) != "\r\n\r\n" )
                 {
-                for(const auto& c : kv.second.comments())
-                {
-                    token += '#';
-                    token += c;
                     token += '\n';
                 }
             }
+            else
+            {
+                token += write_comments(kv.second);
                 token += tmp;
+                token += '\n';
+            }
         }
         return token;
     }
 
+    std::string make_array_of_tables(const array_type& v) const
+    {
+        // if it's not inlined, we need to add `[[table.key]]`.
+        // but if it can be inlined, we can format it as the following.
+        // ```
+        // table.key = [
+        //   {...},
+        //   # comment
+        //   {...},
+        // ]
+        // ```
+        // This function checks if inlinization is possible or not, and then
+        // format the array-of-tables in a proper way.
+        //
+        // Note about comments:
+        //
+        // If the array itself has a comment (value_has_comment_ == true), we
+        // should try to make it inline.
+        // ```toml
+        // # comment about array
+        // array = [
+        //   # comment about table element
+        //   {of = "table"}
+        // ]
+        // ```
+        // If it is formatted as a multiline table, the two comments becomes
+        // indistinguishable.
+        // ```toml
+        // # comment about array
+        // # comment about table element
+        // [[array]]
+        // of = "table"
+        // ```
+        // So we need to try to make it inline, and it force-inlines regardless
+        // of the line width limit.
+        //     It may fail if the element of a table has comment. In that case,
+        // the array-of-tables will be formatted as a multiline table.
+        if(this->can_be_inlined_ || this->value_has_comment_)
+        {
+            std::string token;
+            if(!keys_.empty())
+            {
+                token += format_key(keys_.back());
+                token += " = ";
+            }
+
+            bool failed = false;
+            token += "[\n";
+            for(const auto& item : v)
+            {
+                // if an element of the table has a comment, the table
+                // cannot be inlined.
+                if(this->has_comment_inside(item.as_table()))
+                {
+                    failed = true;
+                    break;
+                }
+                // write comments for the table itself
+                token += write_comments(item);
+
+                const auto t = this->make_inline_table(item.as_table());
+
+                if(t.size() + 1 > width_ || // +1 for the last comma {...},
+                   std::find(t.cbegin(), t.cend(), '\n') != t.cend())
+                {
+                    // if the value itself has a comment, ignore the line width limit
+                    if( ! this->value_has_comment_)
+                    {
+                        failed = true;
+                        break;
+                    }
+                }
+                token += t;
+                token += ",\n";
+            }
+
+            if( ! failed)
+            {
+                token += "]\n";
+                return token;
+            }
+            // if failed, serialize them as [[array.of.tables]].
+        }
+
+        std::string token;
+        for(const auto& item : v)
+        {
+            token += write_comments(item);
+            token += "[[";
+            token += format_keys(keys_);
+            token += "]]\n";
+            token += this->make_multiline_table(item.as_table());
+        }
+        return token;
+    }
+
+    std::string write_comments(const value_type& v) const
+    {
+        std::string retval;
+        if(this->no_comment_) {return retval;}
+
+        for(const auto& c : v.comments())
+        {
+            retval += '#';
+            retval += c;
+            retval += '\n';
+        }
+        return retval;
+    }
+
     bool is_array_of_tables(const value_type& v) const
     {
-        if(!v.is_array()) {return false;}
+        if(!v.is_array() || v.as_array().empty()) {return false;}
-        const auto& a = v.as_array();
+        return is_array_of_tables(v.as_array());
-        return !a.empty() && a.front().is_table();
+    }
+    bool is_array_of_tables(const array_type& v) const
+    {
+        // Since TOML v0.5.0, heterogeneous arrays are allowed. So we need to
+        // check all the element in an array to check if the array is an array
+        // of tables.
+        return std::all_of(v.begin(), v.end(), [](const value_type& elem) {
+                return elem.is_table();
+            });
     }
 
   private:
 
     bool        can_be_inlined_;
     bool        no_comment_;
+    bool        value_has_comment_;
     int         float_prec_;
     std::size_t width_;
     std::vector<toml::key> keys_;

toml/source_location.hpp

@@ -3,6 +3,7 @@
 #ifndef TOML11_SOURCE_LOCATION_HPP
 #define TOML11_SOURCE_LOCATION_HPP
 #include <cstdint>
+#include <sstream>
 
 #include "region.hpp"