Compare commits

...

16 Commits

Author SHA1 Message Date
ToruNiina
ac1130f9f4 tag: update patch version v3.3.1 2020-02-16 22:02:40 +09:00
ToruNiina
d290c3b7e5 doc: add contributor to README 2020-02-14 19:25:41 +09:00
Toru Niina
e4140ac1fd Merge pull request #102 from jwillikers/cmake_cache_variables
Set CMake Standard Cache Variables. Fixes #101
2020-02-13 13:22:25 +09:00
Jordan Williams
ef33c10ba8 use cache variables for the CMake standard and extensions settings 2020-02-12 07:44:47 -06:00
Toru Niina
ced710bb4c Merge pull request #100 from jwillikers/clang_warnings
Silence Clang Warnings. Fixes #98 & #99
2020-02-12 13:37:58 +09:00
Jordan Williams
6b5944e839 fix -Wundef warnings 2020-02-11 06:30:18 -06:00
Jordan Williams
76cae8c057 enable -Wundef flag for tests 2020-02-11 06:25:10 -06:00
Jordan Williams
3930a44ccd enable range-loop-analysis flag for tests 2020-02-11 06:17:54 -06:00
Jordan Williams
3b6417de00 fix clang range-loop-analysis warnings 2020-02-11 06:13:55 -06:00
ToruNiina
573a6f1d81 test: use JSON format_key to format a key in JSON
The rule to format a basic string and a key is different. `"""` cannot
be used with JSON keys. Also, toml key is basically not wrapped by `"`.
So we need a function to format a key in JSON.
2020-02-06 01:28:37 +09:00
ToruNiina
f6a41d986c feat: handle quotes in strings in the better way
- if a basic string contains any double quote, make it multiline.
  - because 1 or 2 consecutive "s do not require escape sequence in it.
- if a basic string will be sufficiently long, make it multiline.
- if 3 consecutive "s appeared, insert backslash to break it down.
2020-02-05 22:42:10 +09:00
ToruNiina
16fc172b21 feat: check string length before adding newline
In literal strings, only the first newline will be trimmed.
```toml
str = '''
The first newline will be trimmed.'''
```
The previous code always adds this first-newline, but after this commit
it checks the length of the string and adds newline if the string is
sufficiently long.
2020-02-05 22:39:08 +09:00
ToruNiina
7d03eb489a test: add test cases with quotes in ml-string
some of these example strings are copied from toml-lang/toml:README.md
and some are modified.
2020-02-04 22:37:11 +09:00
ToruNiina
0582e1535b fix: handle edge-cases with quotes in ml-string
See comments in the code for detail.
2020-02-04 22:36:39 +09:00
ToruNiina
d495df93a6 refactor: remove trailing whitespaces 2020-02-04 22:21:37 +09:00
ToruNiina
5ca3a3c262 refactor: change ifdef UNRELEASED_FEATURE region
to eliminate dead code after returning from a function
2020-02-04 21:05:03 +09:00
11 changed files with 272 additions and 55 deletions

View File

@@ -5,7 +5,7 @@ project(toml11)
set(toml11_VERSION_MAYOR 3) set(toml11_VERSION_MAYOR 3)
set(toml11_VERSION_MINOR 3) set(toml11_VERSION_MINOR 3)
set(toml11_VERSION_PATCH 0) set(toml11_VERSION_PATCH 1)
set(toml11_VERSION set(toml11_VERSION
"${toml11_VERSION_MAYOR}.${toml11_VERSION_MINOR}.${toml11_VERSION_PATCH}" "${toml11_VERSION_MAYOR}.${toml11_VERSION_MINOR}.${toml11_VERSION_PATCH}"
) )
@@ -16,11 +16,9 @@ option(toml11_TEST_WITH_UBSAN "use LLVM undefined behavior sanitizer" OFF)
include(CheckCXXCompilerFlag) include(CheckCXXCompilerFlag)
if("${CMAKE_VERSION}" VERSION_GREATER 3.1) if("${CMAKE_VERSION}" VERSION_GREATER 3.1)
set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_CXX_STANDARD 11 CACHE STRING "The C++ standard whose features are requested to build all targets.")
if(NOT DEFINED CMAKE_CXX_STANDARD) set(CMAKE_CXX_STANDARD_REQUIRED ON CACHE BOOL "Boolean describing whether the value of CXX_STANDARD is a requirement.")
set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_EXTENSIONS OFF CACHE BOOL "Boolean specifying whether compiler specific extensions are requested.")
endif()
set(CXX_STANDARD_REQUIRED ON)
else() else()
# Manually check for C++11 compiler flag. # Manually check for C++11 compiler flag.
CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11) CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11)

View File

@@ -1783,6 +1783,10 @@ I appreciate the help of the contributors who introduced the great feature to th
- Suppress warnings in Debug mode - Suppress warnings in Debug mode
- OGAWA Kenichi (@kenichiice) - OGAWA Kenichi (@kenichiice)
- Suppress warnings on intel compiler - Suppress warnings on intel compiler
- Jordan Williams (@jwillikers)
- Fixed clang range-loop-analysis warnings
- Fixed feature test macro to suppress -Wundef
- Use cache variables in CMakeLists.txt
## Licensing terms ## Licensing terms

View File

@@ -48,6 +48,8 @@ CHECK_CXX_COMPILER_FLAG("-Wduplicated-branches" COMPILER_SUPPORTS_WDUPLICATED_BR
CHECK_CXX_COMPILER_FLAG("-Wlogical-op" COMPILER_SUPPORTS_WLOGICAL_OP) CHECK_CXX_COMPILER_FLAG("-Wlogical-op" COMPILER_SUPPORTS_WLOGICAL_OP)
CHECK_CXX_COMPILER_FLAG("-Wuseless-cast" COMPILER_SUPPORTS_WUSELESS_CAST) CHECK_CXX_COMPILER_FLAG("-Wuseless-cast" COMPILER_SUPPORTS_WUSELESS_CAST)
CHECK_CXX_COMPILER_FLAG("-Wdouble-promotion" COMPILER_SUPPORTS_WDOUBLE_PROMOTION) CHECK_CXX_COMPILER_FLAG("-Wdouble-promotion" COMPILER_SUPPORTS_WDOUBLE_PROMOTION)
CHECK_CXX_COMPILER_FLAG("-Wrange-loop-analysis" COMPILER_SUPPORTS_WRANGE_LOOP_ANALYSIS)
CHECK_CXX_COMPILER_FLAG("-Wundef" COMPILER_SUPPORTS_WUNDEF)
if(COMPILER_SUPPORTS_WALL) if(COMPILER_SUPPORTS_WALL)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
@@ -82,6 +84,12 @@ endif()
if(COMPILER_SUPPORTS_WDOUBLE_PROMOTION) if(COMPILER_SUPPORTS_WDOUBLE_PROMOTION)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wdouble-promotion") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wdouble-promotion")
endif() endif()
if(COMPILER_SUPPORTS_WRANGE_LOOP_ANALYSIS)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wrange-loop-analysis")
endif()
if(COMPILER_SUPPORTS_WUNDEF)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wundef")
endif()
option(TOML11_USE_UNRELEASED_TOML_FEATURES option(TOML11_USE_UNRELEASED_TOML_FEATURES
"use features in toml-lang/toml master while testing" OFF) "use features in toml-lang/toml master while testing" OFF)

View File

@@ -84,9 +84,7 @@ struct json_serializer
{ {
if(!is_first) {std::cout << ", ";} if(!is_first) {std::cout << ", ";}
is_first = false; is_first = false;
std::cout << toml::format(toml::value(elem.first), std::cout << this->format_key(elem.first) << ':';
std::numeric_limits<std::size_t>::max());
std::cout << ':';
toml::visit(*this, elem.second); toml::visit(*this, elem.second);
} }
std::cout << '}'; std::cout << '}';
@@ -112,6 +110,12 @@ struct json_serializer
} }
return retval; return retval;
} }
std::string format_key(const std::string& s) const
{
const auto quote("\"");
return quote + escape_string(s) + quote;
}
}; };
int main() int main()

View File

@@ -57,6 +57,22 @@ BOOST_AUTO_TEST_CASE(test_ml_basic_string)
TOML11_TEST_LEX_ACCEPT(lex_string, TOML11_TEST_LEX_ACCEPT(lex_string,
"\"\"\"\\\n The quick brown \\\n\n fox jumps over \\\n the lazy dog.\\\n \"\"\"", "\"\"\"\\\n The quick brown \\\n\n fox jumps over \\\n the lazy dog.\\\n \"\"\"",
"\"\"\"\\\n The quick brown \\\n\n fox jumps over \\\n the lazy dog.\\\n \"\"\""); "\"\"\"\\\n The quick brown \\\n\n fox jumps over \\\n the lazy dog.\\\n \"\"\"");
TOML11_TEST_LEX_ACCEPT(lex_string,
"\"\"\"Here are two quotation marks: \"\". Simple enough.\"\"\"",
"\"\"\"Here are two quotation marks: \"\". Simple enough.\"\"\"");
TOML11_TEST_LEX_ACCEPT(lex_string,
"\"\"\"Here are three quotation marks: \"\"\\\".\"\"\"",
"\"\"\"Here are three quotation marks: \"\"\\\".\"\"\"");
TOML11_TEST_LEX_ACCEPT(lex_string,
"\"\"\"Here are fifteen quotation marks: \"\"\\\"\"\"\\\"\"\"\\\"\"\"\\\"\"\"\\\".\"\"\"",
"\"\"\"Here are fifteen quotation marks: \"\"\\\"\"\"\\\"\"\"\\\"\"\"\\\"\"\"\\\".\"\"\"");
TOML11_TEST_LEX_ACCEPT(lex_string,
"\"\"\"\"This,\" she said, \"is just a pointless statement.\"\"\"\"",
"\"\"\"\"This,\" she said, \"is just a pointless statement.\"\"\"\"");
} }
BOOST_AUTO_TEST_CASE(test_literal_string) BOOST_AUTO_TEST_CASE(test_literal_string)
@@ -83,4 +99,16 @@ BOOST_AUTO_TEST_CASE(test_ml_literal_string)
TOML11_TEST_LEX_ACCEPT(lex_string, TOML11_TEST_LEX_ACCEPT(lex_string,
"'''\nThe first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n'''", "'''\nThe first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n'''",
"'''\nThe first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n'''"); "'''\nThe first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n'''");
TOML11_TEST_LEX_ACCEPT(lex_string,
"''''That's still pointless', she said.'''",
"''''That's still pointless', she said.'''");
TOML11_TEST_LEX_ACCEPT(lex_string,
"'''Here are fifteen quotation marks: \"\"\"\"\"\"\"\"\"\"\"\"\"\"\".'''",
"'''Here are fifteen quotation marks: \"\"\"\"\"\"\"\"\"\"\"\"\"\"\".'''");
TOML11_TEST_LEX_ACCEPT(lex_string,
"''''This,' she said, 'is just a pointless statement.''''",
"''''This,' she said, 'is just a pointless statement.''''");
} }

View File

@@ -69,6 +69,9 @@ BOOST_AUTO_TEST_CASE(test_basic_string)
TOML11_TEST_PARSE_EQUAL(parse_string, TOML11_TEST_PARSE_EQUAL(parse_string,
"\" And when \\\"'s are in the along with # \\\"\"", "\" And when \\\"'s are in the along with # \\\"\"",
string(" And when \"'s are in the along with # \"", string_t::basic)); string(" And when \"'s are in the along with # \"", string_t::basic));
TOML11_TEST_PARSE_EQUAL(parse_string,
"\"Here are fifteen apostrophes: '''''''''''''''\"",
string("Here are fifteen apostrophes: '''''''''''''''", string_t::basic));
} }
BOOST_AUTO_TEST_CASE(test_basic_string_value) BOOST_AUTO_TEST_CASE(test_basic_string_value)
@@ -94,6 +97,9 @@ BOOST_AUTO_TEST_CASE(test_basic_string_value)
TOML11_TEST_PARSE_EQUAL_VALUE(parse_value<toml::value>, TOML11_TEST_PARSE_EQUAL_VALUE(parse_value<toml::value>,
"\" And when \\\"'s are in the along with # \\\"\"", "\" And when \\\"'s are in the along with # \\\"\"",
value(" And when \"'s are in the along with # \"", string_t::basic)); value(" And when \"'s are in the along with # \"", string_t::basic));
TOML11_TEST_PARSE_EQUAL_VALUE(parse_value<toml::value>,
"\"Here are fifteen apostrophes: '''''''''''''''\"",
value("Here are fifteen apostrophes: '''''''''''''''", string_t::basic));
} }
BOOST_AUTO_TEST_CASE(test_ml_basic_string) BOOST_AUTO_TEST_CASE(test_ml_basic_string)
@@ -104,6 +110,18 @@ BOOST_AUTO_TEST_CASE(test_ml_basic_string)
TOML11_TEST_PARSE_EQUAL(parse_string, TOML11_TEST_PARSE_EQUAL(parse_string,
"\"\"\"\\\n The quick brown \\\n\n fox jumps over \\\n the lazy dog.\\\n \"\"\"", "\"\"\"\\\n The quick brown \\\n\n fox jumps over \\\n the lazy dog.\\\n \"\"\"",
string("The quick brown fox jumps over the lazy dog.", string_t::basic)); string("The quick brown fox jumps over the lazy dog.", string_t::basic));
TOML11_TEST_PARSE_EQUAL(parse_string,
"\"\"\"Here are two quotation marks: \"\". Simple enough.\"\"\"",
string("Here are two quotation marks: \"\". Simple enough.", string_t::basic));
TOML11_TEST_PARSE_EQUAL(parse_string,
"\"\"\"Here are three quotation marks: \"\"\\\".\"\"\"",
string("Here are three quotation marks: \"\"\".", string_t::basic));
TOML11_TEST_PARSE_EQUAL(parse_string,
"\"\"\"Here are fifteen quotation marks: \"\"\\\"\"\"\\\"\"\"\\\"\"\"\\\"\"\"\\\".\"\"\"",
string("Here are fifteen quotation marks: \"\"\"\"\"\"\"\"\"\"\"\"\"\"\".", string_t::basic));
TOML11_TEST_PARSE_EQUAL(parse_string,
"\"\"\"\"This,\" she said, \"is just a pointless statement.\"\"\"\"",
string("\"This,\" she said, \"is just a pointless statement.\"", string_t::basic));
} }
BOOST_AUTO_TEST_CASE(test_ml_basic_string_value) BOOST_AUTO_TEST_CASE(test_ml_basic_string_value)
@@ -114,6 +132,19 @@ BOOST_AUTO_TEST_CASE(test_ml_basic_string_value)
TOML11_TEST_PARSE_EQUAL_VALUE(parse_value<toml::value>, TOML11_TEST_PARSE_EQUAL_VALUE(parse_value<toml::value>,
"\"\"\"\\\n The quick brown \\\n\n fox jumps over \\\n the lazy dog.\\\n \"\"\"", "\"\"\"\\\n The quick brown \\\n\n fox jumps over \\\n the lazy dog.\\\n \"\"\"",
value("The quick brown fox jumps over the lazy dog.", string_t::basic)); value("The quick brown fox jumps over the lazy dog.", string_t::basic));
TOML11_TEST_PARSE_EQUAL_VALUE(parse_value<toml::value>,
"\"\"\"Here are two quotation marks: \"\". Simple enough.\"\"\"",
value("Here are two quotation marks: \"\". Simple enough.", string_t::basic));
TOML11_TEST_PARSE_EQUAL_VALUE(parse_value<toml::value>,
"\"\"\"Here are three quotation marks: \"\"\\\".\"\"\"",
value("Here are three quotation marks: \"\"\".", string_t::basic));
TOML11_TEST_PARSE_EQUAL_VALUE(parse_value<toml::value>,
"\"\"\"Here are fifteen quotation marks: \"\"\\\"\"\"\\\"\"\"\\\"\"\"\\\"\"\"\\\".\"\"\"",
value("Here are fifteen quotation marks: \"\"\"\"\"\"\"\"\"\"\"\"\"\"\".", string_t::basic));
TOML11_TEST_PARSE_EQUAL_VALUE(parse_value<toml::value>,
"\"\"\"\"This,\" she said, \"is just a pointless statement.\"\"\"\"",
value("\"This,\" she said, \"is just a pointless statement.\"", string_t::basic));
} }
BOOST_AUTO_TEST_CASE(test_literal_string) BOOST_AUTO_TEST_CASE(test_literal_string)
@@ -156,6 +187,15 @@ BOOST_AUTO_TEST_CASE(test_ml_literal_string)
TOML11_TEST_PARSE_EQUAL(parse_string, TOML11_TEST_PARSE_EQUAL(parse_string,
"'''\nThe first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n'''", "'''\nThe first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n'''",
string("The first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n", string_t::literal)); string("The first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n", string_t::literal));
TOML11_TEST_PARSE_EQUAL(parse_string,
"''''That's still pointless', she said.'''",
string("'That's still pointless', she said.", string_t::literal));
TOML11_TEST_PARSE_EQUAL(parse_string,
"'''Here are fifteen quotation marks: \"\"\"\"\"\"\"\"\"\"\"\"\"\"\".'''",
string("Here are fifteen quotation marks: \"\"\"\"\"\"\"\"\"\"\"\"\"\"\".", string_t::literal));
TOML11_TEST_PARSE_EQUAL(parse_string,
"''''This,' she said, 'is just a pointless statement.''''",
string("'This,' she said, 'is just a pointless statement.'", string_t::literal));
} }
BOOST_AUTO_TEST_CASE(test_ml_literal_string_value) BOOST_AUTO_TEST_CASE(test_ml_literal_string_value)
@@ -166,6 +206,15 @@ BOOST_AUTO_TEST_CASE(test_ml_literal_string_value)
TOML11_TEST_PARSE_EQUAL_VALUE(parse_value<toml::value>, TOML11_TEST_PARSE_EQUAL_VALUE(parse_value<toml::value>,
"'''\nThe first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n'''", "'''\nThe first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n'''",
value("The first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n", string_t::literal)); value("The first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n", string_t::literal));
TOML11_TEST_PARSE_EQUAL_VALUE(parse_value<toml::value>,
"''''That's still pointless', she said.'''",
value("'That's still pointless', she said.", string_t::literal));
TOML11_TEST_PARSE_EQUAL_VALUE(parse_value<toml::value>,
"'''Here are fifteen quotation marks: \"\"\"\"\"\"\"\"\"\"\"\"\"\"\".'''",
value("Here are fifteen quotation marks: \"\"\"\"\"\"\"\"\"\"\"\"\"\"\".", string_t::literal));
TOML11_TEST_PARSE_EQUAL_VALUE(parse_value<toml::value>,
"''''This,' she said, 'is just a pointless statement.''''",
value("'This,' she said, 'is just a pointless statement.'", string_t::literal));
} }
BOOST_AUTO_TEST_CASE(test_unicode_escape_sequence) BOOST_AUTO_TEST_CASE(test_unicode_escape_sequence)

View File

@@ -20,7 +20,7 @@ namespace toml
namespace detail namespace detail
{ {
// TODO: find more sophisticated way to handle this // TODO: find more sophisticated way to handle this
#if _POSIX_C_SOURCE >= 1 || defined(_XOPEN_SOURCE) || defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || defined(_POSIX_SOURCE) #if (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 1) || defined(_XOPEN_SOURCE) || defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || defined(_POSIX_SOURCE)
inline std::tm localtime_s(const std::time_t* src) inline std::tm localtime_s(const std::time_t* src)
{ {
std::tm dst; std::tm dst;
@@ -35,7 +35,7 @@ inline std::tm gmtime_s(const std::time_t* src)
if (!result) { throw std::runtime_error("gmtime_r failed."); } if (!result) { throw std::runtime_error("gmtime_r failed."); }
return dst; return dst;
} }
#elif _MSC_VER #elif defined(_MSC_VER)
inline std::tm localtime_s(const std::time_t* src) inline std::tm localtime_s(const std::time_t* src)
{ {
std::tm dst; std::tm dst;

View File

@@ -154,12 +154,53 @@ using lex_basic_string = sequence<lex_quotation_mark,
repeat<lex_basic_char, unlimited>, repeat<lex_basic_char, unlimited>,
lex_quotation_mark>; lex_quotation_mark>;
// After toml post-v0.5.0, it is explicitly clarified how quotes in ml-strings
// are allowed to be used.
// After this, the following strings are *explicitly* allowed.
// - One or two `"`s in a multi-line basic string is allowed wherever it is.
// - Three consecutive `"`s in a multi-line basic string is considered as a delimiter.
// - One or two `"`s can appear just before or after the delimiter.
// ```toml
// str4 = """Here are two quotation marks: "". Simple enough."""
// str5 = """Here are three quotation marks: ""\"."""
// str6 = """Here are fifteen quotation marks: ""\"""\"""\"""\"""\"."""
// str7 = """"This," she said, "is just a pointless statement.""""
// ```
// In the current implementation (v3.3.0), it is difficult to parse `str7` in
// the above example. It is difficult to recognize `"` at the end of string body
// collectly. It will be misunderstood as a `"""` delimiter and an additional,
// invalid `"`. Like this:
// ```console
// what(): [error] toml::parse_table: invalid line format
// --> hoge.toml
// |
// 13 | str7 = """"This," she said, "is just a pointless statement.""""
// | ^- expected newline, but got '"'.
// ```
// As a quick workaround for this problem, `lex_ml_basic_string_delim` was
// splitted into two, `lex_ml_basic_string_open` and `lex_ml_basic_string_close`.
// `lex_ml_basic_string_open` allows only `"""`. `_close` allows 3-5 `"`s.
// In parse_ml_basic_string() function, the trailing `"`s will be attached to
// the string body.
//
// Note: This feature is a "clarification". Therefore this change is considered
// as a spec that has been defined since the time when the multi-line
// basic string was introduced. Although it is a post-v0.5.0 changes,
// this change will be activated regardless of the flag,
// `TOML11_USE_UNRELEASED_TOML_FEATURES`.
//
using lex_ml_basic_string_delim = repeat<lex_quotation_mark, exactly<3>>; using lex_ml_basic_string_delim = repeat<lex_quotation_mark, exactly<3>>;
using lex_ml_basic_string_open = lex_ml_basic_string_delim;
using lex_ml_basic_string_close = sequence<
repeat<lex_quotation_mark, exactly<3>>,
maybe<lex_quotation_mark>, maybe<lex_quotation_mark>
>;
#ifdef TOML11_USE_UNRELEASED_TOML_FEATURES #ifdef TOML11_USE_UNRELEASED_TOML_FEATURES
using lex_ml_basic_unescaped = exclude<either<in_range<0x00, 0x08>, // 0x09 using lex_ml_basic_unescaped = exclude<either<in_range<0x00, 0x08>, // 0x09
in_range<0x0a, 0x1F>, // is tab in_range<0x0a, 0x1F>, // is tab
character<0x5C>, character<0x5C>, // backslash
character<0x7F>, character<0x7F>, // DEL
lex_ml_basic_string_delim>>; lex_ml_basic_string_delim>>;
#else // TOML v0.5.0 #else // TOML v0.5.0
using lex_ml_basic_unescaped = exclude<either<in_range<0x00,0x1F>, using lex_ml_basic_unescaped = exclude<either<in_range<0x00,0x1F>,
@@ -176,9 +217,9 @@ using lex_ml_basic_char = either<lex_ml_basic_unescaped, lex_escaped>;
using lex_ml_basic_body = repeat<either<lex_ml_basic_char, lex_newline, using lex_ml_basic_body = repeat<either<lex_ml_basic_char, lex_newline,
lex_ml_basic_escaped_newline>, lex_ml_basic_escaped_newline>,
unlimited>; unlimited>;
using lex_ml_basic_string = sequence<lex_ml_basic_string_delim, using lex_ml_basic_string = sequence<lex_ml_basic_string_open,
lex_ml_basic_body, lex_ml_basic_body,
lex_ml_basic_string_delim>; lex_ml_basic_string_close>;
using lex_literal_char = exclude<either<in_range<0x00, 0x08>, using lex_literal_char = exclude<either<in_range<0x00, 0x08>,
in_range<0x10, 0x19>, character<0x27>>>; in_range<0x10, 0x19>, character<0x27>>>;
@@ -187,7 +228,13 @@ using lex_literal_string = sequence<lex_apostrophe,
repeat<lex_literal_char, unlimited>, repeat<lex_literal_char, unlimited>,
lex_apostrophe>; lex_apostrophe>;
// the same reason as above.
using lex_ml_literal_string_delim = repeat<lex_apostrophe, exactly<3>>; using lex_ml_literal_string_delim = repeat<lex_apostrophe, exactly<3>>;
using lex_ml_literal_string_open = lex_ml_literal_string_delim;
using lex_ml_literal_string_close = sequence<
repeat<lex_apostrophe, exactly<3>>,
maybe<lex_apostrophe>, maybe<lex_apostrophe>
>;
using lex_ml_literal_char = exclude<either<in_range<0x00, 0x08>, using lex_ml_literal_char = exclude<either<in_range<0x00, 0x08>,
in_range<0x10, 0x1F>, in_range<0x10, 0x1F>,
@@ -195,9 +242,9 @@ using lex_ml_literal_char = exclude<either<in_range<0x00, 0x08>,
lex_ml_literal_string_delim>>; lex_ml_literal_string_delim>>;
using lex_ml_literal_body = repeat<either<lex_ml_literal_char, lex_newline>, using lex_ml_literal_body = repeat<either<lex_ml_literal_char, lex_newline>,
unlimited>; unlimited>;
using lex_ml_literal_string = sequence<lex_ml_literal_string_delim, using lex_ml_literal_string = sequence<lex_ml_literal_string_open,
lex_ml_literal_body, lex_ml_literal_body,
lex_ml_literal_string_delim>; lex_ml_literal_string_close>;
using lex_string = either<lex_ml_basic_string, lex_basic_string, using lex_string = either<lex_ml_basic_string, lex_basic_string,
lex_ml_literal_string, lex_literal_string>; lex_ml_literal_string, lex_literal_string>;

View File

@@ -375,7 +375,7 @@ parse_ml_basic_string(location<Container>& loc)
std::string retval; std::string retval;
retval.reserve(token.unwrap().size()); retval.reserve(token.unwrap().size());
auto delim = lex_ml_basic_string_delim::invoke(inner_loc); auto delim = lex_ml_basic_string_open::invoke(inner_loc);
if(!delim) if(!delim)
{ {
throw internal_error(format_underline( throw internal_error(format_underline(
@@ -410,7 +410,26 @@ parse_ml_basic_string(location<Container>& loc)
{{std::addressof(inner_loc), "not sufficient token"}}), {{std::addressof(inner_loc), "not sufficient token"}}),
source_location(std::addressof(inner_loc))); source_location(std::addressof(inner_loc)));
} }
delim = lex_ml_basic_string_delim::invoke(inner_loc); delim = lex_ml_basic_string_close::invoke(inner_loc);
}
// `lex_ml_basic_string_close` allows 3 to 5 `"`s to allow 1 or 2 `"`s
// at just before the delimiter. Here, we need to attach `"`s at the
// end of the string body, if it exists.
// For detail, see the definition of `lex_ml_basic_string_close`.
assert(std::all_of(delim.unwrap().first(), delim.unwrap().last(),
[](const char c) noexcept {return c == '\"';}));
switch(delim.unwrap().size())
{
case 3: {break;}
case 4: {retval += "\""; break;}
case 5: {retval += "\"\""; break;}
default:
{
throw internal_error(format_underline(
"parse_ml_basic_string: closing delimiter has invalid length",
{{std::addressof(inner_loc), "end of this"}}),
source_location(std::addressof(inner_loc)));
}
} }
return ok(std::make_pair(toml::string(retval), token.unwrap())); return ok(std::make_pair(toml::string(retval), token.unwrap()));
} }
@@ -485,7 +504,7 @@ parse_ml_literal_string(location<Container>& loc)
{ {
location<std::string> inner_loc(loc.name(), token.unwrap().str()); location<std::string> inner_loc(loc.name(), token.unwrap().str());
const auto open = lex_ml_literal_string_delim::invoke(inner_loc); const auto open = lex_ml_literal_string_open::invoke(inner_loc);
if(!open) if(!open)
{ {
throw internal_error(format_underline( throw internal_error(format_underline(
@@ -498,7 +517,7 @@ parse_ml_literal_string(location<Container>& loc)
const auto body = lex_ml_literal_body::invoke(inner_loc); const auto body = lex_ml_literal_body::invoke(inner_loc);
const auto close = lex_ml_literal_string_delim::invoke(inner_loc); const auto close = lex_ml_literal_string_close::invoke(inner_loc);
if(!close) if(!close)
{ {
throw internal_error(format_underline( throw internal_error(format_underline(
@@ -506,8 +525,28 @@ parse_ml_literal_string(location<Container>& loc)
{{std::addressof(inner_loc), "should be '''"}}), {{std::addressof(inner_loc), "should be '''"}}),
source_location(std::addressof(inner_loc))); source_location(std::addressof(inner_loc)));
} }
return ok(std::make_pair( // `lex_ml_literal_string_close` allows 3 to 5 `'`s to allow 1 or 2 `'`s
toml::string(body.unwrap().str(), toml::string_t::literal), // at just before the delimiter. Here, we need to attach `'`s at the
// end of the string body, if it exists.
// For detail, see the definition of `lex_ml_basic_string_close`.
std::string retval = body.unwrap().str();
assert(std::all_of(close.unwrap().first(), close.unwrap().last(),
[](const char c) noexcept {return c == '\'';}));
switch(close.unwrap().size())
{
case 3: {break;}
case 4: {retval += "'"; break;}
case 5: {retval += "''"; break;}
default:
{
throw internal_error(format_underline(
"parse_ml_literal_string: closing delimiter has invalid length",
{{std::addressof(inner_loc), "end of this"}}),
source_location(std::addressof(inner_loc)));
}
}
return ok(std::make_pair(toml::string(retval, toml::string_t::literal),
token.unwrap())); token.unwrap()));
} }
else else

View File

@@ -511,7 +511,7 @@ inline std::string format_underline(const std::string& message,
retval << '\n'; retval << '\n';
retval << make_string(static_cast<std::size_t>(line_num_width + 1), ' '); retval << make_string(static_cast<std::size_t>(line_num_width + 1), ' ');
retval << color::bold << color::blue << " | " << color::reset; retval << color::bold << color::blue << " | " << color::reset;
for(const auto help : helps) for(const auto& help : helps)
{ {
retval << color::bold << "\nHint: " << color::reset; retval << color::bold << "\nHint: " << color::reset;
retval << help; retval << help;

View File

@@ -119,12 +119,17 @@ struct serializer
return token; // there is no exponent part. just return it. return token; // there is no exponent part. just return it.
} }
#ifdef TOML11_USE_UNRELEASED_TOML_FEATURES #ifdef TOML11_USE_UNRELEASED_TOML_FEATURES
// Although currently it is not released yet, TOML will allow // Although currently it is not released yet as a tagged version,
// zero-prefix in an exponent part such as 1.234e+01. // TOML will allow zero-prefix in an exponent part, such as `1.234e+01`.
// The following code removes the zero prefixes. // ```toml
// If the feature is activated, the following codes can be skipped. // num1 = 1.234e+1 # OK in TOML v0.5.0
// num2 = 1.234e+01 # error in TOML v0.5.0 but will be allowed soon
// ```
// To avoid `e+01`, the following `else` section removes the zero
// prefixes in the exponent part.
// If the feature is activated, it can be skipped.
return token; return token;
#endif #else
// zero-prefix in an exponent is NOT allowed in TOML v0.5.0. // zero-prefix in an exponent is NOT allowed in TOML v0.5.0.
// remove it if it exists. // remove it if it exists.
bool sign_exists = false; bool sign_exists = false;
@@ -143,17 +148,29 @@ struct serializer
zero_prefix); zero_prefix);
} }
return token; return token;
#endif
} }
std::string operator()(const string_type& s) const std::string operator()(const string_type& s) const
{ {
if(s.kind == string_t::basic) if(s.kind == string_t::basic)
{ {
if(std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend()) if(std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend() ||
std::find(s.str.cbegin(), s.str.cend(), '\"') != s.str.cend())
{ {
// if linefeed is contained, make it multiline-string. // if linefeed or double-quote is contained,
const std::string open("\"\"\"\n"); // make it multiline basic string.
const std::string close("\\\n\"\"\""); const auto escaped = this->escape_ml_basic_string(s.str);
return open + this->escape_ml_basic_string(s.str) + close; std::string open("\"\"\"");
std::string close("\"\"\"");
if(escaped.find('\n') != std::string::npos ||
this->width_ < escaped.size() + 6)
{
// if the string body contains newline or is enough long,
// add newlines after and before delimiters.
open += "\n";
close = std::string("\\\n") + close;
}
return open + escaped + close;
} }
// no linefeed. try to make it oneline-string. // no linefeed. try to make it oneline-string.
@@ -194,7 +211,11 @@ struct serializer
if(std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend() || if(std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend() ||
std::find(s.str.cbegin(), s.str.cend(), '\'') != s.str.cend() ) std::find(s.str.cbegin(), s.str.cend(), '\'') != s.str.cend() )
{ {
const std::string open("'''\n"); std::string open("'''");
if(this->width_ + 6 < s.str.size())
{
open += '\n'; // the first newline is ignored by TOML spec
}
const std::string close("'''"); const std::string close("'''");
return open + s.str + close; return open + s.str + close;
} }
@@ -489,7 +510,9 @@ struct serializer
switch(*i) switch(*i)
{ {
case '\\': {retval += "\\\\"; break;} case '\\': {retval += "\\\\"; break;}
case '\"': {retval += "\\\""; break;} // One or two consecutive "s are allowed.
// Later we will check there are no three consecutive "s.
// case '\"': {retval += "\\\""; break;}
case '\b': {retval += "\\b"; break;} case '\b': {retval += "\\b"; break;}
case '\t': {retval += "\\t"; break;} case '\t': {retval += "\\t"; break;}
case '\f': {retval += "\\f"; break;} case '\f': {retval += "\\f"; break;}
@@ -510,6 +533,23 @@ struct serializer
default: {retval += *i; break;} default: {retval += *i; break;}
} }
} }
// Only 1 or 2 consecutive `"`s are allowed in multiline basic string.
// 3 consecutive `"`s are considered as a closing delimiter.
// We need to check if there are 3 or more consecutive `"`s and insert
// backslash to break them down into several short `"`s like the `str6`
// in the following example.
// ```toml
// str4 = """Here are two quotation marks: "". Simple enough."""
// # str5 = """Here are three quotation marks: """.""" # INVALID
// str5 = """Here are three quotation marks: ""\"."""
// str6 = """Here are fifteen quotation marks: ""\"""\"""\"""\"""\"."""
// ```
auto found_3_quotes = retval.find("\"\"\"");
while(found_3_quotes != std::string::npos)
{
retval.replace(found_3_quotes, 3, "\"\"\\\"");
found_3_quotes = retval.find("\"\"\"");
}
return retval; return retval;
} }
@@ -579,7 +619,7 @@ struct serializer
// print non-table stuff first. because after printing [foo.bar], the // print non-table stuff first. because after printing [foo.bar], the
// remaining non-table values will be assigned into [foo.bar], not [foo] // remaining non-table values will be assigned into [foo.bar], not [foo]
for(const auto kv : v) for(const auto& kv : v)
{ {
if(kv.second.is_table() || is_array_of_tables(kv.second)) if(kv.second.is_table() || is_array_of_tables(kv.second))
{ {