Add tests for quoted strings (using csv style as is commonly used), change divider for escape tests to something better aligned to the use-case (spaces).

test/tokenizer_test.cxx

@@ -64,13 +64,37 @@ TEST(TokenizerTest, MaxSizeWithEmptyCanResultInTokenWithDividerPrefix) {
 }
 
 TEST(TokenizerTest, EscapableTokensStickTogether) {
-  std::string const input = R"(A.B\.C)";
-  std::vector<std::string> const expected{"A", "B.C"};
-  EXPECT_THAT(tokenizer(".").escapable(true)(input), expected);
+  std::string const input = R"(A B\ C)";
+  std::vector<std::string> const expected{"A", "B C"};
+  EXPECT_THAT(tokenizer(" ").escapable(true)(input), expected);
 }
 
 TEST(TokenizerTest, CorrectlySplitsWhenEvenEscapes) {
-  std::string const input = R"(A.B\\.C)";
+  std::string const input = R"(A B\\ C)";
   std::vector<std::string> const expected{"A", R"(B\\)", "C"};
-  EXPECT_THAT(tokenizer(".").escapable(true)(input), expected);
+  EXPECT_THAT(tokenizer(" ").escapable(true)(input), expected);
+}
+
+TEST(TokenizerTest, QuotesAreDiscarded) {
+  std::string const input = R"(A,"B",C)";
+  std::vector<std::string> const expected{"A", "B", "C"};
+  EXPECT_THAT(tokenizer(",", "\"")(input), expected);
+}
+
+TEST(TokenizerTest, QuotedTokensStickTogether) {
+  std::string const input = R"(A,"B,C")";
+  std::vector<std::string> const expected{"A", "B,C"};
+  EXPECT_THAT(tokenizer(",", "\"")(input), expected);
+}
+
+TEST(TokenizerTest, QuotedTokensAreAlwaysEscapable) {
+  std::string const input = R"(A,"B\",C")";
+  std::vector<std::string> const expected{"A", "B\",C"};
+  EXPECT_THAT(tokenizer(",", "\"")(input), expected);
+}
+
+TEST(TokenizerTest, QuotedTokensDontApplyOutOfFirstChar) {
+  std::string const input = R"(A,B",C")";
+  std::vector<std::string> const expected{"A", "B\"", "C\""};
+  EXPECT_THAT(tokenizer(",", "\"")(input), expected);
 }