Add coverage, and fill out tests.

include/string_utils/tokenizer.h

@@ -19,7 +19,7 @@ public:
 private:
   std::string divider_;
   size_t max_outputs_{infinite_outputs};
-  bool truncate_{true};
+  bool truncate_{false};
   bool ignore_empty_tokens_{true};
 
 public:

src/tokenizer.cxx

@@ -31,7 +31,7 @@ std::vector<std::string> tokenizer::operator()(std::string const &input) const {
   std::vector<std::string> rval;
   // If max_outputs_ == infinite_outputs, this will be infinite enough to work
   // since we'll hit overflow on the string itself before this.
-  size_t const max = max_outputs_ - truncate_;
+  size_t const max = max_outputs_ - !truncate_;
   size_t i = 0;
   for (size_t n = input.find(divider_);
        n != std::string::npos && rval.size() < max;
@@ -41,6 +41,10 @@ std::vector<std::string> tokenizer::operator()(std::string const &input) const {
     }
     rval.emplace_back(input.substr(i, n - i));
   }
+  // Special Handling for the final token
+  if (ignore_empty_tokens_ && input.find(divider_, i) == i) {
+    ++i;
+  }
   if (rval.size() < max_outputs_) {
     rval.emplace_back(input.substr(i));
   }

string-utils.xcodeproj/xcshareddata/xcschemes/string-utils.xcscheme

@@ -0,0 +1,78 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Scheme
+   LastUpgradeVersion = "1130"
+   version = "1.3">
+   <BuildAction
+      parallelizeBuildables = "YES"
+      buildImplicitDependencies = "YES">
+      <BuildActionEntries>
+         <BuildActionEntry
+            buildForTesting = "YES"
+            buildForRunning = "YES"
+            buildForProfiling = "YES"
+            buildForArchiving = "YES"
+            buildForAnalyzing = "YES">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "CD266861252FF4B600B3E667"
+               BuildableName = "libstring-utils.a"
+               BlueprintName = "string-utils"
+               ReferencedContainer = "container:string-utils.xcodeproj">
+            </BuildableReference>
+         </BuildActionEntry>
+      </BuildActionEntries>
+   </BuildAction>
+   <TestAction
+      buildConfiguration = "Debug"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      shouldUseLaunchSchemeArgsEnv = "YES"
+      codeCoverageEnabled = "YES">
+      <Testables>
+         <TestableReference
+            skipped = "NO">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "CD266885252FFAAE00B3E667"
+               BuildableName = "string_utils-test.xctest"
+               BlueprintName = "string_utils-test"
+               ReferencedContainer = "container:string-utils.xcodeproj">
+            </BuildableReference>
+         </TestableReference>
+      </Testables>
+   </TestAction>
+   <LaunchAction
+      buildConfiguration = "Debug"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      launchStyle = "0"
+      useCustomWorkingDirectory = "NO"
+      ignoresPersistentStateOnLaunch = "NO"
+      debugDocumentVersioning = "YES"
+      debugServiceExtension = "internal"
+      allowLocationSimulation = "YES">
+   </LaunchAction>
+   <ProfileAction
+      buildConfiguration = "Release"
+      shouldUseLaunchSchemeArgsEnv = "YES"
+      savedToolIdentifier = ""
+      useCustomWorkingDirectory = "NO"
+      debugDocumentVersioning = "YES">
+      <MacroExpansion>
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "CD266861252FF4B600B3E667"
+            BuildableName = "libstring-utils.a"
+            BlueprintName = "string-utils"
+            ReferencedContainer = "container:string-utils.xcodeproj">
+         </BuildableReference>
+      </MacroExpansion>
+   </ProfileAction>
+   <AnalyzeAction
+      buildConfiguration = "Debug">
+   </AnalyzeAction>
+   <ArchiveAction
+      buildConfiguration = "Release"
+      revealArchiveInOrganizer = "YES">
+   </ArchiveAction>
+</Scheme>

test/tokenizer_test.cxx

@@ -25,3 +25,48 @@ TEST(TokenizerTest, SplitsStringOverToken) {
   std::vector<std::string> const expected{"A", "B", "C", "D"};
   EXPECT_THAT(split(input, "."), expected);
 }
+
+TEST(TokenizerTest, SplitsStringUpToNTimes) {
+  std::string const input = "A.B.C.D";
+  std::vector<std::string> const expected{"A", "B", "C.D"};
+  EXPECT_THAT(split(input, ".", 3), expected);
+}
+
+TEST(TokenizerTest, IgnoresEmptyElementsAtStart) {
+  std::string const input = ".A.B.C";
+  std::vector<std::string> const expected{"A", "B", "C"};
+  EXPECT_THAT(split(input, ".", 3), expected);
+}
+
+TEST(TokenizerTest, IgnoresEmptyElements) {
+  std::string const input = "A..B.C";
+  std::vector<std::string> const expected{"A", "B", "C"};
+  EXPECT_THAT(split(input, ".", 3), expected);
+}
+
+TEST(TokenizerTest, IgnoresEmptyElementsOnEnd) {
+  std::string const input = "A.B..C";
+  std::vector<std::string> const expected{"A", "B", "C"};
+  EXPECT_THAT(split(input, ".", 3), expected);
+}
+
+TEST(TokenizerTest, TruncateDiscardsOverageInsteadOfNotParsingPast) {
+  std::string const input = "A.B.C.D";
+  std::vector<std::string> const expected{"A", "B", "C"};
+  EXPECT_THAT(tokenizer(".").max_outputs(3).truncate(true)(input),
+              expected);
+}
+
+TEST(TokenizerTest, EmptyIsPlacedCorrectlyWhenEnabled) {
+  std::string const input = "A..B.C";
+  std::vector<std::string> const expected{"A", "", "B.C"};
+  EXPECT_THAT(tokenizer(".").max_outputs(3).ignore_empty_tokens(false)(input),
+              expected);
+}
+
+TEST(TokenizerTest, MaxSizeWithEmptyCanResultInTokenWithDividerPrefix) {
+  std::string const input = "A.B..C";
+  std::vector<std::string> const expected{"A", "B", ".C"};
+  EXPECT_THAT(tokenizer(".").max_outputs(3).ignore_empty_tokens(false)(input),
+              expected);
+}