|
@@ -29,7 +29,9 @@ tokenizer &tokenizer::ignore_empty_tokens(bool new_ignore_empty_tokens) {
|
|
|
|
|
|
|
|
std::vector<std::string> tokenizer::operator()(std::string const &input) const {
|
|
std::vector<std::string> tokenizer::operator()(std::string const &input) const {
|
|
|
std::vector<std::string> rval;
|
|
std::vector<std::string> rval;
|
|
|
- size_t const max = max_outputs_ ? max_outputs_ - truncate_ : 0xFFFFFFFF;
|
|
|
|
|
|
|
+ // If max_outputs_ == infinite_outputs, this will be infinite enough to work
|
|
|
|
|
+ // since we'll hit overflow on the string itself before this.
|
|
|
|
|
+ size_t const max = max_outputs_ - truncate_;
|
|
|
size_t i = 0;
|
|
size_t i = 0;
|
|
|
for (size_t n = input.find(divider_);
|
|
for (size_t n = input.find(divider_);
|
|
|
n != std::string::npos && rval.size() < max;
|
|
n != std::string::npos && rval.size() < max;
|
|
@@ -39,7 +41,7 @@ std::vector<std::string> tokenizer::operator()(std::string const &input) const {
|
|
|
}
|
|
}
|
|
|
rval.emplace_back(input.substr(i, n - i));
|
|
rval.emplace_back(input.substr(i, n - i));
|
|
|
}
|
|
}
|
|
|
- if (rval.size() < max_outputs_ || !max_outputs_) {
|
|
|
|
|
|
|
+ if (rval.size() < max_outputs_) {
|
|
|
rval.emplace_back(input.substr(i));
|
|
rval.emplace_back(input.substr(i));
|
|
|
}
|
|
}
|
|
|
return rval;
|
|
return rval;
|