Use the simple tokenizer, no need to pull in boost

2 files changed, 26 insertions, 15 deletions

diff --git a/vespalib/src/vespa/vespalib/text/stringtokenizer.cpp b/vespalib/src/vespa/vespalib/text/stringtokenizer.cpp
index 17f08ea7f05..da7fc262669 100644
--- a/vespalib/src/vespa/vespalib/text/stringtokenizer.cpp
+++ b/vespalib/src/vespa/vespalib/text/stringtokenizer.cpp
@@ -7,13 +7,13 @@ namespace {
 class AsciiSet
 {
 public:
-    AsciiSet(vespalib::stringref s) {
+    explicit AsciiSet(vespalib::stringref s) {
         memset(_set, 0, sizeof(_set));
-        for (size_t i(0), m(s.size()); i < m; i++) {
-            add(s[i]);
+        for (char c : s) {
+            add(c);
         }
     }
-    bool contains(uint8_t c) const {
+    [[nodiscard]] bool contains(uint8_t c) const {
         return _set[c];
     }
     void add(uint8_t c) {
@@ -48,8 +48,8 @@ stripString(vespalib::stringref source, const AsciiSet & strip)
 size_t
 countSeparators(vespalib::stringref source, const AsciiSet & sep) {
     size_t count(0);
-    for (Token::size_type i = 0; i < source.size(); ++i) {
-        if (sep.contains(source[i])) {
+    for (char c : source) {
+        if (sep.contains(c)) {
             count++;
         }
     }
@@ -68,7 +68,7 @@ parse(TokenList& output, vespalib::stringref source, const AsciiSet & separators
     }
     output.push_back(stripString(source.substr(start), strip));
     // Don't keep a single empty element
-    if (output.size() == 1 && output[0].size() == 0) output.pop_back();
+    if (output.size() == 1 && output[0].empty()) output.pop_back();
 }
 
 } // private namespace
@@ -86,6 +86,8 @@ StringTokenizer::StringTokenizer(vespalib::stringref source,
     parse(_tokens, source, sep, str);
 }
 
+StringTokenizer::~StringTokenizer() = default;
+
 void
 StringTokenizer::removeEmptyTokens()
 {
diff --git a/vespalib/src/vespa/vespalib/text/stringtokenizer.h b/vespalib/src/vespa/vespalib/text/stringtokenizer.h
index a2c9b3520a1..49f0e6d1050 100644
--- a/vespalib/src/vespa/vespalib/text/stringtokenizer.h
+++ b/vespalib/src/vespa/vespalib/text/stringtokenizer.h
@@ -41,25 +41,34 @@ public:
      * @param separators The characters to be used as token separators
      * @param strip      Characters to be stripped from both ends of each token
      **/
+    explicit StringTokenizer(vespalib::stringref source)
+        : StringTokenizer(source, ",")
+    {}
     StringTokenizer(vespalib::stringref source,
-                    vespalib::stringref separators = ",",
-                    vespalib::stringref strip = " \t\f\r\n");
+                             vespalib::stringref separators)
+        : StringTokenizer(source, separators, " \t\f\r\n")
+    {}
+    StringTokenizer(vespalib::stringref source,
+                    vespalib::stringref separators,
+                    vespalib::stringref strip);
+    StringTokenizer(StringTokenizer &&) noexcept = default;
+    StringTokenizer & operator=(StringTokenizer &&) noexcept = default;
+    ~StringTokenizer();
 
     /** Remove any empty tokens from the token list */
     void removeEmptyTokens();
 
     /** How many tokens is in the current token list? */
-    unsigned int size() const { return _tokens.size(); }
+    [[nodiscard]] unsigned int size() const { return _tokens.size(); }
 
     /** Access a token from the current token list */
-    const Token & operator[](unsigned int index) const
-        { return _tokens[index]; }
+    const Token & operator[](unsigned int index) const { return _tokens[index]; }
 
-    Iterator begin() const { return _tokens.begin(); }
-    Iterator end() const { return _tokens.end(); }
+    [[nodiscard]] Iterator begin() const { return _tokens.begin(); }
+    [[nodiscard]] Iterator end() const { return _tokens.end(); }
 
     /** Access the entire token list */
-    const TokenList & getTokens() const { return _tokens; }
+    [[nodiscard]] const TokenList & getTokens() const { return _tokens; }
 
 private:
     TokenList _tokens;