1 files changed, 4 insertions, 0 deletions
diff --git a/streamingvisitors/src/vespa/vsm/searcher/tokenizereader.h b/streamingvisitors/src/vespa/vsm/searcher/tokenizereader.h
index 76ca2e8d24b..f10c8910e82 100644
--- a/streamingvisitors/src/vespa/vsm/searcher/tokenizereader.h
+++ b/streamingvisitors/src/vespa/vsm/searcher/tokenizereader.h
@@ -7,6 +7,10 @@
 
 namespace vsm {
 
+/**
+ * Handles tokenization of utf8 input with on the fly normalization.
+ * It handles Normalizing::NONE, Normalizing::LOWERCASE, and Normalizing::LOWERCASE_AND_FOLD
+ */
 class TokenizeReader {
 public:
     using byte = search::byte;