Move BufferWriter back to searchlib.

23 files changed, 113 insertions, 20 deletions

diff --git a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp
index eeaf85472c3..d8761f69d71 100644
--- a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp
+++ b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp
@@ -19,7 +19,7 @@
 #include <vespa/searchlib/parsequery/parse.h>
 #include <vespa/searchlib/util/randomgenerator.h>
 #include <vespa/vespalib/testkit/testapp.h>
-#include <vespa/vespalib/util/bufferwriter.h>
+#include <vespa/searchlib/util/bufferwriter.h>
 #include <vespa/vespalib/util/compress.h>
 
 #include <vespa/searchlib/attribute/attributevector.hpp>
diff --git a/searchlib/src/tests/attribute/save_target/attribute_save_target_test.cpp b/searchlib/src/tests/attribute/save_target/attribute_save_target_test.cpp
index c746a0aa120..103551dd93a 100644
--- a/searchlib/src/tests/attribute/save_target/attribute_save_target_test.cpp
+++ b/searchlib/src/tests/attribute/save_target/attribute_save_target_test.cpp
@@ -7,7 +7,7 @@
 #include <vespa/searchlib/test/directory_handler.h>
 #include <vespa/searchlib/util/fileutil.h>
 #include <vespa/vespalib/gtest/gtest.h>
-#include <vespa/vespalib/util/bufferwriter.h>
+#include <vespa/searchlib/util/bufferwriter.h>
 #include <vespa/vespalib/util/exceptions.h>
 
 #include <vespa/log/log.h>
diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp
index 39a7e53ca8c..56b8460fbb6 100644
--- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp
+++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp
@@ -22,7 +22,7 @@
 #include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/test/insertion_operators.h>
 #include <vespa/vespalib/testkit/test_kit.h>
-#include <vespa/vespalib/util/bufferwriter.h>
+#include <vespa/searchlib/util/bufferwriter.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP("tensorattribute_test");
diff --git a/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp b/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp
index b9e27d413f3..7aa78fbe06b 100644
--- a/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp
+++ b/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp
@@ -3,7 +3,7 @@
 #include <vespa/searchlib/tensor/hnsw_graph.h>
 #include <vespa/searchlib/tensor/hnsw_index_saver.h>
 #include <vespa/searchlib/tensor/hnsw_index_loader.h>
-#include <vespa/vespalib/util/bufferwriter.h>
+#include <vespa/searchlib/util/bufferwriter.h>
 #include <vespa/searchlib/util/fileutil.h>
 #include <vespa/vespalib/gtest/gtest.h>
 #include <vector>
diff --git a/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp b/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp
index 8787712663d..04addfa19f6 100644
--- a/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp
+++ b/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp
@@ -2,7 +2,7 @@
 
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/vespalib/stllike/string.h>
-#include <vespa/vespalib/util/bufferwriter.h>
+#include <vespa/searchlib/util/bufferwriter.h>
 #include <vespa/searchlib/util/drainingbufferwriter.h>
 #include <vespa/vespalib/util/rand48.h>
 
diff --git a/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.h b/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.h
index c4a26ff79c5..a6052f5f6b6 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.h
+++ b/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.h
@@ -3,7 +3,7 @@
 #pragma once
 
 #include "iattributefilewriter.h"
-#include <vespa/vespalib/util/bufferwriter.h>
+#include <vespa/searchlib/util/bufferwriter.h>
 
 namespace search {
 
diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp
index cda2bea9fb1..cd2a423782e 100644
--- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp
@@ -9,7 +9,7 @@
 #include <vespa/vespalib/btree/btreeroot.hpp>
 #include <vespa/vespalib/datastore/datastore.hpp>
 #include <vespa/vespalib/datastore/unique_store_dictionary.hpp>
-#include <vespa/vespalib/util/bufferwriter.h>
+#include <vespa/searchlib/util/bufferwriter.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".searchlib.attribute.enum_store_dictionary");
diff --git a/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp
index cf394623abd..e0ce0ef155e 100644
--- a/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp
@@ -3,7 +3,7 @@
 #include "enumattributesaver.h"
 #include "i_enum_store_dictionary.h"
 #include "iattributesavetarget.h"
-#include <vespa/vespalib/util/bufferwriter.h>
+#include <vespa/searchlib/util/bufferwriter.h>
 #include <vespa/vespalib/datastore/unique_store_enumerator.hpp>
 
 namespace search {
diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp
index e39ecc17aac..b76cf0c4192 100644
--- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp
@@ -17,7 +17,7 @@
 #include <vespa/vespalib/datastore/unique_store.hpp>
 #include <vespa/vespalib/datastore/unique_store_string_allocator.hpp>
 #include <vespa/vespalib/util/array.hpp>
-#include <vespa/vespalib/util/bufferwriter.h>
+#include <vespa/searchlib/util/bufferwriter.h>
 #include <vespa/searchcommon/common/compaction_strategy.h>
 
 namespace search {
diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp
index 51821389a92..fe19bf236aa 100644
--- a/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp
@@ -3,7 +3,7 @@
 #include "multinumericattributesaver.h"
 #include "multivalueattributesaverutils.h"
 #include "multivalue.h"
-#include <vespa/vespalib/util/bufferwriter.h>
+#include <vespa/searchlib/util/bufferwriter.h>
 
 using vespalib::GenerationHandler;
 using search::multivalueattributesaver::CountWriter;
diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp
index eafa5bf0e1f..003fb9fa6b7 100644
--- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp
@@ -9,7 +9,7 @@
 #include <vespa/fastlib/io/bufferedfile.h>
 #include <vespa/vespalib/text/utf8.h>
 #include <vespa/vespalib/text/lowercase.h>
-#include <vespa/vespalib/util/bufferwriter.h>
+#include <vespa/searchlib/util/bufferwriter.h>
 #include <vespa/vespalib/util/regexp.h>
 #include <vespa/searchlib/query/query_term_ucs4.h>
 
diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.cpp b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.cpp
index 697dbcfd27b..16acf70eb59 100644
--- a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.cpp
@@ -1,7 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "multivalueattributesaverutils.h"
-#include <vespa/vespalib/util/bufferwriter.h>
+#include <vespa/searchlib/util/bufferwriter.h>
 
 namespace search::multivalueattributesaver {
 
diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h
index 213fd4c4777..e34f43deb0b 100644
--- a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h
+++ b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h
@@ -3,7 +3,7 @@
 #pragma once
 
 #include "iattributesavetarget.h"
-#include <vespa/vespalib/util/bufferwriter.h>
+#include <vespa/searchlib/util/bufferwriter.h>
 #include <vespa/vespalib/util/arrayref.h>
 
 namespace search::multivalueattributesaver {
diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp
index e0a6ea193e0..4385911d0b3 100644
--- a/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp
@@ -2,7 +2,7 @@
 
 #include "reference_attribute_saver.h"
 #include <vespa/vespalib/util/array.hpp>
-#include <vespa/vespalib/util/bufferwriter.h>
+#include <vespa/searchlib/util/bufferwriter.h>
 #include "iattributesavetarget.h"
 
 
diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp
index afa893458b4..f3552db7c05 100644
--- a/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp
@@ -2,7 +2,7 @@
 
 #include "singleenumattributesaver.h"
 #include <vespa/vespalib/util/array.hpp>
-#include <vespa/vespalib/util/bufferwriter.h>
+#include <vespa/searchlib/util/bufferwriter.h>
 #include "iattributesavetarget.h"
 
 
diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp
index 406cbbbe447..e362ecff6cd 100644
--- a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp
@@ -9,7 +9,7 @@
 #include <vespa/fastlib/io/bufferedfile.h>
 #include <vespa/vespalib/text/utf8.h>
 #include <vespa/vespalib/text/lowercase.h>
-#include <vespa/vespalib/util/bufferwriter.h>
+#include <vespa/searchlib/util/bufferwriter.h>
 #include <vespa/vespalib/util/regexp.h>
 #include <vespa/searchlib/query/query_term_ucs4.h>
 
diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp
index fd8d6162f01..362e1b45266 100644
--- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp
+++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp
@@ -3,7 +3,7 @@
 #include "dense_tensor_attribute_saver.h"
 #include "dense_tensor_store.h"
 #include "nearest_neighbor_index_saver.h"
-#include <vespa/vespalib/util/bufferwriter.h>
+#include <vespa/searchlib/util/bufferwriter.h>
 #include <vespa/searchlib/attribute/iattributesavetarget.h>
 
 using vespalib::GenerationHandler;
diff --git a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp b/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp
index 2f0bb731aa4..81ec3a5218e 100644
--- a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp
+++ b/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp
@@ -2,7 +2,7 @@
 
 #include "generic_tensor_attribute_saver.h"
 #include "generic_tensor_store.h"
-#include <vespa/vespalib/util/bufferwriter.h>
+#include <vespa/searchlib/util/bufferwriter.h>
 #include <vespa/searchlib/attribute/iattributesavetarget.h>
 
 using vespalib::GenerationHandler;
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp
index acff30f8cbf..46a988d575e 100644
--- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp
@@ -2,7 +2,7 @@
 
 #include "hnsw_index_saver.h"
 #include "hnsw_graph.h"
-#include <vespa/vespalib/util/bufferwriter.h>
+#include <vespa/searchlib/util/bufferwriter.h>
 
 namespace search::tensor {
 
diff --git a/searchlib/src/vespa/searchlib/util/CMakeLists.txt b/searchlib/src/vespa/searchlib/util/CMakeLists.txt
index ac5fc5f54fc..68dee99339f 100644
--- a/searchlib/src/vespa/searchlib/util/CMakeLists.txt
+++ b/searchlib/src/vespa/searchlib/util/CMakeLists.txt
@@ -1,6 +1,7 @@
 # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 vespa_add_library(searchlib_util OBJECT
     SOURCES
+    bufferwriter.cpp
     comprbuffer.cpp
     comprfile.cpp
     dirtraverse.cpp
diff --git a/searchlib/src/vespa/searchlib/util/bufferwriter.cpp b/searchlib/src/vespa/searchlib/util/bufferwriter.cpp
new file mode 100644
index 00000000000..6e57d6f58d4
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/util/bufferwriter.cpp
@@ -0,0 +1,36 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "bufferwriter.h"
+
+namespace search {
+
+BufferWriter::BufferWriter()
+    : _cur(nullptr),
+      _end(nullptr),
+      _start(nullptr)
+{
+}
+
+BufferWriter::~BufferWriter() = default;
+
+void
+BufferWriter::writeSlow(const void *src, size_t len)
+{
+    size_t residue = len;
+    const char *csrc = static_cast<const char *>(src);
+    for (;;) {
+        size_t maxLen = freeLen();
+        if (residue <= maxLen) {
+            writeFast(csrc, residue);
+            break;
+        }
+        if (maxLen != 0) {
+            writeFast(csrc, maxLen);
+            csrc += maxLen;
+            residue -= maxLen;
+        }
+        flush();
+    }
+}
+
+} // namespace search
diff --git a/searchlib/src/vespa/searchlib/util/bufferwriter.h b/searchlib/src/vespa/searchlib/util/bufferwriter.h
new file mode 100644
index 00000000000..3da6e3f8030
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/util/bufferwriter.h
@@ -0,0 +1,56 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <cstddef>
+
+namespace search {
+
+/**
+ * Abstract class to write to a buffer with an abstract backing store
+ * and abstract backing buffer.  Each time backing buffer is full,
+ * flush() is called to resize it or drain it to the backing store.
+ */
+class BufferWriter
+{
+    char *_cur;
+    char *_end;
+    char *_start;
+protected:
+    void rewind() { _cur = _start; }
+
+    void setup(void *start, size_t len) {
+        _start = static_cast<char *>(start);
+        _end = _start + len;
+        rewind();
+    }
+
+    size_t freeLen() const { return _end - _cur; }
+    size_t usedLen() const { return _cur - _start; }
+
+    void writeFast(const void *src, size_t len)
+    {
+        __builtin_memcpy(_cur, src, len);
+        _cur += len;
+    }
+
+    void writeSlow(const void *src, size_t len);
+
+public:
+    BufferWriter();
+
+    virtual ~BufferWriter();
+
+    virtual void flush() = 0;
+
+    void write(const void *src, size_t len)
+    {
+        if (__builtin_expect(len <= freeLen(), true)) {
+            writeFast(src, len);
+            return;
+        }
+        writeSlow(src, len);
+    }
+};
+
+} // namespace search
diff --git a/searchlib/src/vespa/searchlib/util/drainingbufferwriter.h b/searchlib/src/vespa/searchlib/util/drainingbufferwriter.h
index 38d8337fab5..f0a58b0d25a 100644
--- a/searchlib/src/vespa/searchlib/util/drainingbufferwriter.h
+++ b/searchlib/src/vespa/searchlib/util/drainingbufferwriter.h
@@ -2,7 +2,7 @@
 
 #pragma once
 
-#include <vespa/vespalib/util/bufferwriter.h>
+#include <vespa/searchlib/util/bufferwriter.h>
 #include <vector>
 #include <cstdint>