From 2c0e3e3c68772d06b2a842c1c5403b382763deec Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Sat, 25 Apr 2020 23:42:21 +0200 Subject: Move BufferWriter back to searchlib. --- .../proton/documentmetastore/documentmetastore.cpp | 2 +- .../documentmetastore/documentmetastoresaver.cpp | 2 +- .../enumeratedsave/enumeratedsave_test.cpp | 2 +- .../save_target/attribute_save_target_test.cpp | 2 +- .../tensorattribute/tensorattribute_test.cpp | 2 +- .../tensor/hnsw_saver/hnsw_save_load_test.cpp | 2 +- .../tests/util/bufferwriter/bufferwriter_test.cpp | 2 +- .../attribute/attributefilebufferwriter.h | 2 +- .../searchlib/attribute/enum_store_dictionary.cpp | 2 +- .../searchlib/attribute/enumattributesaver.cpp | 2 +- .../src/vespa/searchlib/attribute/enumstore.hpp | 2 +- .../attribute/multinumericattributesaver.cpp | 2 +- .../searchlib/attribute/multistringattribute.hpp | 2 +- .../attribute/multivalueattributesaverutils.cpp | 2 +- .../attribute/multivalueattributesaverutils.h | 2 +- .../attribute/reference_attribute_saver.cpp | 2 +- .../attribute/singleenumattributesaver.cpp | 2 +- .../searchlib/attribute/singlestringattribute.hpp | 2 +- .../tensor/dense_tensor_attribute_saver.cpp | 2 +- .../tensor/generic_tensor_attribute_saver.cpp | 2 +- .../vespa/searchlib/tensor/hnsw_index_saver.cpp | 2 +- searchlib/src/vespa/searchlib/util/CMakeLists.txt | 1 + .../src/vespa/searchlib/util/bufferwriter.cpp | 36 ++++++++++++++ searchlib/src/vespa/searchlib/util/bufferwriter.h | 56 ++++++++++++++++++++++ .../vespa/searchlib/util/drainingbufferwriter.h | 2 +- .../src/vespa/vespalib/datastore/unique_store.hpp | 1 - vespalib/src/vespa/vespalib/util/CMakeLists.txt | 1 - vespalib/src/vespa/vespalib/util/bufferwriter.cpp | 36 -------------- vespalib/src/vespa/vespalib/util/bufferwriter.h | 56 ---------------------- 29 files changed, 115 insertions(+), 116 deletions(-) create mode 100644 searchlib/src/vespa/searchlib/util/bufferwriter.cpp create mode 100644 searchlib/src/vespa/searchlib/util/bufferwriter.h delete mode 100644 vespalib/src/vespa/vespalib/util/bufferwriter.cpp delete mode 100644 vespalib/src/vespa/vespalib/util/bufferwriter.h diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index 3e9ef787f74..c36ee8e474a 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include #include "document_meta_store_versions.h" diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp index 2a3fd14c6d2..7d85c01ee74 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "documentmetastoresaver.h" -#include +#include #include "document_meta_store_versions.h" #include diff --git a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp index eeaf85472c3..d8761f69d71 100644 --- a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp +++ b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include diff --git a/searchlib/src/tests/attribute/save_target/attribute_save_target_test.cpp b/searchlib/src/tests/attribute/save_target/attribute_save_target_test.cpp index c746a0aa120..103551dd93a 100644 --- a/searchlib/src/tests/attribute/save_target/attribute_save_target_test.cpp +++ b/searchlib/src/tests/attribute/save_target/attribute_save_target_test.cpp @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 39a7e53ca8c..56b8460fbb6 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include LOG_SETUP("tensorattribute_test"); diff --git a/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp b/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp index b9e27d413f3..7aa78fbe06b 100644 --- a/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp b/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp index 8787712663d..04addfa19f6 100644 --- a/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp +++ b/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include diff --git a/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.h b/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.h index c4a26ff79c5..a6052f5f6b6 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.h +++ b/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.h @@ -3,7 +3,7 @@ #pragma once #include "iattributefilewriter.h" -#include +#include namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp index cda2bea9fb1..cd2a423782e 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include LOG_SETUP(".searchlib.attribute.enum_store_dictionary"); diff --git a/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp index cf394623abd..e0ce0ef155e 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp @@ -3,7 +3,7 @@ #include "enumattributesaver.h" #include "i_enum_store_dictionary.h" #include "iattributesavetarget.h" -#include +#include #include namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp index e39ecc17aac..b76cf0c4192 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp index 51821389a92..fe19bf236aa 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp @@ -3,7 +3,7 @@ #include "multinumericattributesaver.h" #include "multivalueattributesaverutils.h" #include "multivalue.h" -#include +#include using vespalib::GenerationHandler; using search::multivalueattributesaver::CountWriter; diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp index eafa5bf0e1f..003fb9fa6b7 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.cpp b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.cpp index 697dbcfd27b..16acf70eb59 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "multivalueattributesaverutils.h" -#include +#include namespace search::multivalueattributesaver { diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h index 213fd4c4777..e34f43deb0b 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h @@ -3,7 +3,7 @@ #pragma once #include "iattributesavetarget.h" -#include +#include #include namespace search::multivalueattributesaver { diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp index e0a6ea193e0..4385911d0b3 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp @@ -2,7 +2,7 @@ #include "reference_attribute_saver.h" #include -#include +#include #include "iattributesavetarget.h" diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp index afa893458b4..f3552db7c05 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp @@ -2,7 +2,7 @@ #include "singleenumattributesaver.h" #include -#include +#include #include "iattributesavetarget.h" diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp index 406cbbbe447..e362ecff6cd 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp index fd8d6162f01..362e1b45266 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp @@ -3,7 +3,7 @@ #include "dense_tensor_attribute_saver.h" #include "dense_tensor_store.h" #include "nearest_neighbor_index_saver.h" -#include +#include #include using vespalib::GenerationHandler; diff --git a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp b/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp index 2f0bb731aa4..81ec3a5218e 100644 --- a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp +++ b/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp @@ -2,7 +2,7 @@ #include "generic_tensor_attribute_saver.h" #include "generic_tensor_store.h" -#include +#include #include using vespalib::GenerationHandler; diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp index acff30f8cbf..46a988d575e 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp @@ -2,7 +2,7 @@ #include "hnsw_index_saver.h" #include "hnsw_graph.h" -#include +#include namespace search::tensor { diff --git a/searchlib/src/vespa/searchlib/util/CMakeLists.txt b/searchlib/src/vespa/searchlib/util/CMakeLists.txt index ac5fc5f54fc..68dee99339f 100644 --- a/searchlib/src/vespa/searchlib/util/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/util/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchlib_util OBJECT SOURCES + bufferwriter.cpp comprbuffer.cpp comprfile.cpp dirtraverse.cpp diff --git a/searchlib/src/vespa/searchlib/util/bufferwriter.cpp b/searchlib/src/vespa/searchlib/util/bufferwriter.cpp new file mode 100644 index 00000000000..6e57d6f58d4 --- /dev/null +++ b/searchlib/src/vespa/searchlib/util/bufferwriter.cpp @@ -0,0 +1,36 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "bufferwriter.h" + +namespace search { + +BufferWriter::BufferWriter() + : _cur(nullptr), + _end(nullptr), + _start(nullptr) +{ +} + +BufferWriter::~BufferWriter() = default; + +void +BufferWriter::writeSlow(const void *src, size_t len) +{ + size_t residue = len; + const char *csrc = static_cast(src); + for (;;) { + size_t maxLen = freeLen(); + if (residue <= maxLen) { + writeFast(csrc, residue); + break; + } + if (maxLen != 0) { + writeFast(csrc, maxLen); + csrc += maxLen; + residue -= maxLen; + } + flush(); + } +} + +} // namespace search diff --git a/searchlib/src/vespa/searchlib/util/bufferwriter.h b/searchlib/src/vespa/searchlib/util/bufferwriter.h new file mode 100644 index 00000000000..3da6e3f8030 --- /dev/null +++ b/searchlib/src/vespa/searchlib/util/bufferwriter.h @@ -0,0 +1,56 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include + +namespace search { + +/** + * Abstract class to write to a buffer with an abstract backing store + * and abstract backing buffer. Each time backing buffer is full, + * flush() is called to resize it or drain it to the backing store. + */ +class BufferWriter +{ + char *_cur; + char *_end; + char *_start; +protected: + void rewind() { _cur = _start; } + + void setup(void *start, size_t len) { + _start = static_cast(start); + _end = _start + len; + rewind(); + } + + size_t freeLen() const { return _end - _cur; } + size_t usedLen() const { return _cur - _start; } + + void writeFast(const void *src, size_t len) + { + __builtin_memcpy(_cur, src, len); + _cur += len; + } + + void writeSlow(const void *src, size_t len); + +public: + BufferWriter(); + + virtual ~BufferWriter(); + + virtual void flush() = 0; + + void write(const void *src, size_t len) + { + if (__builtin_expect(len <= freeLen(), true)) { + writeFast(src, len); + return; + } + writeSlow(src, len); + } +}; + +} // namespace search diff --git a/searchlib/src/vespa/searchlib/util/drainingbufferwriter.h b/searchlib/src/vespa/searchlib/util/drainingbufferwriter.h index 38d8337fab5..f0a58b0d25a 100644 --- a/searchlib/src/vespa/searchlib/util/drainingbufferwriter.h +++ b/searchlib/src/vespa/searchlib/util/drainingbufferwriter.h @@ -2,7 +2,7 @@ #pragma once -#include +#include #include #include diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp index 33830bbe4ab..956d4c16d40 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp @@ -10,7 +10,6 @@ #include "unique_store_builder.hpp" #include "unique_store_dictionary.hpp" #include "unique_store_enumerator.hpp" -#include #include #include diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index e3397b54740..4029c4881c4 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -12,7 +12,6 @@ vespa_add_library(vespalib_vespalib_util OBJECT benchmark_timer.cpp blockingthreadstackexecutor.cpp box.cpp - bufferwriter.cpp classname.cpp closuretask.cpp compress.cpp diff --git a/vespalib/src/vespa/vespalib/util/bufferwriter.cpp b/vespalib/src/vespa/vespalib/util/bufferwriter.cpp deleted file mode 100644 index 6e57d6f58d4..00000000000 --- a/vespalib/src/vespa/vespalib/util/bufferwriter.cpp +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "bufferwriter.h" - -namespace search { - -BufferWriter::BufferWriter() - : _cur(nullptr), - _end(nullptr), - _start(nullptr) -{ -} - -BufferWriter::~BufferWriter() = default; - -void -BufferWriter::writeSlow(const void *src, size_t len) -{ - size_t residue = len; - const char *csrc = static_cast(src); - for (;;) { - size_t maxLen = freeLen(); - if (residue <= maxLen) { - writeFast(csrc, residue); - break; - } - if (maxLen != 0) { - writeFast(csrc, maxLen); - csrc += maxLen; - residue -= maxLen; - } - flush(); - } -} - -} // namespace search diff --git a/vespalib/src/vespa/vespalib/util/bufferwriter.h b/vespalib/src/vespa/vespalib/util/bufferwriter.h deleted file mode 100644 index 3da6e3f8030..00000000000 --- a/vespalib/src/vespa/vespalib/util/bufferwriter.h +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include - -namespace search { - -/** - * Abstract class to write to a buffer with an abstract backing store - * and abstract backing buffer. Each time backing buffer is full, - * flush() is called to resize it or drain it to the backing store. - */ -class BufferWriter -{ - char *_cur; - char *_end; - char *_start; -protected: - void rewind() { _cur = _start; } - - void setup(void *start, size_t len) { - _start = static_cast(start); - _end = _start + len; - rewind(); - } - - size_t freeLen() const { return _end - _cur; } - size_t usedLen() const { return _cur - _start; } - - void writeFast(const void *src, size_t len) - { - __builtin_memcpy(_cur, src, len); - _cur += len; - } - - void writeSlow(const void *src, size_t len); - -public: - BufferWriter(); - - virtual ~BufferWriter(); - - virtual void flush() = 0; - - void write(const void *src, size_t len) - { - if (__builtin_expect(len <= freeLen(), true)) { - writeFast(src, len); - return; - } - writeSlow(src, len); - } -}; - -} // namespace search -- cgit v1.2.3