diff options
11 files changed, 96 insertions, 118 deletions
diff --git a/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp b/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp index a1dc619b3e6..34a2d139498 100644 --- a/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp +++ b/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp @@ -28,7 +28,7 @@ #include <vespa/searchlib/attribute/attributevector.hpp> #include <vespa/searchlib/attribute/reference_attribute.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> -#include <vespa/searchlib/tensor/generic_tensor_attribute.h> +#include <vespa/searchlib/tensor/serialized_tensor_attribute.h> #include <vespa/searchlib/test/weighted_type_test_utils.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/testkit/testapp.h> @@ -48,7 +48,7 @@ using search::attribute::Reference; using search::attribute::ReferenceAttribute; using search::tensor::ITensorAttribute; using search::tensor::DenseTensorAttribute; -using search::tensor::GenericTensorAttribute; +using search::tensor::SerializedTensorAttribute; using search::tensor::TensorAttribute; using vespalib::eval::ValueType; using vespalib::eval::TensorSpec; @@ -457,7 +457,7 @@ TEST_F("require that tensor modify update is applied", } TEST_F("require that tensor add update is applied", - TensorFixture<GenericTensorAttribute>("tensor(x{})", "sparse_tensor")) + TensorFixture<SerializedTensorAttribute>("tensor(x{})", "sparse_tensor")) { f.setTensor(TensorSpec(f.type).add({{"x", "a"}}, 2)); f.applyValueUpdate(*f.attribute, 1, @@ -466,7 +466,7 @@ TEST_F("require that tensor add update is applied", } TEST_F("require that tensor add update to non-existing tensor creates empty tensor first", - TensorFixture<GenericTensorAttribute>("tensor(x{})", "sparse_tensor")) + TensorFixture<SerializedTensorAttribute>("tensor(x{})", "sparse_tensor")) { f.applyValueUpdate(*f.attribute, 1, TensorAddUpdate(makeTensorFieldValue(TensorSpec(f.type).add({{"x", "a"}}, 3)))); @@ -474,7 +474,7 @@ TEST_F("require that tensor add update to non-existing tensor creates empty tens } TEST_F("require that tensor remove update is applied", - TensorFixture<GenericTensorAttribute>("tensor(x{})", "sparse_tensor")) + TensorFixture<SerializedTensorAttribute>("tensor(x{})", "sparse_tensor")) { f.setTensor(TensorSpec(f.type).add({{"x", "a"}}, 2).add({{"x", "b"}}, 3)); f.applyValueUpdate(*f.attribute, 1, diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index efd17f773f3..1a342a92b3d 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -12,11 +12,11 @@ #include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/tensor/direct_tensor_attribute.h> #include <vespa/searchlib/tensor/doc_vector_access.h> -#include <vespa/searchlib/tensor/generic_tensor_attribute.h> #include <vespa/searchlib/tensor/hnsw_index.h> #include <vespa/searchlib/tensor/nearest_neighbor_index.h> #include <vespa/searchlib/tensor/nearest_neighbor_index_factory.h> #include <vespa/searchlib/tensor/nearest_neighbor_index_saver.h> +#include <vespa/searchlib/tensor/serialized_tensor_attribute.h> #include <vespa/searchlib/tensor/tensor_attribute.h> #include <vespa/searchlib/test/directory_handler.h> #include <vespa/searchlib/util/fileutil.h> @@ -40,7 +40,7 @@ using search::tensor::DefaultNearestNeighborIndexFactory; using search::tensor::DenseTensorAttribute; using search::tensor::DirectTensorAttribute; using search::tensor::DocVectorAccess; -using search::tensor::GenericTensorAttribute; +using search::tensor::SerializedTensorAttribute; using search::tensor::HnswIndex; using search::tensor::HnswNode; using search::tensor::NearestNeighborIndex; @@ -361,7 +361,7 @@ struct Fixture { } else if (_traits.use_direct_tensor_attribute) { return std::make_shared<DirectTensorAttribute>(_name, _cfg); } else { - return std::make_shared<GenericTensorAttribute>(_name, _cfg); + return std::make_shared<SerializedTensorAttribute>(_name, _cfg); } } diff --git a/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp b/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp index a0cf47f64e0..148d18f79ff 100644 --- a/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp +++ b/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp @@ -7,8 +7,8 @@ #include "singlenumericattribute.hpp" #include "singlestringattribute.h" #include "singleboolattribute.h" -#include <vespa/searchlib/tensor/generic_tensor_attribute.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> +#include <vespa/searchlib/tensor/serialized_tensor_attribute.h> namespace search { @@ -46,7 +46,7 @@ AttributeFactory::createSingleStd(stringref name, const Config & info) if (info.tensorType().is_dense()) { return std::make_shared<tensor::DenseTensorAttribute>(name, info); } else { - return std::make_shared<tensor::GenericTensorAttribute>(name, info); + return std::make_shared<tensor::SerializedTensorAttribute>(name, info); } case BasicType::REFERENCE: return std::make_shared<attribute::ReferenceAttribute>(name, info); diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt index 55e83fc6147..fac6d015a5f 100644 --- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt @@ -10,9 +10,6 @@ vespa_add_library(searchlib_tensor OBJECT direct_tensor_saver.cpp distance_function_factory.cpp distance_functions.cpp - generic_tensor_attribute.cpp - generic_tensor_attribute_saver.cpp - generic_tensor_store.cpp hnsw_graph.cpp hnsw_index.cpp hnsw_index_loader.cpp @@ -22,6 +19,9 @@ vespa_add_library(searchlib_tensor OBJECT inv_log_level_generator.cpp nearest_neighbor_index.cpp nearest_neighbor_index_saver.cpp + serialized_tensor_attribute.cpp + serialized_tensor_attribute_saver.cpp + serialized_tensor_store.cpp tensor_attribute.cpp tensor_deserialize.cpp tensor_store.cpp diff --git a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.h b/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.h deleted file mode 100644 index 92beef49136..00000000000 --- a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.h +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/searchlib/attribute/attributesaver.h> -#include "tensor_attribute.h" - -namespace search { - -namespace tensor { - -class GenericTensorStore; - -/* - * Class for saving a tensor attribute. - */ -class GenericTensorAttributeSaver : public AttributeSaver -{ -public: - using RefCopyVector = TensorAttribute::RefCopyVector; -private: - RefCopyVector _refs; - const GenericTensorStore &_tensorStore; - using GenerationHandler = vespalib::GenerationHandler; - - virtual bool onSave(IAttributeSaveTarget &saveTarget) override; -public: - GenericTensorAttributeSaver(GenerationHandler::Guard &&guard, - const attribute::AttributeHeader &header, - RefCopyVector &&refs, - const GenericTensorStore &tensorStore); - - virtual ~GenericTensorAttributeSaver(); -}; - -} // namespace search::tensor - -} // namespace search diff --git a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_attribute.cpp index 6864fb52120..d4a20abf2fd 100644 --- a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_attribute.cpp @@ -1,9 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "generic_tensor_attribute.h" -#include "generic_tensor_attribute_saver.h" -#include "tensor_attribute.hpp" #include "blob_sequence_reader.h" +#include "serialized_tensor_attribute.h" +#include "serialized_tensor_attribute_saver.h" +#include "tensor_attribute.hpp" #include <vespa/eval/tensor/tensor.h> #include <vespa/fastlib/io/bufferedfile.h> #include <vespa/searchlib/attribute/readerbase.h> @@ -21,29 +21,29 @@ constexpr uint32_t TENSOR_ATTRIBUTE_VERSION = 0; } -GenericTensorAttribute::GenericTensorAttribute(stringref name, const Config &cfg) - : TensorAttribute(name, cfg, _genericTensorStore) +SerializedTensorAttribute::SerializedTensorAttribute(stringref name, const Config &cfg) + : TensorAttribute(name, cfg, _serializedTensorStore) { } -GenericTensorAttribute::~GenericTensorAttribute() +SerializedTensorAttribute::~SerializedTensorAttribute() { getGenerationHolder().clearHoldLists(); _tensorStore.clearHoldLists(); } void -GenericTensorAttribute::setTensor(DocId docId, const Tensor &tensor) +SerializedTensorAttribute::setTensor(DocId docId, const Tensor &tensor) { checkTensorType(tensor); - EntryRef ref = _genericTensorStore.setTensor(tensor); + EntryRef ref = _serializedTensorStore.setTensor(tensor); setTensorRef(docId, ref); } std::unique_ptr<Tensor> -GenericTensorAttribute::getTensor(DocId docId) const +SerializedTensorAttribute::getTensor(DocId docId) const { EntryRef ref; if (docId < getCommittedDocIdLimit()) { @@ -52,17 +52,17 @@ GenericTensorAttribute::getTensor(DocId docId) const if (!ref.valid()) { return std::unique_ptr<Tensor>(); } - return _genericTensorStore.getTensor(ref); + return _serializedTensorStore.getTensor(ref); } void -GenericTensorAttribute::getTensor(DocId, vespalib::tensor::MutableDenseTensorView &) const +SerializedTensorAttribute::getTensor(DocId, vespalib::tensor::MutableDenseTensorView &) const { notImplemented(); } bool -GenericTensorAttribute::onLoad() +SerializedTensorAttribute::onLoad() { BlobSequenceReader tensorReader(*this); if (!tensorReader.hasData()) { @@ -75,7 +75,7 @@ GenericTensorAttribute::onLoad() _refVector.unsafe_reserve(numDocs); for (uint32_t lid = 0; lid < numDocs; ++lid) { uint32_t tensorSize = tensorReader.getNextSize(); - auto raw = _genericTensorStore.allocRawBuffer(tensorSize); + auto raw = _serializedTensorStore.allocRawBuffer(tensorSize); if (tensorSize != 0) { tensorReader.readBlob(raw.data, tensorSize); } @@ -88,21 +88,21 @@ GenericTensorAttribute::onLoad() std::unique_ptr<AttributeSaver> -GenericTensorAttribute::onInitSave(vespalib::stringref fileName) +SerializedTensorAttribute::onInitSave(vespalib::stringref fileName) { vespalib::GenerationHandler::Guard guard(getGenerationHandler(). takeGuard()); - return std::make_unique<GenericTensorAttributeSaver> + return std::make_unique<SerializedTensorAttributeSaver> (std::move(guard), this->createAttributeHeader(fileName), getRefCopy(), - _genericTensorStore); + _serializedTensorStore); } void -GenericTensorAttribute::compactWorst() +SerializedTensorAttribute::compactWorst() { - doCompactWorst<GenericTensorStore::RefType>(); + doCompactWorst<SerializedTensorStore::RefType>(); } } diff --git a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_attribute.h index 9dd3788511e..5596341a5b7 100644 --- a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_attribute.h @@ -2,22 +2,19 @@ #pragma once +#include "serialized_tensor_store.h" #include "tensor_attribute.h" -#include "generic_tensor_store.h" -namespace search { - -namespace tensor { +namespace search::tensor { /** * Attribute vector class used to store tensors for all documents in memory. */ -class GenericTensorAttribute : public TensorAttribute -{ - GenericTensorStore _genericTensorStore; // data store for serialized tensors +class SerializedTensorAttribute : public TensorAttribute { + SerializedTensorStore _serializedTensorStore; // data store for serialized tensors public: - GenericTensorAttribute(vespalib::stringref baseFileName, const Config &cfg); - virtual ~GenericTensorAttribute(); + SerializedTensorAttribute(vespalib::stringref baseFileName, const Config &cfg); + virtual ~SerializedTensorAttribute(); virtual void setTensor(DocId docId, const Tensor &tensor) override; virtual std::unique_ptr<Tensor> getTensor(DocId docId) const override; virtual void getTensor(DocId docId, vespalib::tensor::MutableDenseTensorView &tensor) const override; @@ -26,7 +23,4 @@ public: virtual void compactWorst() override; }; - -} // namespace search::tensor - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_attribute_saver.cpp index 81ec3a5218e..4c41c3a449e 100644 --- a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp +++ b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_attribute_saver.cpp @@ -1,21 +1,19 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "generic_tensor_attribute_saver.h" -#include "generic_tensor_store.h" +#include "serialized_tensor_attribute_saver.h" +#include "serialized_tensor_store.h" #include <vespa/searchlib/util/bufferwriter.h> #include <vespa/searchlib/attribute/iattributesavetarget.h> using vespalib::GenerationHandler; -namespace search { +namespace search::tensor { -namespace tensor { - -GenericTensorAttributeSaver:: -GenericTensorAttributeSaver(GenerationHandler::Guard &&guard, - const attribute::AttributeHeader &header, - RefCopyVector &&refs, - const GenericTensorStore &tensorStore) +SerializedTensorAttributeSaver:: +SerializedTensorAttributeSaver(GenerationHandler::Guard &&guard, + const attribute::AttributeHeader &header, + RefCopyVector &&refs, + const SerializedTensorStore &tensorStore) : AttributeSaver(std::move(guard), header), _refs(std::move(refs)), _tensorStore(tensorStore) @@ -23,13 +21,13 @@ GenericTensorAttributeSaver(GenerationHandler::Guard &&guard, } -GenericTensorAttributeSaver::~GenericTensorAttributeSaver() +SerializedTensorAttributeSaver::~SerializedTensorAttributeSaver() { } bool -GenericTensorAttributeSaver::onSave(IAttributeSaveTarget &saveTarget) +SerializedTensorAttributeSaver::onSave(IAttributeSaveTarget &saveTarget) { std::unique_ptr<BufferWriter> datWriter(saveTarget.datWriter().allocBufferWriter()); @@ -45,7 +43,4 @@ GenericTensorAttributeSaver::onSave(IAttributeSaveTarget &saveTarget) return true; } - -} // namespace search::tensor - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_tensor_attribute_saver.h b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_attribute_saver.h new file mode 100644 index 00000000000..1ae2279b893 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_attribute_saver.h @@ -0,0 +1,33 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/attribute/attributesaver.h> +#include "tensor_attribute.h" + +namespace search::tensor { + +class SerializedTensorStore; + +/* + * Class for saving a tensor attribute. + */ +class SerializedTensorAttributeSaver : public AttributeSaver { +public: + using RefCopyVector = TensorAttribute::RefCopyVector; +private: + RefCopyVector _refs; + const SerializedTensorStore &_tensorStore; + using GenerationHandler = vespalib::GenerationHandler; + + virtual bool onSave(IAttributeSaveTarget &saveTarget) override; +public: + SerializedTensorAttributeSaver(GenerationHandler::Guard &&guard, + const attribute::AttributeHeader &header, + RefCopyVector &&refs, + const SerializedTensorStore &tensorStore); + + virtual ~SerializedTensorAttributeSaver(); +}; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/generic_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_store.cpp index 8c695c32719..77903291e13 100644 --- a/searchlib/src/vespa/searchlib/tensor/generic_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_store.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "generic_tensor_store.h" +#include "serialized_tensor_store.h" #include "tensor_deserialize.h" #include <vespa/eval/tensor/tensor.h> #include <vespa/eval/tensor/serialization/typed_binary_format.h> @@ -17,7 +17,7 @@ namespace search::tensor { constexpr size_t MIN_BUFFER_ARRAYS = 1024; -GenericTensorStore::GenericTensorStore() +SerializedTensorStore::SerializedTensorStore() : TensorStore(_concreteStore), _concreteStore(), _bufferType(RefType::align(1), @@ -28,13 +28,13 @@ GenericTensorStore::GenericTensorStore() _store.initActiveBuffers(); } -GenericTensorStore::~GenericTensorStore() +SerializedTensorStore::~SerializedTensorStore() { _store.dropBuffers(); } std::pair<const void *, uint32_t> -GenericTensorStore::getRawBuffer(RefType ref) const +SerializedTensorStore::getRawBuffer(RefType ref) const { if (!ref.valid()) { return std::make_pair(nullptr, 0u); @@ -45,7 +45,7 @@ GenericTensorStore::getRawBuffer(RefType ref) const } Handle<char> -GenericTensorStore::allocRawBuffer(uint32_t size) +SerializedTensorStore::allocRawBuffer(uint32_t size) { if (size == 0) { return Handle<char>(); @@ -63,7 +63,7 @@ GenericTensorStore::allocRawBuffer(uint32_t size) } void -GenericTensorStore::holdTensor(EntryRef ref) +SerializedTensorStore::holdTensor(EntryRef ref) { if (!ref.valid()) { return; @@ -75,7 +75,7 @@ GenericTensorStore::holdTensor(EntryRef ref) } TensorStore::EntryRef -GenericTensorStore::move(EntryRef ref) +SerializedTensorStore::move(EntryRef ref) { if (!ref.valid()) { return RefType(); @@ -88,7 +88,7 @@ GenericTensorStore::move(EntryRef ref) } std::unique_ptr<Tensor> -GenericTensorStore::getTensor(EntryRef ref) const +SerializedTensorStore::getTensor(EntryRef ref) const { auto raw = getRawBuffer(ref); if (raw.second == 0u) { @@ -98,7 +98,7 @@ GenericTensorStore::getTensor(EntryRef ref) const } TensorStore::EntryRef -GenericTensorStore::setTensor(const Tensor &tensor) +SerializedTensorStore::setTensor(const Tensor &tensor) { vespalib::nbostream stream; TypedBinaryFormat::serialize(stream, tensor); diff --git a/searchlib/src/vespa/searchlib/tensor/generic_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_store.h index 14b65e1ec5a..7c0a8e5ed16 100644 --- a/searchlib/src/vespa/searchlib/tensor/generic_tensor_store.h +++ b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_store.h @@ -4,9 +4,7 @@ #include "tensor_store.h" -namespace search { - -namespace tensor { +namespace search::tensor { /** * Class for storing serialized tensors in memory, used by TensorAttribute. @@ -15,8 +13,7 @@ namespace tensor { * might also require corresponding changes to implemented optimized tensor * operations that use the serialized tensor as argument. */ -class GenericTensorStore : public TensorStore -{ +class SerializedTensorStore : public TensorStore { public: using RefType = vespalib::datastore::AlignedEntryRefT<22, 2>; using DataStoreType = vespalib::datastore::DataStoreT<RefType>; @@ -24,9 +21,9 @@ private: DataStoreType _concreteStore; vespalib::datastore::BufferType<char> _bufferType; public: - GenericTensorStore(); + SerializedTensorStore(); - virtual ~GenericTensorStore(); + virtual ~SerializedTensorStore(); std::pair<const void *, uint32_t> getRawBuffer(RefType ref) const; @@ -41,7 +38,4 @@ public: EntryRef setTensor(const Tensor &tensor); }; - -} // namespace search::tensor - -} // namespace search +} |