From f284ec21d313045e031b6a58c1943acea269865c Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Fri, 26 Mar 2021 13:04:45 +0000 Subject: Remove experimental use of mmap file allocator for dense tensor data. --- .../tensorattribute/tensorattribute_test.cpp | 23 ---------------------- .../dense_tensor_store/dense_tensor_store_test.cpp | 8 ++++---- .../searchlib/tensor/dense_tensor_attribute.cpp | 13 +----------- .../vespa/searchlib/tensor/dense_tensor_store.cpp | 15 ++++---------- .../vespa/searchlib/tensor/dense_tensor_store.h | 6 ++---- 5 files changed, 11 insertions(+), 54 deletions(-) (limited to 'searchlib') diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 8efccf9b51b..f1d910f2635 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -26,7 +26,6 @@ #include #include #include -#include #include #include @@ -257,7 +256,6 @@ struct FixtureTraits { bool use_direct_tensor_attribute = false; bool enable_hnsw_index = false; bool use_mock_index = false; - bool use_mmap_file_allocator = false; FixtureTraits dense() && { use_dense_tensor_attribute = true; @@ -265,11 +263,6 @@ struct FixtureTraits { return *this; } - FixtureTraits mmap_file_allocator() && { - use_mmap_file_allocator = true; - return *this; - } - FixtureTraits hnsw() && { use_dense_tensor_attribute = true; enable_hnsw_index = true; @@ -334,9 +327,6 @@ struct Fixture { if (_cfg.tensorType().is_dense()) { _denseTensors = true; } - if (_traits.use_mmap_file_allocator) { - _cfg.setHuge(true); - } if (_traits.use_mock_index) { _index_factory = std::make_unique(); } else { @@ -1008,17 +998,4 @@ TEST_F("NN blueprint handles strong filter triggering brute force search", Neare EXPECT_FALSE(bp->may_approximate()); } -TEST("Dense tensor attribute with huge flag uses mmap file allocator") -{ - vespalib::string basedir("mmap-file-allocator-factory-dir"); - vespalib::alloc::MmapFileAllocatorFactory::instance().setup(basedir); - { - Fixture f(vec_2d_spec, FixtureTraits().dense().mmap_file_allocator()); - vespalib::string allocator_dir(basedir + "/0.my_attr"); - EXPECT_TRUE(vespalib::isDirectory(allocator_dir)); - } - vespalib::alloc::MmapFileAllocatorFactory::instance().setup(""); - vespalib::rmdir(basedir, true); -} - TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp b/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp index 75e231a815c..eb8d89edc07 100644 --- a/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp +++ b/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp @@ -1,8 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include -LOG_SETUP("dense_tensor_store_test"); #include -#include #include #include #include @@ -10,6 +7,9 @@ LOG_SETUP("dense_tensor_store_test"); #include #include +#include +LOG_SETUP("dense_tensor_store_test"); + using search::tensor::DenseTensorStore; using vespalib::eval::SimpleValue; using vespalib::eval::TensorSpec; @@ -28,7 +28,7 @@ struct Fixture { DenseTensorStore store; Fixture(const vespalib::string &tensorType) - : store(ValueType::from_spec(tensorType), {}) + : store(ValueType::from_spec(tensorType)) {} void assertSetAndGetTensor(const TensorSpec &tensorSpec) { Value::UP expTensor = makeTensor(tensorSpec); diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index 0f39816257f..be127a8be3e 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -10,8 +10,6 @@ #include #include #include -#include -#include #include LOG_SETUP(".searchlib.tensor.dense_tensor_attribute"); @@ -74,15 +72,6 @@ can_use_index_save_file(const search::attribute::Config &config, const search::a return true; } -std::unique_ptr -make_memory_allocator(const vespalib::string& name, bool huge) -{ - if (huge) { - return vespalib::alloc::MmapFileAllocatorFactory::instance().make_memory_allocator(name); - } - return {}; -} - } void @@ -115,7 +104,7 @@ DenseTensorAttribute::memory_usage() const DenseTensorAttribute::DenseTensorAttribute(vespalib::stringref baseFileName, const Config& cfg, const NearestNeighborIndexFactory& index_factory) : TensorAttribute(baseFileName, cfg, _denseTensorStore), - _denseTensorStore(cfg.tensorType(), make_memory_allocator(getName(), cfg.huge())), + _denseTensorStore(cfg.tensorType()), _index() { if (cfg.hnsw_index_params().has_value()) { diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp index 13796d35dec..e6da839da2e 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp @@ -40,9 +40,8 @@ DenseTensorStore::TensorSizeCalc::alignedSize() const return my_align(bufSize(), DENSE_TENSOR_ALIGNMENT); } -DenseTensorStore::BufferType::BufferType(const TensorSizeCalc &tensorSizeCalc, std::unique_ptr allocator) - : vespalib::datastore::BufferType(tensorSizeCalc.alignedSize(), MIN_BUFFER_ARRAYS, RefType::offsetSize()), - _allocator(std::move(allocator)) +DenseTensorStore::BufferType::BufferType(const TensorSizeCalc &tensorSizeCalc) + : vespalib::datastore::BufferType(tensorSizeCalc.alignedSize(), MIN_BUFFER_ARRAYS, RefType::offsetSize()) {} DenseTensorStore::BufferType::~BufferType() = default; @@ -54,17 +53,11 @@ DenseTensorStore::BufferType::cleanHold(void *buffer, size_t offset, memset(static_cast(buffer) + offset, 0, numElems); } -const vespalib::alloc::MemoryAllocator* -DenseTensorStore::BufferType::get_memory_allocator() const -{ - return _allocator.get(); -} - -DenseTensorStore::DenseTensorStore(const ValueType &type, std::unique_ptr allocator) +DenseTensorStore::DenseTensorStore(const ValueType &type) : TensorStore(_concreteStore), _concreteStore(), _tensorSizeCalc(type), - _bufferType(_tensorSizeCalc, std::move(allocator)), + _bufferType(_tensorSizeCalc), _type(type), _emptySpace() { diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h index dad28642e67..638f602d613 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h @@ -36,12 +36,10 @@ public: class BufferType : public vespalib::datastore::BufferType { using CleanContext = vespalib::datastore::BufferType::CleanContext; - std::unique_ptr _allocator; public: - BufferType(const TensorSizeCalc &tensorSizeCalc, std::unique_ptr allocator); + BufferType(const TensorSizeCalc &tensorSizeCalc); ~BufferType() override; void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; - const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; }; private: DataStoreType _concreteStore; @@ -57,7 +55,7 @@ private: setDenseTensor(const TensorType &tensor); public: - DenseTensorStore(const ValueType &type, std::unique_ptr allocator); + DenseTensorStore(const ValueType &type); ~DenseTensorStore() override; const ValueType &type() const { return _type; } -- cgit v1.2.3