diff options
Diffstat (limited to 'searchlib/src/tests/tensor')
3 files changed, 79 insertions, 10 deletions
diff --git a/searchlib/src/tests/tensor/tensor_buffer_operations/tensor_buffer_operations_test.cpp b/searchlib/src/tests/tensor/tensor_buffer_operations/tensor_buffer_operations_test.cpp index 9bed7a11bee..321e174862f 100644 --- a/searchlib/src/tests/tensor/tensor_buffer_operations/tensor_buffer_operations_test.cpp +++ b/searchlib/src/tests/tensor/tensor_buffer_operations/tensor_buffer_operations_test.cpp @@ -78,7 +78,7 @@ TensorBufferOperationsTest::get_array_sizes(uint32_t max_subspaces) { std::vector<size_t> array_sizes; for (uint32_t num_subspaces = 0; num_subspaces < max_subspaces; ++num_subspaces) { - array_sizes.emplace_back(_ops.get_array_size(num_subspaces)); + array_sizes.emplace_back(_ops.get_buffer_size(num_subspaces)); } return array_sizes; } @@ -88,7 +88,7 @@ TensorBufferOperationsTest::store_tensor(const Value& tensor) { EXPECT_EQ(_tensor_type, tensor.type()); uint32_t num_subspaces = tensor.index().size(); - auto array_size = _ops.get_array_size(num_subspaces); + auto array_size = _ops.get_buffer_size(num_subspaces); std::vector<char> buf; buf.resize(array_size); _ops.store_tensor(buf, tensor); diff --git a/searchlib/src/tests/tensor/tensor_buffer_store/tensor_buffer_store_test.cpp b/searchlib/src/tests/tensor/tensor_buffer_store/tensor_buffer_store_test.cpp index ec7fc2334c4..0a69d8149ed 100644 --- a/searchlib/src/tests/tensor/tensor_buffer_store/tensor_buffer_store_test.cpp +++ b/searchlib/src/tests/tensor/tensor_buffer_store/tensor_buffer_store_test.cpp @@ -6,6 +6,8 @@ #include <vespa/eval/eval/value.h> #include <vespa/eval/eval/value_codec.h> #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/stllike/hash_set.h> using search::tensor::TensorBufferStore; using vespalib::datastore::EntryRef; @@ -177,4 +179,33 @@ TEST_F(TensorBufferStoreTest, get_vectors) EXPECT_EQ(0, _store.get_vectors(EntryRef()).subspaces()); } +TEST_F(TensorBufferStoreTest, buffer_handles_range_of_subspaces) +{ + auto offset_bits = TensorBufferStore::get_offset_bits(); + TensorBufferStore store(_tensor_type, {}, 400); + std::vector<EntryRef> refs; + vespalib::hash_set<uint32_t> buffers; + TensorSpec tensor_spec(tensor_type_spec); + for (uint32_t x = 0; x < 400; ++x) { + vespalib::asciistream x_stream; + x_stream << x; + vespalib::string x_as_string = x_stream.str(); + tensor_spec.add({{"x", x_as_string.c_str()}}, x + 1.5); + auto tensor = value_from_spec(tensor_spec, FastValueBuilderFactory::get()); + EXPECT_EQ(_tensor_type, tensor->type()); + auto ref = store.store_tensor(*tensor); + refs.emplace_back(ref); + auto buffer_id = ref.buffer_id(offset_bits); + buffers.insert(buffer_id); + } + EXPECT_EQ(156u, buffers.size()); + uint32_t x = 0; + for (auto ref : refs) { + auto tensor = store.get_tensor(ref); + ++x; + EXPECT_EQ(x, tensor->index().size()); + EXPECT_EQ(x, store.get_vectors(ref).subspaces()); + } +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/tensor/tensor_buffer_type_mapper/tensor_buffer_type_mapper_test.cpp b/searchlib/src/tests/tensor/tensor_buffer_type_mapper/tensor_buffer_type_mapper_test.cpp index d7eaa6b633d..17612f08271 100644 --- a/searchlib/src/tests/tensor/tensor_buffer_type_mapper/tensor_buffer_type_mapper_test.cpp +++ b/searchlib/src/tests/tensor/tensor_buffer_type_mapper/tensor_buffer_type_mapper_test.cpp @@ -15,14 +15,18 @@ const vespalib::string tensor_type_2d_mixed_spec("tensor(x{},y[2])"); const vespalib::string float_tensor_type_spec("tensor<float>(y{})"); const vespalib::string tensor_type_dense_spec("tensor(x[2])"); +constexpr double grow_factor = 1.02; + struct TestParam { vespalib::string _name; std::vector<size_t> _array_sizes; + std::vector<size_t> _large_array_sizes; vespalib::string _tensor_type_spec; - TestParam(vespalib::string name, std::vector<size_t> array_sizes, const vespalib::string& tensor_type_spec) + TestParam(vespalib::string name, std::vector<size_t> array_sizes, std::vector<size_t> large_array_sizes, const vespalib::string& tensor_type_spec) : _name(std::move(name)), _array_sizes(std::move(array_sizes)), + _large_array_sizes(std::move(large_array_sizes)), _tensor_type_spec(tensor_type_spec) { } @@ -49,6 +53,7 @@ protected: TensorBufferTypeMapperTest(); ~TensorBufferTypeMapperTest() override; std::vector<size_t> get_array_sizes(); + std::vector<size_t> get_large_array_sizes(); void select_type_ids(); }; @@ -56,7 +61,7 @@ TensorBufferTypeMapperTest::TensorBufferTypeMapperTest() : testing::TestWithParam<TestParam>(), _tensor_type(ValueType::from_spec(GetParam()._tensor_type_spec)), _ops(_tensor_type), - _mapper(GetParam()._array_sizes.size(), &_ops) + _mapper(GetParam()._array_sizes.size(), grow_factor, &_ops) { } @@ -70,11 +75,33 @@ TensorBufferTypeMapperTest::get_array_sizes() for (uint32_t type_id = 1; type_id <= max_small_subspaces_type_id; ++type_id) { auto num_subspaces = type_id - 1; array_sizes.emplace_back(_mapper.get_array_size(type_id)); - EXPECT_EQ(_ops.get_array_size(num_subspaces), array_sizes.back()); + EXPECT_EQ(_ops.get_buffer_size(num_subspaces), array_sizes.back()); } return array_sizes; } +std::vector<size_t> +TensorBufferTypeMapperTest::get_large_array_sizes() +{ + auto& large_array_sizes = GetParam()._large_array_sizes; + uint32_t max_large = large_array_sizes.size(); + TensorBufferTypeMapper mapper(max_large * 100, grow_factor, &_ops); + std::vector<size_t> result; + for (uint32_t i = 0; i < max_large; ++i) { + uint32_t type_id = (i + 1) * 100; + auto array_size = mapper.get_array_size(type_id); + result.emplace_back(array_size); + EXPECT_EQ(type_id, mapper.get_type_id(array_size)); + EXPECT_EQ(type_id, mapper.get_type_id(array_size - 1)); + if (array_size == large_array_sizes.back()) { + EXPECT_EQ(0u, mapper.get_type_id(array_size + 1)); + } else { + EXPECT_EQ(type_id + 1, mapper.get_type_id(array_size + 1)); + } + } + return result; +} + void TensorBufferTypeMapperTest::select_type_ids() { @@ -101,11 +128,11 @@ TensorBufferTypeMapperTest::select_type_ids() INSTANTIATE_TEST_SUITE_P(TensorBufferTypeMapperMultiTest, TensorBufferTypeMapperTest, - testing::Values(TestParam("1d", {8, 16, 32, 40, 64}, tensor_type_sparse_spec), - TestParam("1dfloat", {4, 12, 20, 28, 36}, float_tensor_type_spec), - TestParam("2d", {8, 24, 40, 56, 80}, tensor_type_2d_spec), - TestParam("2dmixed", {8, 24, 48, 64, 96}, tensor_type_2d_mixed_spec), - TestParam("dense", {8, 24}, tensor_type_dense_spec)), + testing::Values(TestParam("1d", {8, 16, 32, 40, 64}, {1760, 10880, 76896, 555248, 4020512}, tensor_type_sparse_spec), + TestParam("1dfloat", {4, 12, 20, 28, 36}, {1728, 11104, 79168, 572128, 4143664}, float_tensor_type_spec), + TestParam("2d", {8, 24, 40, 56, 80}, {1600, 9184, 63872, 460416, 3332976}, tensor_type_2d_spec), + TestParam("2dmixed", {8, 24, 48, 64, 96}, {1984, 11472, 79824, 575504, 4166208}, tensor_type_2d_mixed_spec), + TestParam("dense", {8, 24}, {}, tensor_type_dense_spec)), testing::PrintToStringParamName()); TEST_P(TensorBufferTypeMapperTest, array_sizes_are_calculated) @@ -118,4 +145,15 @@ TEST_P(TensorBufferTypeMapperTest, type_ids_are_selected) select_type_ids(); } +TEST_P(TensorBufferTypeMapperTest, large_arrays_grows_exponentially) +{ + EXPECT_EQ(GetParam()._large_array_sizes, get_large_array_sizes()); +} + +TEST_P(TensorBufferTypeMapperTest, avoid_array_size_overflow) +{ + TensorBufferTypeMapper mapper(400, 2.0, &_ops); + EXPECT_GE(30, mapper.get_max_small_array_type_id(1000)); +} + GTEST_MAIN_RUN_ALL_TESTS() |