diff options
author | Håvard Pettersen <havardpe@oath.com> | 2019-06-06 10:25:54 +0000 |
---|---|---|
committer | Håvard Pettersen <havardpe@oath.com> | 2019-06-06 10:40:49 +0000 |
commit | 3f438ca14463cfeef12b2965be8feb5b4f8d8295 (patch) | |
tree | 874710484f9f79f26a46047e5651d96fd2acc370 /eval | |
parent | 38bc4ec9556352af0117d8fa9c41c6446e95fb95 (diff) |
remove sparse tensor builder
also remove sparse tensor unsorted address builder
Diffstat (limited to 'eval')
10 files changed, 0 insertions, 401 deletions
diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 6a5e354a0c3..26e046ec52a 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -36,7 +36,6 @@ vespa_define_module( src/tests/tensor/dense_xw_product_function src/tests/tensor/direct_dense_tensor_builder src/tests/tensor/direct_sparse_tensor_builder - src/tests/tensor/sparse_tensor_builder src/tests/tensor/tensor_add_operation src/tests/tensor/tensor_address src/tests/tensor/tensor_conformance diff --git a/eval/src/tests/tensor/sparse_tensor_builder/.gitignore b/eval/src/tests/tensor/sparse_tensor_builder/.gitignore deleted file mode 100644 index e0316d190bb..00000000000 --- a/eval/src/tests/tensor/sparse_tensor_builder/.gitignore +++ /dev/null @@ -1 +0,0 @@ -vespalib_sparse_tensor_builder_test_app diff --git a/eval/src/tests/tensor/sparse_tensor_builder/CMakeLists.txt b/eval/src/tests/tensor/sparse_tensor_builder/CMakeLists.txt deleted file mode 100644 index 2132a37f9fa..00000000000 --- a/eval/src/tests/tensor/sparse_tensor_builder/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(eval_sparse_tensor_builder_test_app TEST - SOURCES - sparse_tensor_builder_test.cpp - DEPENDS - vespaeval -) -vespa_add_test(NAME eval_sparse_tensor_builder_test_app COMMAND eval_sparse_tensor_builder_test_app) diff --git a/eval/src/tests/tensor/sparse_tensor_builder/sparse_tensor_builder_test.cpp b/eval/src/tests/tensor/sparse_tensor_builder/sparse_tensor_builder_test.cpp deleted file mode 100644 index 708c2f761f7..00000000000 --- a/eval/src/tests/tensor/sparse_tensor_builder/sparse_tensor_builder_test.cpp +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/eval/tensor/sparse/sparse_tensor_builder.h> -#include <vespa/eval/tensor/sparse/sparse_tensor_address_combiner.h> -#include <vespa/vespalib/test/insertion_operators.h> - -using namespace vespalib::tensor; -using namespace vespalib::tensor::sparse; -using vespalib::eval::TensorSpec; -using vespalib::eval::ValueType; - -void -assertCellValue(double expValue, const TensorAddress &address, - const ValueType &type, - const SparseTensor::Cells &cells) -{ - SparseTensorAddressBuilder addressBuilder; - auto dimsItr = type.dimensions().cbegin(); - auto dimsItrEnd = type.dimensions().cend(); - for (const auto &element : address.elements()) { - while ((dimsItr < dimsItrEnd) && (dimsItr->name < element.dimension())) { - addressBuilder.add(""); - ++dimsItr; - } - assert((dimsItr != dimsItrEnd) && (dimsItr->name == element.dimension())); - addressBuilder.add(element.label()); - ++dimsItr; - } - while (dimsItr < dimsItrEnd) { - addressBuilder.add(""); - ++dimsItr; - } - SparseTensorAddressRef addressRef(addressBuilder.getAddressRef()); - auto itr = cells.find(addressRef); - EXPECT_FALSE(itr == cells.end()); - EXPECT_EQUAL(expValue, itr->second); -} - -Tensor::UP -buildTensor() -{ - SparseTensorBuilder builder; - builder.define_dimension("c"); - builder.define_dimension("d"); - builder.define_dimension("a"); - builder.define_dimension("b"); - builder.add_label(builder.define_dimension("a"), "1"). - add_label(builder.define_dimension("b"), "2").add_cell(10). - add_label(builder.define_dimension("c"), "3"). - add_label(builder.define_dimension("d"), "4").add_cell(20); - return builder.build(); -} - -TEST("require that tensor can be constructed") -{ - Tensor::UP tensor = buildTensor(); - const SparseTensor &sparseTensor = dynamic_cast<const SparseTensor &>(*tensor); - const ValueType &type = sparseTensor.type(); - const SparseTensor::Cells &cells = sparseTensor.cells(); - EXPECT_EQUAL(2u, cells.size()); - assertCellValue(10, TensorAddress({{"a","1"},{"b","2"}}), type, cells); - assertCellValue(20, TensorAddress({{"c","3"},{"d","4"}}), type, cells); -} - -TEST("require that tensor can be converted to tensor spec") -{ - Tensor::UP tensor = buildTensor(); - TensorSpec expSpec("tensor(a{},b{},c{},d{})"); - expSpec.add({{"a", "1"}, {"b", "2"}, {"c", ""}, {"d", ""}}, 10). - add({{"a", ""},{"b",""},{"c", "3"}, {"d", "4"}}, 20); - TensorSpec actSpec = tensor->toSpec(); - EXPECT_EQUAL(expSpec, actSpec); -} - -TEST("require that dimensions are extracted") -{ - SparseTensorBuilder builder; - builder.define_dimension("c"); - builder.define_dimension("a"); - builder.define_dimension("b"); - builder. - add_label(builder.define_dimension("a"), "1"). - add_label(builder.define_dimension("b"), "2").add_cell(10). - add_label(builder.define_dimension("b"), "3"). - add_label(builder.define_dimension("c"), "4").add_cell(20); - Tensor::UP tensor = builder.build(); - const SparseTensor &sparseTensor = dynamic_cast<const SparseTensor &>(*tensor); - const auto &dims = sparseTensor.type().dimensions(); - EXPECT_EQUAL(3u, dims.size()); - EXPECT_EQUAL("a", dims[0].name); - EXPECT_EQUAL("b", dims[1].name); - EXPECT_EQUAL("c", dims[2].name); - EXPECT_EQUAL("tensor(a{},b{},c{})", sparseTensor.type().to_spec()); -} - -void verifyAddressCombiner(const ValueType & a, const ValueType & b, size_t numDim, size_t numOverlapping) { - TensorAddressCombiner combiner(a, b); - EXPECT_EQUAL(numDim, combiner.numDimensions()); - EXPECT_EQUAL(numOverlapping, combiner.numOverlappingDimensions()); -} -TEST("Test sparse tensor address combiner") { - verifyAddressCombiner(ValueType::tensor_type({{"a"}}), ValueType::tensor_type({{"b"}}), 2, 0); - verifyAddressCombiner(ValueType::tensor_type({{"a"}, {"b"}}), ValueType::tensor_type({{"b"}}), 2, 1); - verifyAddressCombiner(ValueType::tensor_type({{"a"}, {"b"}}), ValueType::tensor_type({{"b"}, {"c"}}), 3, 1); - -} - -TEST("Test essential object sizes") { - EXPECT_EQUAL(16u, sizeof(SparseTensorAddressRef)); - EXPECT_EQUAL(24u, sizeof(std::pair<SparseTensorAddressRef, double>)); - EXPECT_EQUAL(32u, sizeof(vespalib::hash_node<std::pair<SparseTensorAddressRef, double>>)); -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp index 2c113e814e9..b43a127bc60 100644 --- a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp +++ b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp @@ -2,7 +2,6 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/tensor/sparse/sparse_tensor.h> -#include <vespa/eval/tensor/sparse/sparse_tensor_builder.h> #include <vespa/eval/tensor/types.h> #include <vespa/eval/tensor/default_tensor_engine.h> #include <vespa/eval/tensor/serialization/typed_binary_format.h> diff --git a/eval/src/vespa/eval/tensor/sparse/CMakeLists.txt b/eval/src/vespa/eval/tensor/sparse/CMakeLists.txt index 2d142d98ba1..9304d890e34 100644 --- a/eval/src/vespa/eval/tensor/sparse/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/sparse/CMakeLists.txt @@ -8,9 +8,7 @@ vespa_add_library(eval_tensor_sparse OBJECT sparse_tensor_address_padder.cpp sparse_tensor_address_reducer.cpp sparse_tensor_address_ref.cpp - sparse_tensor_builder.cpp sparse_tensor_match.cpp sparse_tensor_modify.cpp sparse_tensor_remove.cpp - sparse_tensor_unsorted_address_builder.cpp ) diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_builder.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_builder.cpp deleted file mode 100644 index 87e5b4c1941..00000000000 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_builder.cpp +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "sparse_tensor_builder.h" -#include <cassert> - -namespace vespalib::tensor { - -SparseTensorBuilder::SparseTensorBuilder() - : _addressBuilder(), - _normalizedAddressBuilder(), - _cells(), - _stash(SparseTensor::STASH_CHUNK_SIZE), - _dimensionsEnum(), - _dimensions(), - _type(eval::ValueType::double_type()), - _type_made(false) -{ -} - -SparseTensorBuilder::~SparseTensorBuilder() = default; - -void -SparseTensorBuilder::makeType() -{ - assert(!_type_made); - assert(_cells.empty()); - std::vector<eval::ValueType::Dimension> dimensions; - dimensions.reserve(_dimensions.size()); - for (const auto &dim : _dimensions) { - dimensions.emplace_back(dim); - } - _type = (dimensions.empty() ? - eval::ValueType::double_type() : - eval::ValueType::tensor_type(std::move(dimensions))); - _type_made = true; -} - - -SparseTensorBuilder::Dimension -SparseTensorBuilder::define_dimension(const vespalib::string &dimension) -{ - auto it = _dimensionsEnum.find(dimension); - if (it != _dimensionsEnum.end()) { - return it->second; - } - assert(!_type_made); - Dimension res = _dimensionsEnum.size(); - auto insres = _dimensionsEnum.insert(std::make_pair(dimension, res)); - (void) insres; - assert(insres.second); - assert(insres.first->second == res); - assert(_dimensions.size() == res); - _dimensions.push_back(dimension); - return res; -} - -SparseTensorBuilder & -SparseTensorBuilder::add_label(Dimension dimension, - const vespalib::string &label) -{ - assert(dimension <= _dimensions.size()); - _addressBuilder.add(_dimensions[dimension], label); - return *this; -} - -SparseTensorBuilder & -SparseTensorBuilder::add_cell(double value) -{ - if (!_type_made) { - makeType(); - } - _addressBuilder.buildTo(_normalizedAddressBuilder, _type); - SparseTensorAddressRef taddress(_normalizedAddressBuilder.getAddressRef()); - // Make a persistent copy of sparse tensor address owned by _stash - SparseTensorAddressRef address(taddress, _stash); - _cells[address] = value; - _addressBuilder.clear(); - _normalizedAddressBuilder.clear(); - return *this; -} - - -Tensor::UP -SparseTensorBuilder::build() -{ - assert(_addressBuilder.empty()); - if (!_type_made) { - makeType(); - } - Tensor::UP ret = std::make_unique<SparseTensor>(std::move(_type), - std::move(_cells), - std::move(_stash)); - SparseTensor::Cells().swap(_cells); - _dimensionsEnum.clear(); - _dimensions.clear(); - _type = eval::ValueType::double_type(); - _type_made = false; - return ret; -} - -} - diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_builder.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_builder.h deleted file mode 100644 index daf8d797a91..00000000000 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_builder.h +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "sparse_tensor.h" -#include "sparse_tensor_address_builder.h" -#include "sparse_tensor_unsorted_address_builder.h" -#include <vespa/eval/tensor/tensor_address.h> -#include <vespa/vespalib/stllike/hash_map.h> -#include <vespa/vespalib/util/stash.h> - -namespace vespalib::tensor { - -/** - * A builder of sparse tensors. - */ -class SparseTensorBuilder -{ -public: - using Dimension = uint32_t; -private: - SparseTensorUnsortedAddressBuilder _addressBuilder; // unsorted dimensions - SparseTensorAddressBuilder _normalizedAddressBuilder; // sorted dimensions - SparseTensor::Cells _cells; - Stash _stash; - vespalib::hash_map<vespalib::string, uint32_t> _dimensionsEnum; - std::vector<vespalib::string> _dimensions; - eval::ValueType _type; - bool _type_made; - - void makeType(); -public: - SparseTensorBuilder(); - ~SparseTensorBuilder(); - - Dimension define_dimension(const vespalib::string &dimension); - SparseTensorBuilder & add_label(Dimension dimension, const vespalib::string &label); - SparseTensorBuilder &add_cell(double value); - Tensor::UP build(); -}; - -} - diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.cpp deleted file mode 100644 index 866956dd23e..00000000000 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.cpp +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "sparse_tensor_unsorted_address_builder.h" -#include "sparse_tensor_address_builder.h" -#include <vespa/eval/eval/value_type.h> -#include <cassert> -#include <algorithm> - -namespace vespalib::tensor { - -SparseTensorUnsortedAddressBuilder::SparseTensorUnsortedAddressBuilder() - : _elementStrings(), - _elements() -{ -} - -SparseTensorUnsortedAddressBuilder::~SparseTensorUnsortedAddressBuilder() = default; - -void -SparseTensorUnsortedAddressBuilder::buildTo(SparseTensorAddressBuilder & builder, - const eval::ValueType &type) -{ - const char *base = &_elementStrings[0]; - std::sort(_elements.begin(), _elements.end(), - [=](const ElementRef &lhs, const ElementRef &rhs) - { return lhs.getDimension(base) < rhs.getDimension(base); }); - // build normalized address with sorted dimensions - auto dimsItr = type.dimensions().cbegin(); - auto dimsItrEnd = type.dimensions().cend(); - for (const auto &element : _elements) { - while ((dimsItr != dimsItrEnd) && - (dimsItr->name < element.getDimension(base))) { - builder.addUndefined(); - ++dimsItr; - } - assert((dimsItr != dimsItrEnd) && - (dimsItr->name == element.getDimension(base))); - builder.add(element.getLabel(base)); - ++dimsItr; - } - while (dimsItr != dimsItrEnd) { - builder.addUndefined(); - ++dimsItr; - } -} - -} - diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.h deleted file mode 100644 index 7e69adb5804..00000000000 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.h +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/vespalib/stllike/string.h> -#include <vector> -#include <vespa/eval/tensor/types.h> - -namespace vespalib::eval { class ValueType; } -namespace vespalib::tensor { - -class SparseTensorAddressBuilder; - -/** - * A builder that buffers up a tensor address with unsorted - * dimensions. - */ -class SparseTensorUnsortedAddressBuilder -{ - struct ElementStringRef - { - uint32_t _base; - uint32_t _len; - ElementStringRef(uint32_t base, uint32_t len) - : _base(base), _len(len) - { - } - vespalib::stringref asStringRef(const char *base) const - { - return vespalib::stringref(base + _base, _len); - } - }; - struct ElementRef - { - ElementStringRef _dimension; - ElementStringRef _label; - ElementRef(ElementStringRef dimension, - ElementStringRef label) - : _dimension(dimension), - _label(label) - { - } - vespalib::stringref getDimension(const char *base) const { - return _dimension.asStringRef(base); - } - vespalib::stringref getLabel(const char *base) const { - return _label.asStringRef(base); - } - }; - std::vector<char> _elementStrings; // unsorted dimensions - std::vector<ElementRef> _elements; // unsorted dimensions - - ElementStringRef - append(vespalib::stringref str) - { - const char *cstr = str.data(); - uint32_t start = _elementStrings.size(); - _elementStrings.insert(_elementStrings.end(), - cstr, cstr + str.size() + 1); - return ElementStringRef(start, str.size()); - } - -public: - SparseTensorUnsortedAddressBuilder(); - ~SparseTensorUnsortedAddressBuilder(); - bool empty() const { return _elementStrings.empty(); } - void add(vespalib::stringref dimension, vespalib::stringref label) - { - _elements.emplace_back(append(dimension), append(label)); - } - /* - * Sort the stored tensor address and pass it over to a strict - * tensor address builder in sorted order. - */ - void buildTo(SparseTensorAddressBuilder &builder, const eval::ValueType &type); - void clear() { _elementStrings.clear(); _elements.clear(); } -}; - -} - |