diff options
8 files changed, 0 insertions, 546 deletions
diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 9830be2d00e..907cab7dbc6 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -76,7 +76,6 @@ vespa_define_module( src/tests/tensor/dense_tensor_builder src/tests/tensor/dense_tensor_operations src/tests/tensor/tensor_address - src/tests/tensor/tensor_address_element_iterator src/tests/tensor/tensor_conformance src/tests/tensor/tensor_function src/tests/tensor/tensor_mapper diff --git a/vespalib/src/testlist.txt b/vespalib/src/testlist.txt index a48777ee858..67982805df7 100644 --- a/vespalib/src/testlist.txt +++ b/vespalib/src/testlist.txt @@ -68,7 +68,6 @@ tests/tensor/dense_tensor_operations tests/tensor/simple_tensor_builder tests/tensor/tensor tests/tensor/tensor_address -tests/tensor/tensor_address_element_iterator tests/tensor/tensor_function tests/tensor/tensor_mapper tests/tensor/tensor_operations diff --git a/vespalib/src/tests/tensor/tensor_address_element_iterator/.gitignore b/vespalib/src/tests/tensor/tensor_address_element_iterator/.gitignore deleted file mode 100644 index c28cf0c86f2..00000000000 --- a/vespalib/src/tests/tensor/tensor_address_element_iterator/.gitignore +++ /dev/null @@ -1 +0,0 @@ -vespalib_tensor_address_element_iterator_test_app diff --git a/vespalib/src/tests/tensor/tensor_address_element_iterator/CMakeLists.txt b/vespalib/src/tests/tensor/tensor_address_element_iterator/CMakeLists.txt deleted file mode 100644 index dad69af7ba3..00000000000 --- a/vespalib/src/tests/tensor/tensor_address_element_iterator/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vespalib_tensor_address_element_iterator_test_app TEST - SOURCES - tensor_address_element_iterator_test.cpp - DEPENDS - vespalib - vespalib_vespalib_tensor -) -vespa_add_test(NAME vespalib_tensor_address_element_iterator_test_app COMMAND vespalib_tensor_address_element_iterator_test_app) diff --git a/vespalib/src/tests/tensor/tensor_address_element_iterator/FILES b/vespalib/src/tests/tensor/tensor_address_element_iterator/FILES deleted file mode 100644 index b185a25973e..00000000000 --- a/vespalib/src/tests/tensor/tensor_address_element_iterator/FILES +++ /dev/null @@ -1 +0,0 @@ -tensor_address_element_iterator_test.cpp diff --git a/vespalib/src/tests/tensor/tensor_address_element_iterator/tensor_address_element_iterator_test.cpp b/vespalib/src/tests/tensor/tensor_address_element_iterator/tensor_address_element_iterator_test.cpp deleted file mode 100644 index 4e953048f67..00000000000 --- a/vespalib/src/tests/tensor/tensor_address_element_iterator/tensor_address_element_iterator_test.cpp +++ /dev/null @@ -1,343 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/stllike/hash_set.h> -#include <vespa/vespalib/tensor/tensor_address.h> -#include <vespa/vespalib/tensor/tensor_address_builder.h> -#include <vespa/vespalib/tensor/sparse/compact_tensor_address.h> -#include <vespa/vespalib/tensor/sparse/compact_tensor_address_builder.h> -#include <vespa/vespalib/tensor/tensor_address_element_iterator.h> - -using namespace vespalib::tensor; - -using TensorAddressMap = std::map<std::string, std::string>; -using TensorAddressElementVec = - std::vector<std::pair<std::string, std::string>>; - -namespace vespalib -{ - -std::ostream & -operator<<(std::ostream &out, const TensorAddressElementVec &vec) -{ - out << "{"; - bool first = true; - for (const auto &elem : vec) { - if (!first) { - out << ","; - } - out << "{\"" << elem.first << "\",\"" << elem.second << "\"}"; - first = false; - } - out << "}"; - return out; -}; - -} - - -class DummyAddressBuilder -{ - TensorAddressElementVec _elements; -public: - void add(vespalib::stringref dimension, vespalib::stringref label) - { - _elements.emplace_back(dimension, label); - } - - const TensorAddressElementVec &elements() const { return _elements; } -}; - - -template <class TensorAddressT> struct FixtureBase; - -template <> struct FixtureBase<TensorAddress> -{ - using AddressType = TensorAddress; - using AddressBuilderType = TensorAddressBuilder; - - static TensorAddress create(TensorAddressBuilder &builder) { - return builder.build(); - } -}; - - -template <> struct FixtureBase<CompactTensorAddress> -{ - using AddressType = CompactTensorAddress; - using AddressBuilderType = CompactTensorAddressBuilder; - - vespalib::Stash _stash; - - CompactTensorAddress - create(CompactTensorAddressBuilder &builder) - { - CompactTensorAddressRef oldRef = builder.getAddressRef(); - CompactTensorAddressRef newRef(oldRef, _stash); - CompactTensorAddress ret; - ret.deserializeFromSparseAddressRef(newRef); - return ret; - } -}; - -template <> struct FixtureBase<CompactTensorAddressRef> -{ - using AddressType = CompactTensorAddressRef; - using AddressBuilderType = CompactTensorAddressBuilder; - - vespalib::Stash _stash; - - CompactTensorAddressRef - create(CompactTensorAddressBuilder &builder) - { - CompactTensorAddressRef oldRef = builder.getAddressRef(); - CompactTensorAddressRef newRef(oldRef, _stash); - return newRef; - } -}; - -template <class TensorAddressT> struct Fixture - : public FixtureBase<TensorAddressT> -{ - using Parent = FixtureBase<TensorAddressT>; - using AddressType = typename Parent::AddressType; - using AddressBuilderType = typename Parent::AddressBuilderType; - using Parent::create; - - AddressType - create(const TensorAddressMap &address_in) { - AddressBuilderType builder; - for (auto &element : address_in) { - builder.add(element.first, element.second); - } - return create(builder); - } - - void - verifyPlainIterate(const TensorAddressMap &address_in) - { - AddressType address = create(address_in); - TensorAddressElementIterator<AddressType> itr(address); - for (auto &element : address_in) { - EXPECT_TRUE(itr.valid()); - EXPECT_EQUAL(element.first, itr.dimension()); - EXPECT_EQUAL(element.second, itr.label()); - itr.next(); - } - EXPECT_FALSE(itr.valid()); - } - - - void - verifyPlainIterate() - { - TEST_DO(verifyPlainIterate({})); - TEST_DO(verifyPlainIterate({{"a", "1"}})); - TEST_DO(verifyPlainIterate({{"a", "1"}, {"b", "2"}})); - } - - void - verifyBeforeDimension(const TensorAddressMap &lhsAddress_in, - const TensorAddressMap &rhsAddress_in, - bool exp) - { - AddressType lhsAddress = create(lhsAddress_in); - TensorAddressElementIterator<AddressType> lhsItr(lhsAddress); - AddressType rhsAddress = create(rhsAddress_in); - TensorAddressElementIterator<AddressType> rhsItr(rhsAddress); - EXPECT_EQUAL(exp, lhsItr.beforeDimension(rhsItr)); - } - - void - verifyBeforeDimension() { - TEST_DO(verifyBeforeDimension({}, {}, false)); - TEST_DO(verifyBeforeDimension({}, {{"x", "1"}}, false)); - TEST_DO(verifyBeforeDimension({{"x", "1"}}, {}, true)); - TEST_DO(verifyBeforeDimension({{"x", "1"}}, {{"x", "2"}}, false)); - TEST_DO(verifyBeforeDimension({{"x", "1"}}, {{"y", "2"}}, true)); - TEST_DO(verifyBeforeDimension({{"y", "1"}}, {{"x", "2"}}, false)); - } - - void - verifyAtDimension(const TensorAddressMap &address_in, - vespalib::stringref dimension, - bool exp) - { - AddressType address = create(address_in); - TensorAddressElementIterator<AddressType> itr(address); - EXPECT_EQUAL(exp, itr.atDimension(dimension)); - } - - void - verifyAtDimension() - { - TEST_DO(verifyAtDimension({}, "x", false)); - TEST_DO(verifyAtDimension({{"x", "1"}}, "x", true)); - TEST_DO(verifyAtDimension({{"x", "1"}}, "y", false)); - TEST_DO(verifyAtDimension({{"y", "1"}}, "x", false)); - TEST_DO(verifyAtDimension({{"y", "1"}}, "y", true)); - } - - void - verifyAddElements(const TensorAddressMap &lhsAddress_in, - const TensorAddressMap &rhsAddress_in, - const TensorAddressElementVec &exp) - { - AddressType lhsAddress = create(lhsAddress_in); - TensorAddressElementIterator<AddressType> lhsItr(lhsAddress); - AddressType rhsAddress = create(rhsAddress_in); - TensorAddressElementIterator<AddressType> rhsItr(rhsAddress); - DummyAddressBuilder builder; - lhsItr.addElements(builder, rhsItr); - EXPECT_EQUAL(exp, builder.elements()); - } - - void verifyAddElements(const TensorAddressMap &address_in, - const TensorAddressElementVec &exp) - { - AddressType address = create(address_in); - TensorAddressElementIterator<AddressType> itr(address); - DummyAddressBuilder builder; - itr.addElements(builder); - EXPECT_EQUAL(exp, builder.elements()); - } - - void verifyAddElements(const TensorAddressMap &address_in, - const DimensionsSet &dimensions, - bool exp, - const TensorAddressElementVec &expVec) - { - AddressType address = create(address_in); - TensorAddressElementIterator<AddressType> itr(address); - DummyAddressBuilder builder; - EXPECT_EQUAL(exp, itr.addElements(builder, dimensions)); - EXPECT_EQUAL(expVec, builder.elements()); - } - - void verifyAddElements(const TensorAddressMap &lhsAddress_in, - const TensorAddressMap &rhsAddress_in, - const DimensionsSet &dimensions, - bool exp, - const TensorAddressElementVec &expVec) - { - AddressType lhsAddress = create(lhsAddress_in); - TensorAddressElementIterator<AddressType> lhsItr(lhsAddress); - AddressType rhsAddress = create(rhsAddress_in); - TensorAddressElementIterator<AddressType> rhsItr(rhsAddress); - DummyAddressBuilder builder; - ASSERT_TRUE(lhsItr.beforeDimension(rhsItr)); - EXPECT_EQUAL(exp, lhsItr.addElements(builder, dimensions, rhsItr)); - EXPECT_EQUAL(expVec, builder.elements()); - } - - void - verifyAddElements() - { - // Stop according to rhs iterator - TEST_DO(verifyAddElements({}, {}, {})); - TEST_DO(verifyAddElements({{"x", "1"}}, {}, {{"x", "1"}})); - TEST_DO(verifyAddElements({{"x", "1"}}, {{"x", "1"}}, {})); - TEST_DO(verifyAddElements({{"x", "1"}}, {{"y", "1"}}, {{"x", "1"}})); - TEST_DO(verifyAddElements({{"y", "1"}}, {{"x", "1"}}, {})); - TEST_DO(verifyAddElements({{"x", "1"}, {"y", "2"}}, {{"z", "1"}}, - {{"x", "1"}, {"y", "2"}})); - // Pass through everything - TEST_DO(verifyAddElements({}, {})); - TEST_DO(verifyAddElements({{"x", "1"}}, {{"x", "1"}})); - TEST_DO(verifyAddElements({{"x", "1"}, {"y", "2"}}, - {{"x", "1"}, {"y", "2"}})); - // Filter on dimension set - TEST_DO(verifyAddElements({}, {}, true, {})); - TEST_DO(verifyAddElements({{"x", "1"}}, {}, true, {{"x", "1"}})); - TEST_DO(verifyAddElements({{"x", "1"}, {"y", "2"}}, {}, true, - {{"x", "1"}, {"y", "2"}})); - TEST_DO(verifyAddElements({{"x", "1"}, {"y", "2"}}, {"y"}, false, - {{"x", "1"}})); - // Filter on dimension set and stop according to rhs iterator - TEST_DO(verifyAddElements({{"x", "1"}}, {}, {}, true, {{"x", "1"}})); - TEST_DO(verifyAddElements({{"x", "1"}, {"y", "2"}}, {}, {}, true, - {{"x", "1"}, {"y", "2"}})); - TEST_DO(verifyAddElements({{"x", "1"}, {"y", "2"}}, {{"y", "2"}}, {}, - true, {{"x", "1"}})); - TEST_DO(verifyAddElements({{"x", "1"}, {"y", "2"}}, {{"y", "2"}}, {"y"}, - true, {{"x", "1"}})); - TEST_DO(verifyAddElements({{"x", "1"}, {"y", "2"}}, {{"y", "2"}}, {"x"}, - false, {})); - } -}; - - -TEST_F("Test that Tensor address can be iterated", Fixture<TensorAddress>) -{ - f.verifyPlainIterate(); -} - -TEST_F("Test that compact Tensor address can be iterated", - Fixture<CompactTensorAddress>) -{ - f.verifyPlainIterate(); -} - - -TEST_F("Test that compact Tensor address ref can be iterated", - Fixture<CompactTensorAddressRef>) -{ - f.verifyPlainIterate(); -} - -TEST_F("Test that Tensor address works with beforeDimension", - Fixture<TensorAddress>) -{ - f.verifyBeforeDimension(); -} - -TEST_F("Test that compact Tensor address works with beforeDimension", - Fixture<CompactTensorAddress>) -{ - f.verifyBeforeDimension(); -} - -TEST_F("Test that compat Tensor address ref works with beforeDimension", - Fixture<CompactTensorAddressRef>) -{ - f.verifyBeforeDimension(); -} - -TEST_F("Test that Tensor address works with atDimension", - Fixture<TensorAddress>) -{ - f.verifyAtDimension(); -} - -TEST_F("Test that compact Tensor address works with atDimension", - Fixture<CompactTensorAddress>) -{ - f.verifyAtDimension(); -} - -TEST_F("Test that compat Tensor address ref works with atDimension", - Fixture<CompactTensorAddressRef>) -{ - f.verifyAtDimension(); -} - -TEST_F("Test that Tensor address works with addElements", - Fixture<TensorAddress>) -{ - f.verifyAddElements(); -} - -TEST_F("Test that compact Tensor address works with addElements", - Fixture<CompactTensorAddress>) -{ - f.verifyAddElements(); -} - -TEST_F("Test that compat Tensor address ref works with addElements", - Fixture<CompactTensorAddressRef>) -{ - f.verifyAddElements(); -} - - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/vespa/vespalib/tensor/tensor_address_element_iterator.h b/vespalib/src/vespa/vespalib/tensor/tensor_address_element_iterator.h index 3a260c7c693..a250331de5f 100644 --- a/vespalib/src/vespa/vespalib/tensor/tensor_address_element_iterator.h +++ b/vespalib/src/vespa/vespalib/tensor/tensor_address_element_iterator.h @@ -25,68 +25,7 @@ public: bool valid() const { return (_itr != _itrEnd); } vespalib::stringref dimension() const { return _itr->dimension(); } vespalib::stringref label() const { return _itr->label(); } - template <class Iterator> - bool beforeDimension(const Iterator &rhs) const { - if (!valid()) { - return false; - } - if (!rhs.valid()) { - return true; - } - return (_itr->dimension() < rhs.dimension()); - } - bool atDimension(vespalib::stringref rhsDimension) const - { - return (valid() && (_itr->dimension() == rhsDimension)); - } void next() { ++_itr; } - template <class AddressBuilder> - void - addElement(AddressBuilder &builder) { - builder.add(_itr->dimension(), _itr->label()); - } - template <class AddressBuilder, class Iterator> - void addElements(AddressBuilder &builder, const Iterator &limit) - { - while (beforeDimension(limit)) { - addElement(builder); - next(); - } - } - template <class AddressBuilder, class Iterator> - bool addElements(AddressBuilder &builder, const DimensionsSet &dims, - const Iterator &limit) - { - do { - if (dims.find(_itr->dimension()) != dims.end()) { - return false; - } - addElement(builder); - next(); - } while (beforeDimension(limit)); - return true; - } - template <class AddressBuilder> - void addElements(AddressBuilder &builder) - { - while (valid()) { - addElement(builder); - next(); - } - } - template <class AddressBuilder> - bool addElements(AddressBuilder &builder, const DimensionsSet &dims) - { - while (valid()) { - if (dims.find(_itr->dimension()) != dims.end()) { - return false; - } - addElement(builder); - next(); - } - return true; - } - bool skipToDimension(vespalib::stringref rhsDimension) { for (;;) { if (!valid()) { @@ -101,118 +40,5 @@ public: } }; - -/** - * An iterator for tensor address elements used to simplify 3-way merge - * between two tensor addresses and a dimension vector. - * This is a specialization to perform decoding on the fly while iterating. - */ -template <> -class TensorAddressElementIterator<CompactTensorAddressRef> { - const char *_itr; - const char *_itrEnd; - vespalib::stringref _dimension; - vespalib::stringref _label; - - size_t - simple_strlen(const char *str) { - const char *strend = str; - for (; *strend != '\0'; ++strend) { - } - return (strend - str); - } - - void decodeElement() - { - _dimension = vespalib::stringref(_itr, simple_strlen(_itr)); - const char *labelp = _dimension.c_str() + _dimension.size() + 1; - _label = vespalib::stringref(labelp, simple_strlen(labelp)); - _itr = _label.c_str() + _label.size() + 1; - } -public: - TensorAddressElementIterator(CompactTensorAddressRef address) - : _itr(static_cast<const char *>(address.start())), - _itrEnd(_itr + address.size()), - _dimension(), - _label() - { - if (_itr != _itrEnd) { - decodeElement(); - } - } - bool valid() const { return (_dimension.size() != 0u); } - vespalib::stringref dimension() const { return _dimension; } - vespalib::stringref label() const { return _label; } - template <class Iterator> - bool beforeDimension(const Iterator &rhs) const { - if (!valid()) { - return false; - } - if (!rhs.valid()) { - return true; - } - return (_dimension < rhs.dimension()); - } - bool atDimension(vespalib::stringref rhsDimension) const - { - return (_dimension == rhsDimension); - } - void next() { - if (_itr != _itrEnd) { - decodeElement(); - } else { - _dimension = vespalib::stringref(); - _label = vespalib::stringref(); - } - } - template <class AddressBuilder> - void - addElement(AddressBuilder &builder) { - builder.add(_dimension, _label); - } - template <class AddressBuilder, class Iterator> - void addElements(AddressBuilder &builder, const Iterator &limit) - { - while (beforeDimension(limit)) { - addElement(builder); - next(); - } - } - template <class AddressBuilder, class Iterator> - bool addElements(AddressBuilder &builder, const DimensionsSet &dims, - const Iterator &limit) - { - do { - if (dims.find(_dimension) != dims.end()) { - return false; - } - addElement(builder); - next(); - } while (beforeDimension(limit)); - return true; - } - template <class AddressBuilder> - void addElements(AddressBuilder &builder) - { - while (valid()) { - addElement(builder); - next(); - } - } - template <class AddressBuilder> - bool addElements(AddressBuilder &builder, const DimensionsSet &dims) - { - while (valid()) { - if (dims.find(_dimension) != dims.end()) { - return false; - } - addElement(builder); - next(); - } - return true; - } -}; - - } // namespace vespalib::tensor } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp b/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp index 460cb8f8bb4..a527627d786 100644 --- a/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp +++ b/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp @@ -98,22 +98,6 @@ mapAddress(const TensorAddress &address) template <class TensorT> void -SparseTensorMapper<TensorT>::mapAddress(const TensorAddress &address) -{ - _addressBuilder.clear(); - TensorAddressElementIterator<TensorAddress> addressIterator(address); - for (const auto &dimension : _builder.dimensions()) { - if (addressIterator.skipToDimension(dimension)) { - _addressBuilder.add(dimension, addressIterator.label()); - addressIterator.next(); - } else { - // output dimension not in input - } - } -} - -template <class TensorT> -void SparseTensorMapper<TensorT>::visit(const TensorAddress &address, double value) { mapAddress(address); |