diff options
46 files changed, 437 insertions, 866 deletions
diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 26e046ec52a..e0159febdfe 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -25,6 +25,7 @@ vespa_define_module( src/tests/eval/value_cache src/tests/eval/value_type src/tests/gp/ponder_nov2017 + src/tests/tensor/dense_dimension_combiner src/tests/tensor/dense_add_dimension_optimizer src/tests/tensor/dense_dot_product_function src/tests/tensor/dense_fast_rename_optimizer @@ -32,7 +33,6 @@ vespa_define_module( src/tests/tensor/dense_inplace_map_function src/tests/tensor/dense_remove_dimension_optimizer src/tests/tensor/dense_replace_type_function - src/tests/tensor/dense_tensor_address_combiner src/tests/tensor/dense_xw_product_function src/tests/tensor/direct_dense_tensor_builder src/tests/tensor/direct_sparse_tensor_builder diff --git a/eval/src/tests/tensor/dense_dimension_combiner/CMakeLists.txt b/eval/src/tests/tensor/dense_dimension_combiner/CMakeLists.txt new file mode 100644 index 00000000000..eaee8ebb4e4 --- /dev/null +++ b/eval/src/tests/tensor/dense_dimension_combiner/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +vespa_add_executable(eval_dense_dimension_combiner_test_app TEST + SOURCES + dense_dimension_combiner_test.cpp + DEPENDS + vespaeval +) +vespa_add_test(NAME eval_dense_dimension_combiner_test_app COMMAND eval_dense_dimension_combiner_test_app) diff --git a/eval/src/tests/tensor/dense_dimension_combiner/dense_dimension_combiner_test.cpp b/eval/src/tests/tensor/dense_dimension_combiner/dense_dimension_combiner_test.cpp new file mode 100644 index 00000000000..b8949e3a7e6 --- /dev/null +++ b/eval/src/tests/tensor/dense_dimension_combiner/dense_dimension_combiner_test.cpp @@ -0,0 +1,185 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/eval/tensor/dense/dense_dimension_combiner.h> + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::tensor; + +void verifyLeft(DenseDimensionCombiner &d, size_t last) { + d.commonReset(); + d.leftReset(); + EXPECT_TRUE(d.leftInRange()); + EXPECT_EQUAL(d.leftIdx(), 0u); + size_t expect = 0; + while (d.leftInRange()) { + d.stepLeft(); + EXPECT_GREATER(d.leftIdx(), expect); + expect = d.leftIdx(); + } + EXPECT_FALSE(d.leftInRange()); + EXPECT_EQUAL(expect, last); + d.leftReset(); + EXPECT_TRUE(d.leftInRange()); + EXPECT_EQUAL(d.leftIdx(), 0u); +} + +void verifyRight(DenseDimensionCombiner &d, size_t last) { + d.commonReset(); + d.rightReset(); + EXPECT_TRUE(d.rightInRange()); + EXPECT_EQUAL(d.rightIdx(), 0u); + size_t expect = 0; + while (d.rightInRange()) { + d.stepRight(); + EXPECT_GREATER(d.rightIdx(), expect); + expect = d.rightIdx(); + } + EXPECT_FALSE(d.rightInRange()); + EXPECT_EQUAL(expect, last); + d.rightReset(); + EXPECT_TRUE(d.rightInRange()); + EXPECT_EQUAL(d.rightIdx(), 0u); +} + + +TEST("require that one left, one common, one right dimension works") { + ValueType t12_lc = ValueType::tensor_type({{"d1_l", 3},{"d2_c", 4}}); + ValueType t23_cr = ValueType::tensor_type({{"d2_c", 4},{"d3_r", 5}}); + + DenseDimensionCombiner d(t12_lc, t23_cr); + + EXPECT_TRUE(d.leftInRange()); + EXPECT_TRUE(d.rightInRange()); + EXPECT_TRUE(d.commonInRange()); + EXPECT_EQUAL(d.leftIdx(), 0u); + EXPECT_EQUAL(d.rightIdx(), 0u); + EXPECT_EQUAL(d.outputIdx(), 0u); + + d.stepCommon(); + EXPECT_TRUE(d.leftInRange()); + EXPECT_TRUE(d.rightInRange()); + EXPECT_TRUE(d.commonInRange()); + EXPECT_EQUAL(d.leftIdx(), 1u); + EXPECT_EQUAL(d.rightIdx(), 5u); + EXPECT_EQUAL(d.outputIdx(), 5u); + + d.stepRight(); + EXPECT_TRUE(d.leftInRange()); + EXPECT_TRUE(d.rightInRange()); + EXPECT_TRUE(d.commonInRange()); + EXPECT_EQUAL(d.leftIdx(), 1u); + EXPECT_EQUAL(d.rightIdx(), 6u); + EXPECT_EQUAL(d.outputIdx(), 6u); + + d.stepLeft(); + EXPECT_TRUE(d.leftInRange()); + EXPECT_TRUE(d.rightInRange()); + EXPECT_TRUE(d.commonInRange()); + EXPECT_EQUAL(d.leftIdx(), 5u); + EXPECT_EQUAL(d.rightIdx(), 6u); + EXPECT_EQUAL(d.outputIdx(), 26u); + + d.stepLeft(); + EXPECT_TRUE(d.leftInRange()); + EXPECT_TRUE(d.rightInRange()); + EXPECT_TRUE(d.commonInRange()); + EXPECT_EQUAL(d.leftIdx(), 9u); + EXPECT_EQUAL(d.rightIdx(), 6u); + EXPECT_EQUAL(d.outputIdx(), 46u); + + d.stepLeft(); + EXPECT_FALSE(d.leftInRange()); + EXPECT_TRUE(d.rightInRange()); + EXPECT_TRUE(d.commonInRange()); + EXPECT_EQUAL(d.leftIdx(), 13u); + EXPECT_EQUAL(d.rightIdx(), 6u); + EXPECT_EQUAL(d.outputIdx(), 6u); + + d.leftReset(); + EXPECT_TRUE(d.leftInRange()); + EXPECT_TRUE(d.rightInRange()); + EXPECT_TRUE(d.commonInRange()); + EXPECT_EQUAL(d.leftIdx(), 1u); + EXPECT_EQUAL(d.rightIdx(), 6u); + EXPECT_EQUAL(d.outputIdx(), 6u); + + d.stepCommon(); + EXPECT_TRUE(d.leftInRange()); + EXPECT_TRUE(d.rightInRange()); + EXPECT_TRUE(d.commonInRange()); + EXPECT_EQUAL(d.leftIdx(), 2u); + EXPECT_EQUAL(d.rightIdx(), 11u); + EXPECT_EQUAL(d.outputIdx(), 11u); + + d.stepRight(); + EXPECT_TRUE(d.leftInRange()); + EXPECT_TRUE(d.rightInRange()); + EXPECT_TRUE(d.commonInRange()); + EXPECT_EQUAL(d.leftIdx(), 2u); + EXPECT_EQUAL(d.rightIdx(), 12u); + EXPECT_EQUAL(d.outputIdx(), 12u); + + TEST_DO(verifyLeft(d, 12)); + TEST_DO(verifyRight(d, 20)); +} + +TEST("require that two left, no common, two right dimensions works") { + ValueType t12_ll = ValueType::tensor_type({{"d1_l", 3},{"d2_l", 4}}); + ValueType t34_rr = ValueType::tensor_type({{"d3_r", 5},{"d4_r", 2}}); + + DenseDimensionCombiner d(t12_ll, t34_rr); + + EXPECT_TRUE(d.leftInRange()); + EXPECT_TRUE(d.rightInRange()); + EXPECT_TRUE(d.commonInRange()); + EXPECT_EQUAL(d.leftIdx(), 0u); + EXPECT_EQUAL(d.rightIdx(), 0u); + EXPECT_EQUAL(d.outputIdx(), 0u); + + d.stepCommon(); + EXPECT_TRUE(d.leftInRange()); + EXPECT_TRUE(d.rightInRange()); + EXPECT_FALSE(d.commonInRange()); + EXPECT_EQUAL(d.leftIdx(), 0u); + EXPECT_EQUAL(d.rightIdx(), 0u); + EXPECT_EQUAL(d.outputIdx(), 120u); + + d.commonReset(); + d.stepRight(); + EXPECT_TRUE(d.leftInRange()); + EXPECT_TRUE(d.rightInRange()); + EXPECT_TRUE(d.commonInRange()); + EXPECT_EQUAL(d.leftIdx(), 0u); + EXPECT_EQUAL(d.rightIdx(), 1u); + EXPECT_EQUAL(d.outputIdx(), 1u); + + d.stepLeft(); + EXPECT_TRUE(d.leftInRange()); + EXPECT_TRUE(d.rightInRange()); + EXPECT_TRUE(d.commonInRange()); + EXPECT_EQUAL(d.leftIdx(), 1u); + EXPECT_EQUAL(d.rightIdx(), 1u); + EXPECT_EQUAL(d.outputIdx(), 11u); + + d.stepLeft(); + d.stepLeft(); + d.stepLeft(); + d.stepLeft(); + d.stepLeft(); + d.stepLeft(); + d.stepLeft(); + + EXPECT_TRUE(d.leftInRange()); + EXPECT_TRUE(d.rightInRange()); + EXPECT_TRUE(d.commonInRange()); + EXPECT_EQUAL(d.leftIdx(), 8u); + EXPECT_EQUAL(d.rightIdx(), 1u); + EXPECT_EQUAL(d.outputIdx(), 81u); + + TEST_DO(verifyLeft(d, 12)); + TEST_DO(verifyRight(d, 10)); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/tensor/dense_tensor_address_combiner/CMakeLists.txt b/eval/src/tests/tensor/dense_tensor_address_combiner/CMakeLists.txt deleted file mode 100644 index a006d70935d..00000000000 --- a/eval/src/tests/tensor/dense_tensor_address_combiner/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(eval_dense_tensor_address_combiner_test_app TEST - SOURCES - dense_tensor_address_combiner_test.cpp - DEPENDS - vespaeval -) -vespa_add_test(NAME eval_dense_tensor_address_combiner_test_app COMMAND eval_dense_tensor_address_combiner_test_app) diff --git a/eval/src/tests/tensor/dense_tensor_address_combiner/dense_tensor_address_combiner_test.cpp b/eval/src/tests/tensor/dense_tensor_address_combiner/dense_tensor_address_combiner_test.cpp deleted file mode 100644 index 91a6087ea3a..00000000000 --- a/eval/src/tests/tensor/dense_tensor_address_combiner/dense_tensor_address_combiner_test.cpp +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/eval/tensor/dense/dense_tensor_address_combiner.h> -#include <vespa/vespalib/test/insertion_operators.h> - -using namespace vespalib::tensor; -using vespalib::eval::ValueType; - -ValueType -combine(const std::vector<ValueType::Dimension> &lhs, - const std::vector<ValueType::Dimension> &rhs) -{ - return DenseTensorAddressCombiner::combineDimensions( - ValueType::tensor_type(lhs), - ValueType::tensor_type(rhs)); -} - -TEST("require that dimensions can be combined") -{ - EXPECT_EQUAL(ValueType::tensor_type({{"a", 3}, {"b", 5}}), combine({{"a", 3}}, {{"b", 5}})); - EXPECT_EQUAL(ValueType::tensor_type({{"a", 3}, {"b", 5}}), combine({{"a", 3}, {"b", 5}}, {{"b", 5}})); - EXPECT_EQUAL(ValueType::tensor_type({{"a", 3}, {"b", 5}}), combine({{"a", 3}, {"b", 5}}, {{"b", 5}})); - EXPECT_EQUAL(ValueType::tensor_type({{"a", 3}, {"b", 11}, {"c", 5}, {"d", 7}, {"e", 17}}), - combine({{"a", 3}, {"c", 5}, {"d", 7}}, - {{"b", 11}, {"c", 5}, {"e", 17}})); - EXPECT_EQUAL(ValueType::tensor_type({{"a", 3}, {"b", 11}, {"c", 5}, {"d", 7}, {"e", 17}}), - combine({{"b", 11}, {"c", 5}, {"e", 17}}, - {{"a", 3}, {"c", 5}, {"d", 7}})); -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/vespa/eval/tensor/dense/CMakeLists.txt b/eval/src/vespa/eval/tensor/dense/CMakeLists.txt index c2638466de6..ce20d6ba6d9 100644 --- a/eval/src/vespa/eval/tensor/dense/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/dense/CMakeLists.txt @@ -2,16 +2,16 @@ vespa_add_library(eval_tensor_dense OBJECT SOURCES dense_add_dimension_optimizer.cpp + dense_dimension_combiner.cpp dense_dot_product_function.cpp dense_fast_rename_optimizer.cpp dense_inplace_join_function.cpp dense_inplace_map_function.cpp dense_remove_dimension_optimizer.cpp dense_replace_type_function.cpp - dense_tensor.cpp - dense_tensor_address_combiner.cpp dense_tensor_address_mapper.cpp dense_tensor_cells_iterator.cpp + dense_tensor.cpp dense_tensor_modify.cpp dense_tensor_reduce.cpp dense_tensor_view.cpp diff --git a/eval/src/vespa/eval/tensor/dense/dense_dimension_combiner.cpp b/eval/src/vespa/eval/tensor/dense/dense_dimension_combiner.cpp new file mode 100644 index 00000000000..22c8ff12ad1 --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/dense_dimension_combiner.cpp @@ -0,0 +1,91 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "dense_dimension_combiner.h" +#include <cassert> + +namespace vespalib::tensor { + +DenseDimensionCombiner::~DenseDimensionCombiner() = default; + +DenseDimensionCombiner::DenseDimensionCombiner(const eval::ValueType &lhs, + const eval::ValueType &rhs) + : _left(), _right(), + _commonDims(), + _outputIndex(0), + _outputSize(1u), + result_type(eval::ValueType::join(lhs, rhs)) +{ + assert(lhs.is_dense()); + assert(rhs.is_dense()); + assert(result_type.is_dense()); + + const auto &lDims = lhs.dimensions(); + const auto &rDims = rhs.dimensions(); + const auto &oDims = result_type.dimensions(); + + size_t i = lDims.size(); + size_t j = rDims.size(); + size_t k = oDims.size(); + + uint32_t lMul = 1; + uint32_t rMul = 1; + uint32_t oMul = 1; + + while (k-- > 0) { + if ((i > 0) && (lDims[i-1].name == oDims[k].name)) { + --i; + // left dim match + if ((j > 0) && (rDims[j-1].name == oDims[k].name)) { + // both dim match + --j; + CommonDim cd; + cd.idx = 0; + cd.leftMultiplier = lMul; + cd.rightMultiplier = rMul; + cd.outputMultiplier = oMul; + assert(lDims[i].size == oDims[k].size); + assert(rDims[j].size == oDims[k].size); + cd.size = oDims[k].size; + lMul *= cd.size; + rMul *= cd.size; + oMul *= cd.size; + _left.totalSize *= cd.size; + _right.totalSize *= cd.size; + _outputSize *= cd.size; + _commonDims.push_back(cd); + } else { + SideDim ld; + ld.idx = 0; + ld.sideMultiplier = lMul; + ld.outputMultiplier = oMul; + assert(lDims[i].size == oDims[k].size); + ld.size = oDims[k].size; + lMul *= ld.size; + oMul *= ld.size; + _outputSize *= ld.size; + _left.totalSize *= ld.size; + _left.dims.push_back(ld); + } + } else { + // right dim match + assert(j > 0); + assert(rDims[j-1].name == oDims[k].name); + --j; + SideDim rd; + rd.idx = 0; + rd.sideMultiplier = rMul; + rd.outputMultiplier = oMul; + assert(rDims[j].size == oDims[k].size); + rd.size = oDims[k].size; + rMul *= rd.size; + oMul *= rd.size; + _outputSize *= rd.size; + _right.totalSize *= rd.size; + _right.dims.push_back(rd); + } + } +} + + +} // namespace + diff --git a/eval/src/vespa/eval/tensor/dense/dense_dimension_combiner.h b/eval/src/vespa/eval/tensor/dense/dense_dimension_combiner.h new file mode 100644 index 00000000000..dd3f74bad9b --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/dense_dimension_combiner.h @@ -0,0 +1,114 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/tensor/tensor.h> +#include <vespa/eval/tensor/types.h> +#include <vespa/eval/eval/value_type.h> + +namespace vespalib::tensor { + +class DenseDimensionCombiner { + + struct SideDim { + uint32_t idx; + uint32_t size; + uint32_t sideMultiplier; + uint32_t outputMultiplier; + }; + struct CommonDim { + uint32_t idx; + uint32_t size; + uint32_t leftMultiplier; + uint32_t rightMultiplier; + uint32_t outputMultiplier; + }; + + struct SideDims { + std::vector<SideDim> dims; + uint32_t index; + uint32_t totalSize; + + SideDims() : dims(), index(0), totalSize(1u) {} + + void reset(uint32_t &outIndex) { + for (SideDim& d : dims) { + index -= d.idx * d.sideMultiplier; + outIndex -= d.idx * d.outputMultiplier; + d.idx = 0; + } + if (index >= totalSize) { + index -= totalSize; + } + } + void step(uint32_t &outIndex) { + for (SideDim& d : dims) { + d.idx++; + index += d.sideMultiplier; + outIndex += d.outputMultiplier; + if (d.idx < d.size) return; + index -= d.idx * d.sideMultiplier; + outIndex -= d.idx * d.outputMultiplier; + d.idx = 0; + } + index += totalSize; + } + }; + SideDims _left; + SideDims _right; + std::vector<CommonDim> _commonDims; + uint32_t _outputIndex; + uint32_t _outputSize; + +public: + size_t leftIdx() const { return _left.index; } + size_t rightIdx() const { return _right.index; } + size_t outputIdx() const { return _outputIndex; } + + bool leftInRange() const { return _left.index < _left.totalSize; } + bool rightInRange() const { return _right.index < _right.totalSize; } + bool commonInRange() const { return _outputIndex < _outputSize; } + + void leftReset() { _left.reset(_outputIndex); } + void stepLeft() { _left.step(_outputIndex); } + + void rightReset() { _right.reset(_outputIndex); } + void stepRight() { _right.step(_outputIndex); } + + void commonReset() { + for (CommonDim& cd : _commonDims) { + _left.index -= cd.idx * cd.leftMultiplier; + _right.index -= cd.idx * cd.rightMultiplier; + _outputIndex -= cd.idx * cd.outputMultiplier; + cd.idx = 0; + } + if (_outputIndex >= _outputSize) { + _outputIndex -= _outputSize; + } + } + + void stepCommon() { + size_t lim = _commonDims.size(); + for (size_t i = 0; i < lim; ++i) { + CommonDim &cd = _commonDims[i]; + cd.idx++; + _left.index += cd.leftMultiplier; + _right.index += cd.rightMultiplier; + _outputIndex += cd.outputMultiplier; + if (cd.idx < cd.size) return; + _left.index -= cd.idx * cd.leftMultiplier; + _right.index -= cd.idx * cd.rightMultiplier; + _outputIndex -= cd.idx * cd.outputMultiplier; + cd.idx = 0; + } + _outputIndex += _outputSize; + } + + const eval::ValueType result_type; + + DenseDimensionCombiner(const eval::ValueType &lhs, const eval::ValueType &rhs); + + ~DenseDimensionCombiner(); +}; + +} diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_address_combiner.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_address_combiner.cpp deleted file mode 100644 index b5c5d9b6a04..00000000000 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_address_combiner.cpp +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "dense_tensor_address_combiner.h" -#include <vespa/vespalib/util/exceptions.h> -#include <cassert> - -namespace vespalib::tensor { - -DenseTensorAddressCombiner::~DenseTensorAddressCombiner() = default; - -DenseTensorAddressCombiner::DenseTensorAddressCombiner(const eval::ValueType &combined, const eval::ValueType &lhs, - const eval::ValueType &rhs) - : _rightAddress(rhs), - _combinedAddress(combined), - _left(), - _commonRight(), - _right() -{ - auto rhsItr = rhs.dimensions().cbegin(); - auto rhsItrEnd = rhs.dimensions().cend(); - uint32_t numDimensions(0); - for (const auto &lhsDim : lhs.dimensions()) { - while ((rhsItr != rhsItrEnd) && (rhsItr->name < lhsDim.name)) { - _right.emplace_back(numDimensions++, rhsItr-rhs.dimensions().cbegin()); - ++rhsItr; - } - if ((rhsItr != rhsItrEnd) && (rhsItr->name == lhsDim.name)) { - _left.emplace_back(numDimensions, _left.size()); - _commonRight.emplace_back(numDimensions, rhsItr-rhs.dimensions().cbegin()); - ++numDimensions; - ++rhsItr; - } else { - _left.emplace_back(numDimensions++, _left.size()); - } - } - while (rhsItr != rhsItrEnd) { - _right.emplace_back(numDimensions++, rhsItr-rhs.dimensions().cbegin()); - ++rhsItr; - } -} - -AddressContext::AddressContext(const eval::ValueType &type) - : _type(type), - _accumulatedSize(_type.dimensions().size()), - _address(type.dimensions().size(), 0) - -{ - size_t multiplier = 1; - for (int32_t i(_address.size() - 1); i >= 0; i--) { - _accumulatedSize[i] = multiplier; - multiplier *= type.dimensions()[i].size; - } -} - -AddressContext::~AddressContext() = default; - -eval::ValueType -DenseTensorAddressCombiner::combineDimensions(const eval::ValueType &lhs, const eval::ValueType &rhs) -{ - return eval::ValueType::join(lhs, rhs); -} - -} diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_address_combiner.h b/eval/src/vespa/eval/tensor/dense/dense_tensor_address_combiner.h deleted file mode 100644 index 3f6e347490c..00000000000 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_address_combiner.h +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "dense_tensor_cells_iterator.h" -#include <vespa/eval/tensor/tensor.h> -#include <vespa/eval/tensor/types.h> -#include <vespa/eval/eval/value_type.h> - -namespace vespalib::tensor { - -class AddressContext { -public: - using Address = DenseTensorCellsIterator::Address; - using size_type = eval::ValueType::Dimension::size_type; - using Mapping = std::vector<std::pair<uint32_t, uint32_t>>; - AddressContext(const eval::ValueType &type); - ~AddressContext(); - size_type dimSize(uint32_t dim) const { return _type.dimensions()[dim].size; } - size_type wholeDimStep(uint32_t dim) const { return _accumulatedSize[dim] * dimSize(dim); } - size_t index() const { - size_t cellIdx(0); - for (uint32_t i(0); i < _address.size(); i++) { - cellIdx += _address[i]*_accumulatedSize[i]; - } - return cellIdx; - } - void update(const Address & addr, const Mapping & mapping) { - for (const auto & m : mapping) { - _address[m.first] = addr[m.second]; - } - } - bool updateCommon(const Address & addr, const Mapping & mapping) { - for (const auto & m : mapping) { - if (addr[m.first] >= dimSize(m.second)) { - return false; - } - _address[m.second] = addr[m.first]; - } - return true; - } - - const eval::ValueType &_type; - std::vector<size_t> _accumulatedSize; - Address _address; - -}; - -/** - * Combines two dense tensor addresses to a new tensor address. - * The resulting dimensions is the union of the input dimensions and - * common dimensions must have matching labels. - */ -class DenseTensorAddressCombiner -{ -public: - using Mapping = AddressContext::Mapping; - -private: - using Address = DenseTensorCellsIterator::Address; - using CellsRef = vespalib::ConstArrayRef<double>; - using size_type = eval::ValueType::Dimension::size_type; - - AddressContext _rightAddress; - AddressContext _combinedAddress; - - Mapping _left; - Mapping _commonRight; - Mapping _right; - -public: - DenseTensorAddressCombiner(const eval::ValueType &combined, const eval::ValueType &lhs, const eval::ValueType &rhs); - ~DenseTensorAddressCombiner(); - void updateLeftAndCommon(const Address & addr) { _combinedAddress.update(addr, _left); } - bool updateCommon() { return _rightAddress.updateCommon(_combinedAddress._address, _commonRight); } - bool hasAnyRightOnlyDimensions() const { return ! _right.empty(); } - - const Address & address() const { return _combinedAddress._address; } - size_t rightCellIndex() const { return _rightAddress.index(); } - - template <typename Func> - void for_each_right(const CellsRef & rhsCells, Func && func) { - // The rightAddress oly holds the starting point for iteration and what is need to efficiently maintain - // an index for addressing th ecells. - const int32_t lastDimension = _right.size() - 1; - int32_t curDimension = lastDimension; - size_t rightCellIdx = _rightAddress.index(); - size_t combinedCellIdx = _combinedAddress.index(); - while (curDimension >= 0) { - const uint32_t rdim = _right[curDimension].second; - const uint32_t cdim = _right[curDimension].first; - size_type & cindex = _combinedAddress._address[cdim]; - if (curDimension == lastDimension) { - for (cindex = 0; cindex < _rightAddress.dimSize(rdim); cindex++) { - func(combinedCellIdx, rhsCells[rightCellIdx]); - rightCellIdx += _rightAddress._accumulatedSize[rdim]; - combinedCellIdx += _combinedAddress._accumulatedSize[cdim]; - } - cindex = 0; - rightCellIdx -= _rightAddress.wholeDimStep(rdim); - combinedCellIdx -= _combinedAddress.wholeDimStep(cdim); - curDimension--; - } else { - if ((cindex + 1) < _rightAddress.dimSize(rdim)) { - cindex++; - rightCellIdx += _rightAddress._accumulatedSize[rdim]; - combinedCellIdx += _combinedAddress._accumulatedSize[cdim]; - curDimension++; - } else { - rightCellIdx -= _rightAddress.wholeDimStep(rdim); - combinedCellIdx -= _combinedAddress.wholeDimStep(cdim); - cindex = 0; - curDimension--; - } - } - } - } - - static eval::ValueType combineDimensions(const eval::ValueType &lhs, const eval::ValueType &rhs); -}; - - -} diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.hpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.hpp index fa1e59c87db..e71840f392c 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.hpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.hpp @@ -3,78 +3,45 @@ #pragma once #include "dense_tensor_apply.h" -#include "dense_tensor_address_combiner.h" +#include "dense_dimension_combiner.h" #include "direct_dense_tensor_builder.h" namespace vespalib::tensor::dense { template <typename Function> std::unique_ptr<Tensor> -apply(DenseTensorAddressCombiner & combiner, DirectDenseTensorBuilder & builder, - const DenseTensorView &lhs, const DenseTensorView::CellsRef & rhsCells, Function &&func) __attribute__((noinline)); +apply(DenseDimensionCombiner & combiner, DirectDenseTensorBuilder & builder, + const DenseTensorView::CellsRef & lhsCells, + const DenseTensorView::CellsRef & rhsCells, Function &&func) __attribute__((noinline)); template <typename Function> std::unique_ptr<Tensor> -apply(DenseTensorAddressCombiner & combiner, DirectDenseTensorBuilder & builder, - const DenseTensorView &lhs, const DenseTensorView::CellsRef & rhsCells, Function &&func) +apply(DenseDimensionCombiner & combiner, DirectDenseTensorBuilder & builder, + const DenseTensorView::CellsRef & lhsCells, + const DenseTensorView::CellsRef & rhsCells, Function &&func) { - for (DenseTensorCellsIterator lhsItr = lhs.cellsIterator(); lhsItr.valid(); lhsItr.next()) { - combiner.updateLeftAndCommon(lhsItr.address()); - if (combiner.updateCommon()) { - combiner.for_each_right(rhsCells, [&func, &builder, &lhsItr](size_t combined, double rhsCell) { - builder.insertCell(combined, func(lhsItr.cell(), rhsCell)); - }); + for (combiner.leftReset(); combiner.leftInRange(); combiner.stepLeft()) { + for (combiner.rightReset(); combiner.rightInRange(); combiner.stepRight()) { + for (combiner.commonReset(); combiner.commonInRange(); combiner.stepCommon()) { + size_t outIdx = combiner.outputIdx(); + size_t l = combiner.leftIdx(); + size_t r = combiner.rightIdx(); + builder.insertCell(outIdx, func(lhsCells[l], rhsCells[r])); + } } } return builder.build(); } - -template <typename Function> -std::unique_ptr<Tensor> -apply_no_rightonly_dimensions(DenseTensorAddressCombiner & combiner, DirectDenseTensorBuilder & builder, - const DenseTensorView &lhs, const DenseTensorView::CellsRef & rhsCells, - Function &&func) __attribute__((noinline)); - -template <typename Function> -std::unique_ptr<Tensor> -apply_no_rightonly_dimensions(DenseTensorAddressCombiner & combiner, DirectDenseTensorBuilder & builder, - const DenseTensorView &lhs, const DenseTensorView::CellsRef & rhsCells, Function &&func) -{ - for (DenseTensorCellsIterator lhsItr = lhs.cellsIterator(); lhsItr.valid(); lhsItr.next()) { - combiner.updateLeftAndCommon(lhsItr.address()); - if (combiner.updateCommon()) { - builder.insertCell(combiner.address(), func(lhsItr.cell(), rhsCells[combiner.rightCellIndex()])); - } - } - return builder.build(); -} - -template <typename Function> -std::unique_ptr<Tensor> -apply(const DenseTensorView &lhs, const DenseTensorView &rhs, Function &&func) -{ - eval::ValueType resultType = DenseTensorAddressCombiner::combineDimensions(lhs.fast_type(), rhs.fast_type()); - DenseTensorAddressCombiner combiner(resultType, lhs.fast_type(), rhs.fast_type()); - DirectDenseTensorBuilder builder(resultType); - if (combiner.hasAnyRightOnlyDimensions()) { - return apply(combiner, builder, lhs, rhs.cellsRef(), std::move(func)); - } else { - return apply_no_rightonly_dimensions(combiner, builder, lhs, rhs.cellsRef(), std::move(func)); - } -} - template <typename Function> std::unique_ptr<Tensor> apply(const DenseTensorView &lhs, const Tensor &rhs, Function &&func) { const DenseTensorView *view = dynamic_cast<const DenseTensorView *>(&rhs); if (view) { - return apply(lhs, *view, func); - } - const DenseTensor *dense = dynamic_cast<const DenseTensor *>(&rhs); - if (dense) { - return apply(lhs, *dense, func); + DenseDimensionCombiner combiner(lhs.fast_type(), view->fast_type()); + DirectDenseTensorBuilder builder(combiner.result_type); + return apply(combiner, builder, lhs.cellsRef(), view->cellsRef(), std::move(func)); } return Tensor::UP(); } diff --git a/metrics/CMakeLists.txt b/metrics/CMakeLists.txt index 938617e09d3..6cf1eadd6f7 100644 --- a/metrics/CMakeLists.txt +++ b/metrics/CMakeLists.txt @@ -10,9 +10,6 @@ vespa_define_module( LIBS src/vespa/metrics - TEST_EXTERNAL_DEPENDS - cppunit - TESTS src/tests ) diff --git a/metrics/src/tests/metricmanagertest.cpp b/metrics/src/tests/metricmanagertest.cpp index f260089ec84..1d954a641b6 100644 --- a/metrics/src/tests/metricmanagertest.cpp +++ b/metrics/src/tests/metricmanagertest.cpp @@ -895,8 +895,6 @@ public: void verifyDimensions(size_t metricIndex, const std::string& name, const Metric::Tags& dimensions) { - // Works to do this outside of main test body because cppunit uses - // exceptions for its failures. EXPECT_EQ(name, nthMetricName(metricIndex)) << _jsonText; EXPECT_EQ(dimensions.size(), nthMetricDimensionCount(metricIndex)) << _jsonText; for (auto& dim : dimensions) { diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index f77c11eb350..418d8dbe430 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -49,9 +49,6 @@ vespa_define_module( vdstestlib atomic - TEST_EXTERNAL_DEPENDS - cppunit - TESTS src/tests src/tests/bucketdb diff --git a/storage/src/tests/CMakeLists.txt b/storage/src/tests/CMakeLists.txt index f93ea7ef00d..e92d15c14e3 100644 --- a/storage/src/tests/CMakeLists.txt +++ b/storage/src/tests/CMakeLists.txt @@ -1,16 +1,3 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Runner for unit tests written in CppUnit (DEPRECATED). -# NOTE: All new tests should be written in gtest. Each test sub-module has a gtest runner. -vespa_add_executable(storage_testrunner_app TEST - SOURCES - testrunner.cpp - DEPENDS - storage_testcommon - storage_testhostreporter -) - -vespa_add_test( - NAME storage_testrunner_app - COMMAND storage_testrunner_app -) +# Currently empty; test executables are in library subdirs diff --git a/storage/src/tests/bucketdb/initializertest.cpp b/storage/src/tests/bucketdb/initializertest.cpp index 57bb3a865d5..63f990f7cc1 100644 --- a/storage/src/tests/bucketdb/initializertest.cpp +++ b/storage/src/tests/bucketdb/initializertest.cpp @@ -12,8 +12,8 @@ #include <vespa/storageapi/message/state.h> #include <tests/common/teststorageapp.h> #include <tests/common/dummystoragelink.h> -#include <tests/common/testhelper.h> // TODO decouple from CppUnit -#include <vespa/vdstestlib/cppunit/dirconfig.hpp> // TODO decouple from CppUnit +#include <tests/common/testhelper.h> +#include <vespa/vdstestlib/config/dirconfig.hpp> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> diff --git a/storage/src/tests/common/testhelper.h b/storage/src/tests/common/testhelper.h index cc7f503e028..8c553ccce40 100644 --- a/storage/src/tests/common/testhelper.h +++ b/storage/src/tests/common/testhelper.h @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once #include <vespa/messagebus/testlib/slobrok.h> -#include <vespa/vdstestlib/cppunit/dirconfig.h> +#include <vespa/vdstestlib/config/dirconfig.h> #include <fstream> #include <sstream> diff --git a/storage/src/tests/gtest_runner.cpp b/storage/src/tests/gtest_runner.cpp deleted file mode 100644 index 0eb65b00d49..00000000000 --- a/storage/src/tests/gtest_runner.cpp +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/gtest/gtest.h> - -#include <vespa/log/log.h> -LOG_SETUP("storage_gtest_runner"); - -GTEST_MAIN_RUN_ALL_TESTS() diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index f6b8fc3b3f0..7db47572e22 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <tests/common/testhelper.h> // FIXME +#include <tests/common/testhelper.h> #include <tests/common/dummystoragelink.h> #include <tests/common/teststorageapp.h> #include <tests/persistence/filestorage/forwardingmessagesender.h> diff --git a/storage/src/tests/storageserver/fnet_listener_test.cpp b/storage/src/tests/storageserver/fnet_listener_test.cpp index b9f2ca74df3..f82af1f8e5c 100644 --- a/storage/src/tests/storageserver/fnet_listener_test.cpp +++ b/storage/src/tests/storageserver/fnet_listener_test.cpp @@ -8,7 +8,7 @@ #include <vespa/storageapi/message/state.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vdstestlib/cppunit/dirconfig.hpp> +#include <vespa/vdstestlib/config/dirconfig.hpp> #include <vespa/messagebus/testlib/slobrok.h> #include <tests/common/testhelper.h> #include <vespa/vespalib/gtest/gtest.h> diff --git a/storage/src/tests/storageserver/service_layer_error_listener_test.cpp b/storage/src/tests/storageserver/service_layer_error_listener_test.cpp index dc5324c00e3..ec38e09a673 100644 --- a/storage/src/tests/storageserver/service_layer_error_listener_test.cpp +++ b/storage/src/tests/storageserver/service_layer_error_listener_test.cpp @@ -3,7 +3,7 @@ #include <vespa/storage/storageserver/service_layer_error_listener.h> #include <vespa/storage/storageserver/mergethrottler.h> #include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h> -#include <vespa/vdstestlib/cppunit/dirconfig.h> +#include <vespa/vdstestlib/config/dirconfig.h> #include <tests/common/testhelper.h> #include <tests/common/teststorageapp.h> #include <vespa/vespalib/gtest/gtest.h> diff --git a/storage/src/tests/testrunner.cpp b/storage/src/tests/testrunner.cpp deleted file mode 100644 index 9f871997873..00000000000 --- a/storage/src/tests/testrunner.cpp +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vdstestlib/cppunit/cppunittestrunner.h> - -#include <vespa/log/log.h> -LOG_SETUP("storagecppunittests"); - -int -main(int argc, const char *argv[]) -{ - vdstestlib::CppUnitTestRunner testRunner; - return testRunner.run(argc, argv); -} diff --git a/storageapi/src/tests/CMakeLists.txt b/storageapi/src/tests/CMakeLists.txt index ddc43c70004..8b820adb467 100644 --- a/storageapi/src/tests/CMakeLists.txt +++ b/storageapi/src/tests/CMakeLists.txt @@ -1,13 +1,12 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Runner for unit tests written in gtest. -# NOTE: All new test classes should be added here. vespa_add_executable(storageapi_gtest_runner_app TEST SOURCES gtest_runner.cpp DEPENDS storageapi_testbuckets storageapi_testmbusprot + storageapi_testmessageapi storageapi gtest ) @@ -17,17 +16,3 @@ vespa_add_test( COMMAND storageapi_gtest_runner_app ) -# Runner for unit tests written in CppUnit (DEPRECATED). -vespa_add_executable(storageapi_testrunner_app TEST - SOURCES - testrunner.cpp - DEPENDS - storageapi_testmessageapi - storageapi - vdstestlib -) - -vespa_add_test( - NAME storageapi_testrunner_app - COMMAND storageapi_testrunner_app -) diff --git a/storageapi/src/tests/testrunner.cpp b/storageapi/src/tests/testrunner.cpp deleted file mode 100644 index f21df6bdb86..00000000000 --- a/storageapi/src/tests/testrunner.cpp +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vdstestlib/cppunit/cppunittestrunner.h> - -#include <vespa/log/log.h> -LOG_SETUP("storageapicppunittestrunner"); - -int -main(int argc, const char *argv[]) -{ - vdstestlib::CppUnitTestRunner testRunner; - return testRunner.run(argc, argv); -} diff --git a/storageframework/CMakeLists.txt b/storageframework/CMakeLists.txt index 2211a4c009d..db6a14d0386 100644 --- a/storageframework/CMakeLists.txt +++ b/storageframework/CMakeLists.txt @@ -20,7 +20,6 @@ vespa_define_module( src/vespa/storageframework/generic/thread TEST_EXTERNAL_DEPENDS - cppunit vdstestlib TESTS diff --git a/storageserver/src/tests/testhelper.h b/storageserver/src/tests/testhelper.h index 7dbaaecdbf4..a0d5da20eb8 100644 --- a/storageserver/src/tests/testhelper.h +++ b/storageserver/src/tests/testhelper.h @@ -4,7 +4,7 @@ #include <fstream> #include <sstream> #include <vespa/messagebus/testlib/slobrok.h> -#include <vespa/vdstestlib/cppunit/dirconfig.h> +#include <vespa/vdstestlib/config/dirconfig.h> namespace storage { diff --git a/streamingvisitors/CMakeLists.txt b/streamingvisitors/CMakeLists.txt index 88278bbc86a..26eb77fa4ce 100644 --- a/streamingvisitors/CMakeLists.txt +++ b/streamingvisitors/CMakeLists.txt @@ -12,9 +12,6 @@ vespa_define_module( vdslib vsm - TEST_EXTERNAL_DEPENDS - cppunit - LIBS src/vespa/searchvisitor diff --git a/valgrind-suppressions.txt b/valgrind-suppressions.txt index 2587552ceff..13be6234a94 100644 --- a/valgrind-suppressions.txt +++ b/valgrind-suppressions.txt @@ -24,57 +24,6 @@ fun:(below main) } { - This is a bug in cppunit. - Memcheck:Leak - fun:_Znwm - fun:*addTestsToSuite*CppUnit*TestSuiteBuilderContextBase* -} -{ - Bug in cppunit. This suppression is created on CentOS7. - Memcheck:Leak - match-leak-kinds: definite - fun:_Znwm - fun:addTestsToSuite - fun:suite - fun:*makeTest* - fun:_ZN7CppUnit19TestFactoryRegistry14addTestToSuiteEPNS_9TestSuiteE - fun:_ZN7CppUnit19TestFactoryRegistry8makeTestEv - fun:_ZN10vdstestlib17CppUnitTestRunner3runEiPKPKc - fun:main -} -{ - Bug in cppunit. This suppression is created on CentOS7. - Memcheck:Leak - match-leak-kinds: definite - fun:_Znwm - fun:allocate - fun:_S_create - fun:_S_construct<char const*> - fun:_S_construct_aux<char const*> - fun:_S_construct<char const*> - fun:_ZNSsC1EPKcRKSaIcE - fun:_ZN7CppUnit10TestRunnerC1Ev - fun:_ZN7CppUnit14TextTestRunnerC1EPNS_9OutputterE - fun:_ZN10vdstestlib17CppUnitTestRunner3runEiPPKc - fun:main -} -{ - Bug in cppunit. This suppression is created on CentOS7. - Memcheck:Leak - match-leak-kinds: definite - fun:_Znwm - fun:allocate - fun:_S_create - fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag - fun:_S_construct_aux<char const*> - fun:_S_construct<char const*> - fun:_ZNSsC1EPKcRKSaIcE - fun:_ZN7CppUnit10TestRunnerC1Ev - fun:_ZN7CppUnit14TextTestRunnerC1EPNS_9OutputterE - fun:_ZN10vdstestlib17CppUnitTestRunner3runEiPPKc - fun:main -} -{ RHEL6 strlen is eager and will read 16 bytes blocks. Memcheck:Cond fun:__strlen_sse42 diff --git a/vdslib/CMakeLists.txt b/vdslib/CMakeLists.txt index 0480ea266e5..b997bf5f983 100644 --- a/vdslib/CMakeLists.txt +++ b/vdslib/CMakeLists.txt @@ -18,7 +18,6 @@ vespa_define_module( TEST_DEPENDS vdstestlib - cppunit TESTS src/tests diff --git a/vdstestlib/CMakeLists.txt b/vdstestlib/CMakeLists.txt index 10ee29bdcc5..8fd97fe8168 100644 --- a/vdstestlib/CMakeLists.txt +++ b/vdstestlib/CMakeLists.txt @@ -5,10 +5,9 @@ vespa_define_module( vespalib TESTS - src/tests/cppunit src/tests/dirconfig LIBS src/vespa/vdstestlib - src/vespa/vdstestlib/cppunit + src/vespa/vdstestlib/config ) diff --git a/vdstestlib/src/tests/cppunit/.gitignore b/vdstestlib/src/tests/cppunit/.gitignore deleted file mode 100644 index ec991c2f0cd..00000000000 --- a/vdstestlib/src/tests/cppunit/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -testrunner -vdstestlib_testrunner_app diff --git a/vdstestlib/src/tests/cppunit/CMakeLists.txt b/vdstestlib/src/tests/cppunit/CMakeLists.txt deleted file mode 100644 index 1b8e857fe30..00000000000 --- a/vdstestlib/src/tests/cppunit/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vdstestlib_testrunner_app TEST - SOURCES - testrunner.cpp - cppunittest.cpp - DEPENDS - vdstestlib -) - -vespa_add_test( - NAME vdstestlib_testrunner_app - NO_VALGRIND - COMMAND vdstestlib_testrunner_app -) diff --git a/vdstestlib/src/tests/cppunit/cppunittest.cpp b/vdstestlib/src/tests/cppunit/cppunittest.cpp deleted file mode 100644 index bf8b38e696e..00000000000 --- a/vdstestlib/src/tests/cppunit/cppunittest.cpp +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vdstestlib/cppunit/macros.h> - -namespace vespalib { - -struct CppunitTest : public CppUnit::TestFixture { - - void testSomething(); - - CPPUNIT_TEST_SUITE(CppunitTest); - CPPUNIT_TEST(testSomething); - CPPUNIT_TEST_SUITE_END(); - -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(CppunitTest); - -void -CppunitTest::testSomething() -{ - CPPUNIT_ASSERT_EQUAL_MESSAGE("hmm", "foo", "foo"); -} - -} // vespalib diff --git a/vdstestlib/src/tests/cppunit/testrunner.cpp b/vdstestlib/src/tests/cppunit/testrunner.cpp deleted file mode 100644 index 3b686d132bb..00000000000 --- a/vdstestlib/src/tests/cppunit/testrunner.cpp +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vdstestlib/cppunit/cppunittestrunner.h> - -int main(int argc, const char *argv[]) -{ - vdstestlib::CppUnitTestRunner testRunner; - return testRunner.run(argc, argv); -} - diff --git a/vdstestlib/src/tests/dirconfig/dirconfigtest.cpp b/vdstestlib/src/tests/dirconfig/dirconfigtest.cpp index 9d8e5c47143..f985150e497 100644 --- a/vdstestlib/src/tests/dirconfig/dirconfigtest.cpp +++ b/vdstestlib/src/tests/dirconfig/dirconfigtest.cpp @@ -2,7 +2,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/exceptions.h> -#include <vespa/vdstestlib/cppunit/dirconfig.h> +#include <vespa/vdstestlib/config/dirconfig.h> #include <fstream> #include <iostream> diff --git a/vdstestlib/src/vespa/vdstestlib/CMakeLists.txt b/vdstestlib/src/vespa/vdstestlib/CMakeLists.txt index ce80f8dd644..86ca3461302 100644 --- a/vdstestlib/src/vespa/vdstestlib/CMakeLists.txt +++ b/vdstestlib/src/vespa/vdstestlib/CMakeLists.txt @@ -1,8 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(vdstestlib SOURCES - $<TARGET_OBJECTS:vdstestlib_vdstestlib_cppunit> + $<TARGET_OBJECTS:vdstestlib_vdstestlib_config> INSTALL lib64 - DEPENDS - cppunit ) diff --git a/vdstestlib/src/vespa/vdstestlib/cppunit/.gitignore b/vdstestlib/src/vespa/vdstestlib/config/.gitignore index 583460ae288..583460ae288 100644 --- a/vdstestlib/src/vespa/vdstestlib/cppunit/.gitignore +++ b/vdstestlib/src/vespa/vdstestlib/config/.gitignore diff --git a/vdstestlib/src/vespa/vdstestlib/cppunit/CMakeLists.txt b/vdstestlib/src/vespa/vdstestlib/config/CMakeLists.txt index e7d8705b07d..90b7c95fe50 100644 --- a/vdstestlib/src/vespa/vdstestlib/cppunit/CMakeLists.txt +++ b/vdstestlib/src/vespa/vdstestlib/config/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(vdstestlib_vdstestlib_cppunit OBJECT +vespa_add_library(vdstestlib_vdstestlib_config OBJECT SOURCES - cppunittestrunner.cpp dirconfig.cpp DEPENDS ) diff --git a/vdstestlib/src/vespa/vdstestlib/cppunit/dirconfig.cpp b/vdstestlib/src/vespa/vdstestlib/config/dirconfig.cpp index 3a26ed9dec8..3a26ed9dec8 100644 --- a/vdstestlib/src/vespa/vdstestlib/cppunit/dirconfig.cpp +++ b/vdstestlib/src/vespa/vdstestlib/config/dirconfig.cpp diff --git a/vdstestlib/src/vespa/vdstestlib/cppunit/dirconfig.h b/vdstestlib/src/vespa/vdstestlib/config/dirconfig.h index 388348c880e..c0a1571f52c 100644 --- a/vdstestlib/src/vespa/vdstestlib/cppunit/dirconfig.h +++ b/vdstestlib/src/vespa/vdstestlib/config/dirconfig.h @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** * \class vdstestlib::DirConfig - * \ingroup cppunit + * \ingroup config * * \brief Helper class for generating dir config * diff --git a/vdstestlib/src/vespa/vdstestlib/cppunit/dirconfig.hpp b/vdstestlib/src/vespa/vdstestlib/config/dirconfig.hpp index f5d886c089a..5d127879e34 100644 --- a/vdstestlib/src/vespa/vdstestlib/cppunit/dirconfig.hpp +++ b/vdstestlib/src/vespa/vdstestlib/config/dirconfig.hpp @@ -2,7 +2,7 @@ #pragma once -#include <vespa/vdstestlib/cppunit/dirconfig.h> +#include <vespa/vdstestlib/config/dirconfig.h> #include <sstream> #include <boost/lexical_cast.hpp> diff --git a/vdstestlib/src/vespa/vdstestlib/cppunit/cppunittestrunner.cpp b/vdstestlib/src/vespa/vdstestlib/cppunit/cppunittestrunner.cpp deleted file mode 100644 index dbedf1ee900..00000000000 --- a/vdstestlib/src/vespa/vdstestlib/cppunit/cppunittestrunner.cpp +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vdstestlib/cppunit/cppunittestrunner.h> - -#include <cppunit/extensions/TestFactoryRegistry.h> -#include <cppunit/ui/text/TestRunner.h> -#include <cppunit/TextTestProgressListener.h> -#include <vespa/log/log.h> -#include <iostream> - -LOG_SETUP(".cppunittestrunner"); - -using CppUnit::Test; -using CppUnit::TestSuite; - -namespace vdstestlib { - -namespace { - struct WantedTestList : public CppUnit::Test::Filter { - std::vector<std::string> _wanted; - bool _includeStressTests; - - WantedTestList(int argc, const char *argv[], - bool includeStressTests) - : _wanted(), - _includeStressTests(includeStressTests) - { - for (int i=1; i<argc; ++i) { - if (argv[i][0] != '-') { - std::cerr << "Running tests matching '*" - << argv[i] << "*'.\n"; - _wanted.push_back(argv[i]); - } - } - char* testpat = getenv("TEST_SUBSET"); - if (testpat != 0) { - std::string pat = std::string("*") + testpat; - if (pat[pat.size() - 1] != '$') pat += "*"; - std::cerr << "Running tests matching '" << pat << "'." - " (Taken from TEST_SUBSET environment variable)\n"; - _wanted.push_back(testpat); - } - if (CppUnit::Test::disabledCount > 0) { - std::cerr << CppUnit::Test::disabledCount - << " tests are currently disabled and won't be " - << "attempted run.\n"; - } - if (CppUnit::Test::ignoredCount > 0) { - std::cerr << CppUnit::Test::ignoredCount - << " tests are currently set to ignore failures.\n"; - } - } - - std::string getWantedString(uint32_t index) const { - std::string s = _wanted[index]; - if (s[s.size() - 1] == '$') { - return s.substr(0, s.size() - 1); - } - return s; - } - - bool requiresTailMatch(uint32_t index) const { - std::string s = _wanted[index]; - return (s[s.size() - 1] == '$'); - } - - bool include(const std::string& name) const override { - if ((name.find("stress") != std::string::npos || - name.find("Stress") != std::string::npos) - && !_includeStressTests) - { - std::cerr << "Excluding stress test " << name << "\n"; - } else { - if (_wanted.size() == 0) return true; - for (uint32_t i=0; i<_wanted.size(); ++i) { - std::string::size_type pos = name.rfind(getWantedString(i)); - if (pos == std::string::npos) continue; - if (!requiresTailMatch(i) - || pos == name.size() - getWantedString(i).size()) - { - return true; - } - } - } - return false; - } - }; - - struct LogHook : public CppUnit::TextTestProgressListener::TestStartHook { - std::string lastTest; - void startedTest(const std::string& testName) override { - LOG(info, "Starting test: %s", testName.c_str()); - lastTest = testName; - } - void stoppedTest() override { - LOG(info, "Stopped test: %s", lastTest.c_str()); - } - }; -} - -void CppUnitTestRunner::listTests(const TestSuite *tests) { - for (const auto & test : tests->getTests()) { - std::cout << test->getName() << std::endl; - } -} - -CppUnitTestRunner::CppUnitTestRunner() -{ - std::ios::sync_with_stdio(); -} - -int -CppUnitTestRunner::run(int argc, const char *argv[]) -{ - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry( - CppUnit::TestFactoryRegistry::getRegistry()); - - Test *tests = registry.makeTest(); - TestSuite *suite = dynamic_cast<TestSuite *>(tests); - - bool includeStressTests = false; - bool logStartStop = false; - bool verbose = false; - if (getenv("TEST_VERBOSE") != 0) { - verbose = true; - } - - for (int i=1; i<argc; ++i) { - std::string arg(argv[i]); - if (arg == "--verbose") { - verbose = true; - logStartStop = true; - } else if (arg == "--includestress") { - includeStressTests = true; - } else if (arg == "--list") { - listTests(suite); - exit(0); - } else if (argv[i][0] == '-') { - std::cerr << "Illegal option " << arg << "\n"; - exit(1); - } else { - // Arguments will be passed as patterns - } - } - - WantedTestList wantedList(argc, argv, includeStressTests); - suite->filter(wantedList); - runner.addTest(tests); - CppUnit::TextTestProgressListener::verboseProgress = verbose; - if (logStartStop) { - CppUnit::TextTestProgressListener::startHook.reset(new LogHook); - } - return (runner.run("", false) ? 0 : -1); -} - -} // vdstestlib diff --git a/vdstestlib/src/vespa/vdstestlib/cppunit/cppunittestrunner.h b/vdstestlib/src/vespa/vdstestlib/cppunit/cppunittestrunner.h deleted file mode 100644 index 057f116a7d2..00000000000 --- a/vdstestlib/src/vespa/vdstestlib/cppunit/cppunittestrunner.h +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class CppUnitTestRunner - * @ingroup cppunit - * - * @brief Application for running cppunit tests. - * - * This is an application to use when running cppunit tests, currently used - * in document,vdslib,storageapi and storage. - * - * It is built like a library, as one need to create one per project, but the - * cppunit test application file in each project can now only contain a little - * main method, creating an instance of this class and calling run. - * - * See storage/src/cpp/tests/testrunner.h for an example of simple app using - * this. - * - * When using this test binary you have the following options. - * - * If the TEST_SUBSET environment variable is set, only tests matching the - * pattern given in the environment is run. For instance, by doing - * TEST_SUBSET=foo ./testrunner, only tests that match the regular expression - * .*foo.* will be run. Optionally you can postfix your expression with a - * dollar, to only run tests that end with the given string. You can match - * against any part of the function shown in verbose mode. For instance - * TEST_SUBSET=foo::bar$ will run tests whose test class ends in foo, and - * having test name bar. - * - * You can specify --verbose mode. In verbose mode, each test name is printed - * to stdout when started, and after completion, the runtime of the test is - * shown. This is very useful to identify slow unit tests which should be - * improved, and also to see in what test the system might be hung up in. In - * addition, in verbose mode, a vespa log entry is given at the start and end - * of each test, such that one can identify which parts of the vespa log belongs - * to each test, in case you are redirecting the log to a file. - * - * You can also use the --includestress option to also include tests that match - * the regular expression '.*[sS]tress.*'. These are excluded by default, such - * that regular test runs can be quick. - */ - -#pragma once - -#include <cppunit/TestSuite.h> - -namespace vdstestlib { - -class CppUnitTestRunner { -public: - CppUnitTestRunner(); - - void listTests(const CppUnit::TestSuite *tests); - int run(int argc, const char * argv[]); - -}; - -} // vdstestlib - diff --git a/vdstestlib/src/vespa/vdstestlib/cppunit/macros.h b/vdstestlib/src/vespa/vdstestlib/cppunit/macros.h deleted file mode 100644 index fdde94bbbd6..00000000000 --- a/vdstestlib/src/vespa/vdstestlib/cppunit/macros.h +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * This file contains additional CPPUNIT macros to simplify tests. - */ -#pragma once -#include <cppunit/extensions/HelperMacros.h> -#include <vespa/vespalib/test/insertion_operators.h> - - -// Wrapper for CPPUNIT_ASSERT_EQUAL_MESSAGE to prevent it from evaluating -// message if val1 is equal to val2 -#define CPPUNIT_ASSERT_EQUAL_MSG(message, val1, val2) \ - { \ - if (!((val1) == (val2))) { \ - CPPUNIT_ASSERT_EQUAL_MESSAGE(message, val1, val2); \ - } \ - } - -#define CPPUNIT_ASSERT_EQUAL_ESCAPED(val1, val2) \ - { \ - if (!((val1) == (val2))) { \ - std::ostringstream out1; \ - std::ostringstream out2; \ - out1 << "[" << val1 << "]"; \ - out2 << "[" << val2 << "]"; \ - CPPUNIT_ASSERT_EQUAL( \ - document::StringUtil::escape(out1.str()), \ - document::StringUtil::escape(out2.str())); \ - } \ - } - -// Wrapper for CPPUNIT_ASSERT_MESSAGE to prevent it from evaluating message if -// val is true -#define CPPUNIT_ASSERT_MSG(message, val) \ - { \ - if (!(val)) { \ - CPPUNIT_ASSERT_MESSAGE(message, val); \ - } \ - } - -// Assert that value starts with prefix -#define CPPUNIT_ASSERT_PREFIX(prefix, value) \ - { \ - std::ostringstream pre; \ - pre << prefix; \ - std::ostringstream val; \ - val << value; \ - if (val.str().find(pre.str()) != 0) { \ - CPPUNIT_FAIL("Value of '" + val.str() + "' does not contain " \ - "prefix '" + pre.str() + "'."); \ - } \ - } - -// Assert that value contains given substring -#define CPPUNIT_ASSERT_CONTAIN(contained, value) \ - { \ - std::ostringstream cont; \ - cont << contained; \ - std::ostringstream val; \ - val << value; \ - if (val.str().find(cont.str()) == std::string::npos) { \ - CPPUNIT_FAIL("Value of '" + val.str() + "' does not contain '" \ - + cont.str() + "'."); \ - } \ - } - -// Assert that value contains given substring, add message to output on error -#define CPPUNIT_ASSERT_CONTAIN_MESSAGE(message, contained, value) \ - { \ - std::ostringstream cont; \ - cont << contained; \ - std::ostringstream val; \ - val << value; \ - std::string mess = message; \ - if (val.str().find(cont.str()) == std::string::npos) { \ - CPPUNIT_FAIL(mess + ": Value of '" + val.str() \ - + "' does not contain '" + cont.str() + "'."); \ - } \ - } - -// Assert that given expression matches the given regular expression. -#include <vespa/vespalib/util/regexp.h> -#define CPPUNIT_ASSERT_MATCH_REGEX(expression, value) \ - { \ - std::ostringstream _ost_; \ - _ost_ << value; \ - std::string _s_(_ost_.str()); \ - vespalib::Regexp _myregex_(expression); \ - if (!_myregex_.match(_s_)) { \ - CPPUNIT_FAIL("Value of '" + _s_ + "' does not match regex '" \ - + expression + "'."); \ - } \ - } - -// Assert that given expression matches the given regular expression. -#include <vespa/vespalib/util/regexp.h> -#define CPPUNIT_ASSERT_MATCH_REGEX_MSG(message, expression, value) \ - { \ - std::ostringstream _ost_; \ - _ost_ << value; \ - std::string _s_(_ost_.str()); \ - vespalib::Regexp _myregex_(expression); \ - std::string mess = message; \ - if (!_myregex_.match(_s_)) { \ - CPPUNIT_FAIL("Value of '" + _s_ + "' does not match regex '" \ - + expression + "'. Message: '" + mess + "'"); \ - } \ - } - -#define CPPUNIT_ASSERT_FILE_CONTAINS(expected, filename) \ - { \ - std::ostringstream value; \ - value << expected; \ - std::ostringstream ost; \ - std::string line; \ - std::ifstream input(filename); \ - while (std::getline(input, line, '\n')) { \ - ost << line << '\n'; \ - } \ - CPPUNIT_ASSERT_EQUAL(value.str(), ost.str()); \ - } - -#define CPPUNIT_ASSERT_SUBSTRING_COUNT(source, expectedCount, substring) \ - { \ - uint32_t count = 0; \ - std::ostringstream value; /* Let value be non-strings */ \ - value << source; \ - std::string s(value.str()); \ - std::string::size_type pos = s.find(substring); \ - while (pos != std::string::npos) { \ - ++count; \ - pos = s.find(substring, pos+1); \ - } \ - if (count != (uint32_t) expectedCount) { \ - std::ostringstream error; \ - error << "Value of '" << s << "' contained " << count \ - << " instances of substring '" << substring << "', not " \ - << expectedCount << " as expected."; \ - CPPUNIT_FAIL(error.str()); \ - } \ - } - -#include <ostream> -#include <map> -#include <unordered_map> -#include <vector> - -// Create output operator for containers. -// Needed so we can use CPPUNIT_ASSERT_EQUAL with them. - -template<typename S, typename T> -std::ostream& -operator<<(std::ostream& out, const std::unordered_map<S, T>& umap) -{ - out << "std::unordered_map(" << umap.size() << ") {"; - for (auto keyValue : umap) { - out << "\n " << keyValue.first << ": " << keyValue.second; - } - if (!umap.empty()) { - out << "\n"; - } - out << "}"; - return out; -} diff --git a/vespalib/src/tests/testkit-testhook/CMakeLists.txt b/vespalib/src/tests/testkit-testhook/CMakeLists.txt index c4662adc8a8..4febe6ebb75 100644 --- a/vespalib/src/tests/testkit-testhook/CMakeLists.txt +++ b/vespalib/src/tests/testkit-testhook/CMakeLists.txt @@ -4,6 +4,5 @@ vespa_add_executable(vespalib_testkit-testhook_test_app TEST testkit-testhook_test.cpp DEPENDS vespalib - cppunit ) vespa_add_test(NAME vespalib_testkit-testhook_test_app COMMAND vespalib_testkit-testhook_test_app) diff --git a/vespalib/src/tests/testkit-testhook/testkit-testhook_test.cpp b/vespalib/src/tests/testkit-testhook/testkit-testhook_test.cpp index 164f4cd7247..df7916dd56d 100644 --- a/vespalib/src/tests/testkit-testhook/testkit-testhook_test.cpp +++ b/vespalib/src/tests/testkit-testhook/testkit-testhook_test.cpp @@ -1,7 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> - -#include <cppunit/extensions/HelperMacros.h> #include <stdexcept> //----------------------------------------------------------------------------- @@ -150,12 +148,7 @@ IGNORE_TEST("passed tests can also be ignored") { //----------------------------------------------------------------------------- -TEST("cppunit unwind will result in 1 failed test and 1 failed check") { - CPPUNIT_ASSERT_EQUAL_MESSAGE("cppunit happy", 1, 1); - CPPUNIT_ASSERT_EQUAL_MESSAGE("cppunit not happy", 1, 2); -} - -TEST("std::excpetion unwind will result in 1 failed test and 1 failed check") { +TEST("std::exception unwind will result in 1 failed test and 1 failed check") { throw std::runtime_error("something failed"); } @@ -172,7 +165,7 @@ TEST("verify and ignore check failures from previous tests") { TEST("verify that all appropriate tests have been executed") { TEST_FLUSH(); - EXPECT_EQUAL(25u, TEST_MASTER.getProgress().passCnt); + EXPECT_EQUAL(24u, TEST_MASTER.getProgress().passCnt); } //----------------------------------------------------------------------------- |