diff options
author | Geir Storli <geirst@yahoo-inc.com> | 2016-09-15 09:56:34 +0200 |
---|---|---|
committer | Geir Storli <geirst@yahoo-inc.com> | 2016-09-15 09:56:34 +0200 |
commit | fe36281238d229f859f8ba7a414aeac53889eaeb (patch) | |
tree | 53cc4a0d74e80cb39495da4eca7b342b2ebbe6e2 | |
parent | 0d15262410021732ea150f057cf3213751db4d98 (diff) |
Remove tensor::TensorType (replaced by eval::ValueType).
17 files changed, 2 insertions, 864 deletions
diff --git a/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h b/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h index ebee17a9230..641f533980f 100644 --- a/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h +++ b/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h @@ -7,7 +7,6 @@ #include <vespa/vespalib/eval/value.h> #include <vespa/vespalib/eval/value_type.h> #include <vespa/vespalib/tensor/default_tensor_engine.h> -#include <vespa/vespalib/tensor/tensor_type.h> #include <memory> namespace search { @@ -40,9 +39,6 @@ public: static fef::FeatureExecutor::LP createEmpty() { return createEmpty(vespalib::eval::ValueType::double_type()); } - static fef::FeatureExecutor::LP createEmpty(const vespalib::tensor::TensorType &tensorType) { - return createEmpty(tensorType.as_value_type()); - } }; } // namespace features diff --git a/searchlib/src/vespa/searchlib/features/queryfeature.cpp b/searchlib/src/vespa/searchlib/features/queryfeature.cpp index c150f158d79..cf45f8112b9 100644 --- a/searchlib/src/vespa/searchlib/features/queryfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/queryfeature.cpp @@ -9,7 +9,6 @@ LOG_SETUP(".features.queryfeature"); #include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/properties.h> -#include <vespa/vespalib/tensor/tensor_type.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/tensor/default_tensor.h> #include <vespa/vespalib/tensor/tensor_mapper.h> diff --git a/searchlib/src/vespa/searchlib/features/queryfeature.h b/searchlib/src/vespa/searchlib/features/queryfeature.h index b6dd4e77308..4d09196f31b 100644 --- a/searchlib/src/vespa/searchlib/features/queryfeature.h +++ b/searchlib/src/vespa/searchlib/features/queryfeature.h @@ -3,7 +3,7 @@ #pragma once #include <vespa/searchlib/fef/blueprint.h> -#include <vespa/vespalib/tensor/tensor_type.h> +#include <vespa/vespalib/eval/value_type.h> namespace search { namespace features { diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index cfd8cc87d90..ec43cee3498 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -88,7 +88,6 @@ vespa_define_module( src/tests/tensor/tensor_performance src/tests/tensor/tensor_serialization src/tests/tensor/tensor_slime_serialization - src/tests/tensor/tensor_type src/tests/testapp-debug src/tests/testapp-generic src/tests/testapp-main diff --git a/vespalib/src/testlist.txt b/vespalib/src/testlist.txt index bd5c2e0d735..5267b19cdfd 100644 --- a/vespalib/src/testlist.txt +++ b/vespalib/src/testlist.txt @@ -77,7 +77,6 @@ tests/tensor/tensor_operations tests/tensor/tensor_performance tests/tensor/tensor_serialization tests/tensor/tensor_slime_serialization -tests/tensor/tensor_type tests/testapp-debug tests/testapp-generic tests/testapp-main diff --git a/vespalib/src/tests/tensor/dense_tensor_operations/dense_tensor_operations_test.cpp b/vespalib/src/tests/tensor/dense_tensor_operations/dense_tensor_operations_test.cpp index f64043e8988..a380a0861c8 100644 --- a/vespalib/src/tests/tensor/dense_tensor_operations/dense_tensor_operations_test.cpp +++ b/vespalib/src/tests/tensor/dense_tensor_operations/dense_tensor_operations_test.cpp @@ -6,7 +6,6 @@ #include <vespa/vespalib/tensor/types.h> #include <vespa/vespalib/tensor/tensor_function.h> #include <vespa/vespalib/tensor/tensor_visitor.h> -#include <vespa/vespalib/tensor/tensor_type.h> #include <iostream> using namespace vespalib::tensor; diff --git a/vespalib/src/tests/tensor/tensor_function/tensor_function_test.cpp b/vespalib/src/tests/tensor/tensor_function/tensor_function_test.cpp index 6c38b18eaa6..d96ab46bc18 100644 --- a/vespalib/src/tests/tensor/tensor_function/tensor_function_test.cpp +++ b/vespalib/src/tests/tensor/tensor_function/tensor_function_test.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/tensor/tensor_type.h> #include <vespa/vespalib/tensor/tensor_function.h> using namespace vespalib::tensor; diff --git a/vespalib/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp b/vespalib/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp index 21e964b95e1..a9153ded096 100644 --- a/vespalib/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp +++ b/vespalib/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/tensor/tensor_type.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/tensor/simple/simple_tensor.h> #include <vespa/vespalib/tensor/simple/simple_tensor_builder.h> diff --git a/vespalib/src/tests/tensor/tensor_type/.gitignore b/vespalib/src/tests/tensor/tensor_type/.gitignore deleted file mode 100644 index fca663cd6c4..00000000000 --- a/vespalib/src/tests/tensor/tensor_type/.gitignore +++ /dev/null @@ -1 +0,0 @@ -vespalib_tensor_type_test_app diff --git a/vespalib/src/tests/tensor/tensor_type/CMakeLists.txt b/vespalib/src/tests/tensor/tensor_type/CMakeLists.txt deleted file mode 100644 index e09801f3c79..00000000000 --- a/vespalib/src/tests/tensor/tensor_type/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vespalib_tensor_type_test_app TEST - SOURCES - tensor_type_test.cpp - DEPENDS - vespalib - vespalib_vespalib_tensor -) -vespa_add_test(NAME vespalib_tensor_type_test_app COMMAND vespalib_tensor_type_test_app) diff --git a/vespalib/src/tests/tensor/tensor_type/tensor_type_test.cpp b/vespalib/src/tests/tensor/tensor_type/tensor_type_test.cpp deleted file mode 100644 index 1cb32d34149..00000000000 --- a/vespalib/src/tests/tensor/tensor_type/tensor_type_test.cpp +++ /dev/null @@ -1,312 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/eval/value_type.h> -#include <vespa/vespalib/tensor/tensor_type.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <ostream> - -using namespace vespalib::tensor; -using vespalib::eval::ValueType; - -TEST("require that INVALID tensor type can be created") { - TensorType t = TensorType::invalid(); - EXPECT_TRUE(t.type() == TensorType::Type::INVALID); - EXPECT_EQUAL(t.dimensions().size(), 0u); -} - -TEST("require that NUMBER tensor type can be created") { - TensorType t = TensorType::number(); - EXPECT_TRUE(t.type() == TensorType::Type::NUMBER); - EXPECT_EQUAL(t.dimensions().size(), 0u); -} - -TEST("require that SPARSE tensor type can be created") { - TensorType t = TensorType::sparse({"x", "y"}); - EXPECT_TRUE(t.type() == TensorType::Type::SPARSE); - ASSERT_EQUAL(t.dimensions().size(), 2u); - EXPECT_EQUAL(t.dimensions()[0].name, "x"); - EXPECT_EQUAL(t.dimensions()[1].name, "y"); -} - -TEST("require that SPARSE tensor type sorts dimensions") { - TensorType t = TensorType::sparse({"x", "z", "y"}); - EXPECT_TRUE(t.type() == TensorType::Type::SPARSE); - ASSERT_EQUAL(t.dimensions().size(), 3u); - EXPECT_EQUAL(t.dimensions()[0].name, "x"); - EXPECT_EQUAL(t.dimensions()[1].name, "y"); - EXPECT_EQUAL(t.dimensions()[2].name, "z"); -} - -TEST("require that SPARSE tensor type use npos for dimension size") { - TensorType t = TensorType::sparse({"x", "y"}); - EXPECT_TRUE(t.type() == TensorType::Type::SPARSE); - ASSERT_EQUAL(t.dimensions().size(), 2u); - EXPECT_EQUAL(t.dimensions()[0].name, "x"); - EXPECT_EQUAL(t.dimensions()[0].size, TensorType::Dimension::npos); - EXPECT_EQUAL(t.dimensions()[1].name, "y"); - EXPECT_EQUAL(t.dimensions()[1].size, TensorType::Dimension::npos); -} - -TEST("require that DENSE tensor type can be created") { - TensorType t = TensorType::dense({{"x", 10}, {"y", 20}}); - EXPECT_TRUE(t.type() == TensorType::Type::DENSE); - ASSERT_EQUAL(t.dimensions().size(), 2u); - EXPECT_EQUAL(t.dimensions()[0].name, "x"); - EXPECT_EQUAL(t.dimensions()[0].size, 10u); - EXPECT_EQUAL(t.dimensions()[1].name, "y"); - EXPECT_EQUAL(t.dimensions()[1].size, 20u); -} - -TEST("require that DENSE tensor type sorts dimensions") { - TensorType t = TensorType::dense({{"x", 10}, {"z", 30}, {"y", 20}}); - EXPECT_TRUE(t.type() == TensorType::Type::DENSE); - ASSERT_EQUAL(t.dimensions().size(), 3u); - EXPECT_EQUAL(t.dimensions()[0].name, "x"); - EXPECT_EQUAL(t.dimensions()[0].size, 10u); - EXPECT_EQUAL(t.dimensions()[1].name, "y"); - EXPECT_EQUAL(t.dimensions()[1].size, 20u); - EXPECT_EQUAL(t.dimensions()[2].name, "z"); - EXPECT_EQUAL(t.dimensions()[2].size, 30u); -} - -void verify_equal(const TensorType &a, const TensorType &b) { - EXPECT_TRUE(a == b); - EXPECT_TRUE(b == a); - EXPECT_FALSE(a != b); - EXPECT_FALSE(b != a); -} - -void verify_not_equal(const TensorType &a, const TensorType &b) { - EXPECT_TRUE(a != b); - EXPECT_TRUE(b != a); - EXPECT_FALSE(a == b); - EXPECT_FALSE(b == a); -} - -TEST("require that valid tensor types can be compared") { - TEST_DO(verify_equal(TensorType::number(), TensorType::number())); - TEST_DO(verify_not_equal(TensorType::number(), TensorType::sparse({}))); - TEST_DO(verify_not_equal(TensorType::number(), TensorType::dense({}))); - TEST_DO(verify_equal(TensorType::sparse({"x", "y"}), TensorType::sparse({"y", "x"}))); - TEST_DO(verify_not_equal(TensorType::sparse({"x", "y"}), TensorType::sparse({"x", "y", "z"}))); - TEST_DO(verify_not_equal(TensorType::sparse({}), TensorType::dense({}))); - TEST_DO(verify_equal(TensorType::dense({{"x", 10}, {"y", 20}}), TensorType::dense({{"y", 20}, {"x", 10}}))); - TEST_DO(verify_not_equal(TensorType::dense({{"x", 10}, {"y", 20}}), TensorType::dense({{"x", 10}, {"y", 10}}))); -} - -TEST("require that INVALID tensor type is not equal to any type") { - TEST_DO(verify_not_equal(TensorType::invalid(), TensorType::invalid())); - TEST_DO(verify_not_equal(TensorType::invalid(), TensorType::number())); - TEST_DO(verify_not_equal(TensorType::invalid(), TensorType::sparse({}))); - TEST_DO(verify_not_equal(TensorType::invalid(), TensorType::dense({}))); -} - -void verify_predicates(const TensorType &type, bool expect_valid, bool expect_number, bool expect_tensor) { - EXPECT_EQUAL(type.is_valid(), expect_valid); - EXPECT_EQUAL(type.is_number(), expect_number); - EXPECT_EQUAL(type.is_tensor(), expect_tensor); -} - -TEST("require that type-related predicate functions work as expected") { - TEST_DO(verify_predicates(TensorType::invalid(), false, false, false)); - TEST_DO(verify_predicates(TensorType::number(), true, true, false)); - TEST_DO(verify_predicates(TensorType::sparse({}), true, false, true)); - TEST_DO(verify_predicates(TensorType::dense({}), true, false, true)); -} - -TEST("require that duplicate dimension names result in invalid types") { - EXPECT_TRUE(!TensorType::sparse({"x", "x"}).is_valid()); - EXPECT_TRUE(!TensorType::dense({{"x", 10}, {"x", 10}}).is_valid()); - EXPECT_TRUE(!TensorType::dense({{"x", 10}, {"x", 20}}).is_valid()); -} - -TEST("require that removing dimensions from non-tensor types gives invalid type") { - EXPECT_TRUE(!TensorType::invalid().remove_dimensions({"x"}).is_valid()); - EXPECT_TRUE(!TensorType::number().remove_dimensions({"x"}).is_valid()); - EXPECT_TRUE(!TensorType::invalid().remove_dimensions({}).is_valid()); - EXPECT_TRUE(!TensorType::number().remove_dimensions({}).is_valid()); -} - -TEST("require that dimensions can be removed from sparse tensor types") { - TensorType type = TensorType::sparse({"x", "y", "z"}); - EXPECT_EQUAL(TensorType::sparse({"y", "z"}), type.remove_dimensions({"x"})); - EXPECT_EQUAL(TensorType::sparse({"x", "z"}), type.remove_dimensions({"y"})); - EXPECT_EQUAL(TensorType::sparse({"x", "y"}), type.remove_dimensions({"z"})); - EXPECT_EQUAL(TensorType::sparse({"y"}), type.remove_dimensions({"x", "z"})); - EXPECT_EQUAL(TensorType::sparse({"y"}), type.remove_dimensions({"z", "x"})); -} - -TEST("require that dimensions can be removed from dense tensor types") { - TensorType type = TensorType::dense({{"x", 10}, {"y", 20}, {"z", 30}}); - EXPECT_EQUAL(TensorType::dense({{"y", 20}, {"z", 30}}), type.remove_dimensions({"x"})); - EXPECT_EQUAL(TensorType::dense({{"x", 10}, {"z", 30}}), type.remove_dimensions({"y"})); - EXPECT_EQUAL(TensorType::dense({{"x", 10}, {"y", 20}}), type.remove_dimensions({"z"})); - EXPECT_EQUAL(TensorType::dense({{"y", 20}}), type.remove_dimensions({"x", "z"})); - EXPECT_EQUAL(TensorType::dense({{"y", 20}}), type.remove_dimensions({"z", "x"})); -} - -TEST("require that removing non-existing dimensions gives invalid type") { - EXPECT_TRUE(!TensorType::sparse({"y"}).remove_dimensions({"x"}).is_valid()); - EXPECT_TRUE(!TensorType::dense({{"y", 10}}).remove_dimensions({"x"}).is_valid()); -} - -TEST("require that dimensions can be combined for sparse tensor types") { - TensorType sparse = TensorType::sparse({}); - TensorType sparse_xy = TensorType::sparse({"x", "y"}); - TensorType sparse_yz = TensorType::sparse({"y", "z"}); - TensorType sparse_xyz = TensorType::sparse({"x", "y", "z"}); - TensorType sparse_y = TensorType::sparse({"y"}); - EXPECT_EQUAL(sparse_xy.add_dimensions_from(sparse_yz), sparse_xyz); - EXPECT_EQUAL(sparse_yz.add_dimensions_from(sparse_xy), sparse_xyz); - EXPECT_EQUAL(sparse_xy.keep_dimensions_in(sparse_yz), sparse_y); - EXPECT_EQUAL(sparse_yz.keep_dimensions_in(sparse_xy), sparse_y); - EXPECT_EQUAL(sparse_y.add_dimensions_from(sparse_y), sparse_y); - EXPECT_EQUAL(sparse_y.keep_dimensions_in(sparse_y), sparse_y); - EXPECT_EQUAL(sparse.add_dimensions_from(sparse), sparse); - EXPECT_EQUAL(sparse.keep_dimensions_in(sparse), sparse); -} - -TEST("require that dimensions can be combined for dense tensor types") { - TensorType dense = TensorType::dense({}); - TensorType dense_xy = TensorType::dense({{"x", 10}, {"y", 10}}); - TensorType dense_yz = TensorType::dense({{"y", 10}, {"z", 10}}); - TensorType dense_xyz = TensorType::dense({{"x", 10}, {"y", 10}, {"z", 10}}); - TensorType dense_y = TensorType::dense({{"y", 10}}); - EXPECT_EQUAL(dense_xy.add_dimensions_from(dense_yz), dense_xyz); - EXPECT_EQUAL(dense_yz.add_dimensions_from(dense_xy), dense_xyz); - EXPECT_EQUAL(dense_xy.keep_dimensions_in(dense_yz), dense_y); - EXPECT_EQUAL(dense_yz.keep_dimensions_in(dense_xy), dense_y); - EXPECT_EQUAL(dense_y.add_dimensions_from(dense_y), dense_y); - EXPECT_EQUAL(dense_y.keep_dimensions_in(dense_y), dense_y); - EXPECT_EQUAL(dense.add_dimensions_from(dense), dense); - EXPECT_EQUAL(dense.keep_dimensions_in(dense), dense); -} - -void verify_combinable(const TensorType &a, const TensorType &b) { - EXPECT_TRUE(a.add_dimensions_from(b).is_valid()); - EXPECT_TRUE(b.add_dimensions_from(a).is_valid()); - EXPECT_TRUE(a.keep_dimensions_in(b).is_valid()); - EXPECT_TRUE(b.keep_dimensions_in(a).is_valid()); -} - -void verify_not_combinable(const TensorType &a, const TensorType &b) { - EXPECT_TRUE(!a.add_dimensions_from(b).is_valid()); - EXPECT_TRUE(!b.add_dimensions_from(a).is_valid()); - EXPECT_TRUE(!a.keep_dimensions_in(b).is_valid()); - EXPECT_TRUE(!b.keep_dimensions_in(a).is_valid()); -} - -TEST("require that dimensions need to have the same size to be combinable") { - verify_combinable(TensorType::dense({{"x", 10}}), TensorType::dense({{"x", 10}})); - verify_not_combinable(TensorType::dense({{"x", 10}}), TensorType::dense({{"x", 20}})); -} - -TEST("require that dimension combining only works for equal tensor types") { - std::vector<TensorType> types = {TensorType::invalid(), TensorType::number(), - TensorType::sparse({}), TensorType::dense({})}; - for (size_t a = 0; a < types.size(); ++a) { - for (size_t b = a; b < types.size(); ++b) { - TEST_STATE(vespalib::make_string("a=%zu, b=%zu", a, b).c_str()); - if ((a == b) && types[a].is_tensor()) { - verify_combinable(types[a], types[b]); - } else { - verify_not_combinable(types[a], types[b]); - } - } - } -} - -TEST("require that sparse tensor type can make spec") { - TensorType sparse = TensorType::sparse({}); - TensorType sparse_xy = TensorType::sparse({"x", "y"}); - TensorType sparse_yz = TensorType::sparse({"y", "z"}); - TensorType sparse_xyz = TensorType::sparse({"x", "y", "z"}); - TensorType sparse_y = TensorType::sparse({"y"}); - EXPECT_EQUAL("tensor()", sparse.toSpec()); - EXPECT_EQUAL("tensor(x{},y{})", sparse_xy.toSpec()); - EXPECT_EQUAL("tensor(y{},z{})", sparse_yz.toSpec()); - EXPECT_EQUAL("tensor(x{},y{},z{})", sparse_xyz.toSpec()); - EXPECT_EQUAL("tensor(y{})", sparse_y.toSpec()); -} - -TEST("require that dense tensor type can make spec") { - TensorType dense = TensorType::dense({}); - TensorType dense_xy = TensorType::dense({{"x", 10}, {"y", 10}}); - TensorType dense_yz = TensorType::dense({{"y", 10}, {"z", 10}}); - TensorType dense_xyz = TensorType::dense({{"x", 10}, {"y", 10}, {"z", 10}}); - TensorType dense_y = TensorType::dense({{"y", 10}}); - EXPECT_EQUAL("tensor()", dense.toSpec()); - EXPECT_EQUAL("tensor(x[10],y[10])", dense_xy.toSpec()); - EXPECT_EQUAL("tensor(y[10],z[10])", dense_yz.toSpec()); - EXPECT_EQUAL("tensor(x[10],y[10],z[10])", dense_xyz.toSpec()); - EXPECT_EQUAL("tensor(y[10])", dense_y.toSpec()); -} - -TEST("require that sparse tensor type spec can be parsed") { - TensorType sparse_xy = TensorType::sparse({"x", "y"}); - TensorType sparse_yz = TensorType::sparse({"y", "z"}); - TensorType sparse_xyz = TensorType::sparse({"x", "y", "z"}); - TensorType sparse_y = TensorType::sparse({"y"}); - EXPECT_EQUAL(sparse_xy, TensorType::fromSpec("tensor(x{},y{})")); - EXPECT_EQUAL(sparse_xy, - TensorType::fromSpec(" tensor ( x { } , y { } )")); - EXPECT_EQUAL(sparse_yz, TensorType::fromSpec("tensor(y{},z{})")); - EXPECT_EQUAL(sparse_xyz, TensorType::fromSpec("tensor(x{},y{},z{})")); - EXPECT_EQUAL(sparse_xyz, TensorType::fromSpec("tensor(z{},y{},x{})")); - EXPECT_EQUAL(sparse_y, TensorType::fromSpec("tensor(y{})")); -} - -TEST("require that dense tensor type spec can be parsed") { - TensorType dense = TensorType::dense({}); - TensorType dense_xy = TensorType::dense({{"x", 10}, {"y", 10}}); - TensorType dense_yz = TensorType::dense({{"y", 10}, {"z", 10}}); - TensorType dense_xyz = TensorType::dense({{"x", 10}, {"y", 10}, {"z", 10}}); - TensorType dense_y = TensorType::dense({{"y", 10}}); - EXPECT_EQUAL(dense, TensorType::fromSpec("tensor()")); - EXPECT_EQUAL(dense_xy, TensorType::fromSpec("tensor(x[10],y[10])")); - EXPECT_EQUAL(dense_xy, - TensorType::fromSpec(" tensor ( x [ 10 ] , y [ 10 ] ) ")); - EXPECT_EQUAL(dense_yz, TensorType::fromSpec("tensor(y[10],z[10])")); - EXPECT_EQUAL(dense_xyz, TensorType::fromSpec("tensor(x[10],y[10],z[10])")); - EXPECT_EQUAL(dense_xyz, TensorType::fromSpec("tensor(z[10],y[10],x[10])")); - EXPECT_EQUAL(dense_y, TensorType::fromSpec("tensor(y[10])")); -} - -TEST("require that tensor type can be converted to value type") { - EXPECT_TRUE(TensorType::invalid().as_value_type().is_error()); - EXPECT_TRUE(TensorType::number().as_value_type().is_double()); - EXPECT_EQUAL(ValueType::tensor_type({{"x"}, {"y"}, {"z"}}), - TensorType::sparse({"x", "y", "z"}).as_value_type()); - EXPECT_EQUAL(ValueType::tensor_type({{"x", 10}, {"y", 20}, {"z", 30}}), - TensorType::dense({{"x", 10}, {"y", 20}, {"z", 30}}).as_value_type()); - EXPECT_EQUAL(ValueType::double_type(), TensorType::sparse({}).as_value_type()); - EXPECT_EQUAL(ValueType::double_type(), TensorType::dense({}).as_value_type()); -} - -TEST("require that invalid tensor type spec is parsed as invalid") { - TensorType::Type invalid = TensorType::Type::INVALID; - EXPECT_TRUE(invalid == TensorType::fromSpec("tansor(y{})").type()); - EXPECT_TRUE(invalid == TensorType::fromSpec("tensor").type()); - EXPECT_TRUE(invalid == TensorType::fromSpec("tensor(y{10})").type()); - EXPECT_TRUE(invalid == TensorType::fromSpec("tensor(y{}").type()); - EXPECT_TRUE(invalid == TensorType::fromSpec("tensor(y{}),").type()); - EXPECT_TRUE(invalid == TensorType::fromSpec("tensor(x{},y[10])").type()); - EXPECT_TRUE(invalid == TensorType::fromSpec("tansor(y[10])").type()); - EXPECT_TRUE(invalid == TensorType::fromSpec("tensor").type()); - EXPECT_TRUE(invalid == TensorType::fromSpec("tensor(y[])").type()); - EXPECT_TRUE(invalid == TensorType::fromSpec("tensor(y[10]").type()); - EXPECT_TRUE(invalid == TensorType::fromSpec("tensor(y[10]),").type()); - EXPECT_TRUE(invalid == TensorType::fromSpec("tensor(x[10],y{})").type()); - EXPECT_TRUE(invalid == TensorType::fromSpec("invalid").type()); - EXPECT_TRUE(invalid == TensorType::fromSpec("number").type()); - EXPECT_TRUE(invalid == TensorType::fromSpec("dense").type()); - EXPECT_TRUE(invalid == TensorType::fromSpec("sparse").type()); - EXPECT_TRUE(invalid == TensorType::fromSpec("densetensor").type()); - EXPECT_TRUE(invalid == TensorType::fromSpec("sparsetensor").type()); - EXPECT_TRUE(invalid == TensorType::fromSpec("").type()); - EXPECT_TRUE(invalid == TensorType::fromSpec(" ").type()); - EXPECT_TRUE(invalid == TensorType::fromSpec("tensor(y{},y{})").type()); - EXPECT_TRUE(invalid == TensorType::fromSpec("tensor(y[10],y[10])").type()); -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/vespa/vespalib/tensor/CMakeLists.txt b/vespalib/src/vespa/vespalib/tensor/CMakeLists.txt index d2b1f8c9955..93a9f724235 100644 --- a/vespalib/src/vespa/vespalib/tensor/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/tensor/CMakeLists.txt @@ -8,8 +8,6 @@ vespa_add_library(vespalib_vespalib_tensor tensor_factory.cpp tensor_function.cpp tensor_mapper.cpp - tensor_type.cpp - tensor_type_spec.cpp $<TARGET_OBJECTS:vespalib_vespalib_tensor_compact> $<TARGET_OBJECTS:vespalib_vespalib_tensor_dense> $<TARGET_OBJECTS:vespalib_vespalib_tensor_serialization> diff --git a/vespalib/src/vespa/vespalib/tensor/tensor_function.h b/vespalib/src/vespa/vespalib/tensor/tensor_function.h index f60fc6ac881..f47c33adcbe 100644 --- a/vespalib/src/vespa/vespalib/tensor/tensor_function.h +++ b/vespalib/src/vespa/vespalib/tensor/tensor_function.h @@ -2,8 +2,8 @@ #pragma once -#include "tensor_type.h" #include "tensor.h" +#include <vespa/vespalib/eval/value_type.h> #include <memory> namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/tensor/tensor_type.cpp b/vespalib/src/vespa/vespalib/tensor/tensor_type.cpp deleted file mode 100644 index 608902325ec..00000000000 --- a/vespalib/src/vespa/vespalib/tensor/tensor_type.cpp +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/fastos/fastos.h> -#include <vespa/vespalib/util/stringfmt.h> -#include "tensor_type.h" -#include <algorithm> -#include <ostream> -#include "tensor_type_spec.h" - -using vespalib::eval::ValueType; - -namespace vespalib { -namespace tensor { - -namespace { - -void sort_dimensions(std::vector<TensorType::Dimension> &dimensions) { - std::sort(dimensions.begin(), dimensions.end(), - [](const auto &a, const auto &b){ return (a.name < b.name); }); -} - -bool has_duplicates(const std::vector<TensorType::Dimension> &dimensions) { - for (size_t i = 1; i < dimensions.size(); ++i) { - if (dimensions[i - 1].name == dimensions[i].name) { - return true; - } - } - return false; -} - -struct DimensionMatcher { - bool mismatch; - const std::vector<TensorType::Dimension> &dimension_list; - explicit DimensionMatcher(const std::vector<TensorType::Dimension> &dimension_list_in) - : mismatch(false), dimension_list(dimension_list_in) {} - bool find(const TensorType::Dimension &dimension) { - for (const auto &d: dimension_list) { - if (dimension.name == d.name) { - if (dimension.size != d.size) { - mismatch = true; - } - return true; - } - } - return false; - } -}; - -} // namespace vespalib::tensor::<unnamed> - -constexpr size_t TensorType::Dimension::npos; - -TensorType -TensorType::remove_dimensions(const std::vector<vespalib::string> &dimensions_in) const -{ - if (!is_tensor()) { - return invalid(); - } - size_t removed = 0; - std::vector<Dimension> result; - for (const Dimension &d: _dimensions) { - if (std::find(dimensions_in.begin(), dimensions_in.end(), d.name) == dimensions_in.end()) { - result.push_back(d); - } else { - ++removed; - } - } - if (removed != dimensions_in.size()) { - return invalid(); - } - return TensorType(_type, std::move(result)); -} - -TensorType -TensorType::add_dimensions_from(const TensorType &rhs) const -{ - if (!is_tensor() || (_type != rhs._type)) { - return invalid(); - } - std::vector<Dimension> result(_dimensions); - DimensionMatcher matcher(result); - for (const Dimension &d: rhs._dimensions) { - if (!matcher.find(d)) { - result.push_back(d); - } - } - if (matcher.mismatch) { - return invalid(); - } - sort_dimensions(result); - return TensorType(_type, std::move(result)); -} - -TensorType -TensorType::keep_dimensions_in(const TensorType &rhs) const -{ - if (!is_tensor() || (_type != rhs._type)) { - return invalid(); - } - std::vector<Dimension> result; - DimensionMatcher matcher(rhs._dimensions); - for (const Dimension &d: _dimensions) { - if (matcher.find(d)) { - result.push_back(d); - } - } - if (matcher.mismatch) { - return invalid(); - } - return TensorType(_type, std::move(result)); -} - -ValueType -TensorType::as_value_type() const -{ - if (is_number() || (is_tensor() && dimensions().empty())) { - return ValueType::double_type(); - } - if (is_tensor()) { - std::vector<ValueType::Dimension> my_dimensions; - for (const auto &dimension: dimensions()) { - my_dimensions.emplace_back(dimension.name, dimension.size); - } - return ValueType::tensor_type(std::move(my_dimensions)); - } - return ValueType::error_type(); -} - -TensorType -TensorType::sparse(const std::vector<vespalib::string> &dimensions_in) -{ - std::vector<Dimension> dimensions; - for (const auto &dimension_name: dimensions_in) { - dimensions.emplace_back(dimension_name); - } - sort_dimensions(dimensions); - if (has_duplicates(dimensions)) { - return invalid(); - } - return TensorType(Type::SPARSE, std::move(dimensions)); -} - -TensorType -TensorType::dense(std::vector<Dimension> dimensions_in) -{ - sort_dimensions(dimensions_in); - if (has_duplicates(dimensions_in)) { - return invalid(); - } - return TensorType(Type::DENSE, std::move(dimensions_in)); -} - -std::ostream &operator<<(std::ostream &os, const TensorType &type) { - size_t cnt = 0; - switch (type.type()) { - case TensorType::Type::INVALID: - os << "INVALID"; - break; - case TensorType::Type::NUMBER: - os << "NUMBER"; - break; - case TensorType::Type::SPARSE: - os << "SPARSE("; - for (const auto &d: type.dimensions()) { - if (cnt++ > 0) { - os << ","; - } - os << d.name; - } - os << ")"; - break; - case TensorType::Type::DENSE: - os << "DENSE("; - for (const auto &d: type.dimensions()) { - if (cnt++ > 0) { - os << ","; - } - os << vespalib::make_string("{%s:%zu}", d.name.c_str(), d.size); - } - os << ")"; - break; - } - return os; -} - - -TensorType -TensorType::fromSpec(const vespalib::string &str) -{ - return tensor_type::fromSpec(str); -} - - -vespalib::string -TensorType::toSpec() const -{ - return tensor_type::toSpec(*this); -} - - -} // namespace vespalib::tensor -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/tensor/tensor_type.h b/vespalib/src/vespa/vespalib/tensor/tensor_type.h deleted file mode 100644 index 1cc0b70beb8..00000000000 --- a/vespalib/src/vespa/vespalib/tensor/tensor_type.h +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/eval/value_type.h> -#include <vector> - -namespace vespalib { -namespace tensor { - -/** - * The type of 'a single tensor' or 'the result of a tensor - * operation'. - * - * TensorType is a value class containing implementation-independent - * type information for a single tensor or the result of a tensor - * operation. The type of the result of a tensor operation depends - * only on the types of the input values and the type of the operation - * itself. Illegal operations will result in the type INVALID. For the - * collapsing operation 'sum', the result will be of type NUMBER. Both - * SPARSE and DENSE tensor types will contain information about their - * dimensions. Dimension 'size' is only relevant for DENSE tensors and - * will be set to 'npos' for SPARSE tensors. - **/ -class TensorType -{ -public: - enum class Type { INVALID, NUMBER, SPARSE, DENSE }; - struct Dimension { - static constexpr size_t npos = -1; - vespalib::string name; - size_t size; - explicit Dimension(const vespalib::string &name_in) - : name(name_in), size(npos) {} - Dimension(const vespalib::string &name_in, size_t size_in) - : name(name_in), size(size_in) {} - bool operator==(const Dimension &rhs) const { - return ((name == rhs.name) && (size == rhs.size)); - } - bool operator!=(const Dimension &rhs) const { return !(*this == rhs); } - }; - -private: - Type _type; - std::vector<Dimension> _dimensions; - - explicit TensorType(Type type_in) // INVALID/NUMBER - : _type(type_in), _dimensions() {} - TensorType(Type type_in, std::vector<Dimension> &&dimensions_in) // SPARSE/DENSE - : _type(type_in), _dimensions(std::move(dimensions_in)) {} - -public: - Type type() const { return _type; } - bool is_valid() const { return (_type != Type::INVALID); } - bool is_number() const { return (_type == Type::NUMBER); } - bool is_tensor() const { - return ((_type == Type::SPARSE) || (_type == Type::DENSE)); - } - const std::vector<Dimension> &dimensions() const { return _dimensions; } - - bool operator==(const TensorType &rhs) const { - if ((_type == Type::INVALID) || (rhs._type == Type::INVALID)) { - return false; - } - return ((_type == rhs._type) && (_dimensions == rhs._dimensions)); - } - bool operator!=(const TensorType &rhs) const { return !(*this == rhs); } - - TensorType remove_dimensions(const std::vector<vespalib::string> &dimensions_in) const; - TensorType add_dimensions_from(const TensorType &rhs) const; - TensorType keep_dimensions_in(const TensorType &rhs) const; - - eval::ValueType as_value_type() const; - - static TensorType invalid() { return TensorType(Type::INVALID); } - static TensorType number() { return TensorType(Type::NUMBER); } - static TensorType sparse(const std::vector<vespalib::string> &dimensions_in); - static TensorType dense(std::vector<Dimension> dimensions_in); - static TensorType fromSpec(const vespalib::string &str); - vespalib::string toSpec() const; -}; - -std::ostream &operator<<(std::ostream &os, const TensorType &type); - -} // namespace vespalib::tensor -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/tensor/tensor_type_spec.cpp b/vespalib/src/vespa/vespalib/tensor/tensor_type_spec.cpp deleted file mode 100644 index cb0acc498b9..00000000000 --- a/vespalib/src/vespa/vespalib/tensor/tensor_type_spec.cpp +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/fastos/fastos.h> -#include "tensor_type.h" -#include <vespa/vespalib/stllike/asciistream.h> -#include <algorithm> -#include "tensor_type_spec.h" - -namespace vespalib { -namespace tensor { -namespace tensor_type { - -namespace { - -class Tokenizer { - const char *_cur; - const char *_end; - - static bool atSpace(char c) { return ((c == ' ') || (c == '\t') || - (c == '\n') || (c == '\r')); } - static bool atNumber(char c) { return ((c >= '0') && (c <= '9')); } - static bool atNameBreak(char c) { - return (atSpace(c) || (c == '[') || (c == ']') || (c == '{') || - (c == '}') || (c == '(') || (c == ')') || (c == ',')); - } -public: - Tokenizer(const vespalib::string &str) - : _cur(str.c_str()), - _end(_cur + str.size()) - { - } - - bool atEnd() const { return _cur == _end; } - void step() { ++_cur; } - void skipSpace() { while (!atEnd() && atSpace(*_cur)) { step(); } } - char cur() { skipSpace(); return (atEnd() ? '\0' : *_cur); } - - void dimensionNameScan() { - while (!atEnd() && !atNameBreak(*_cur)) { - step(); - } - } - - void numberScan() { - while (!atEnd() && atNumber(*_cur)) { step(); } - } - - stringref getDimensionName() { - skipSpace(); - const char *start = _cur; - dimensionNameScan(); - return stringref(start, _cur - start); - } - - stringref getNumber() { - skipSpace(); - const char *start = _cur; - numberScan(); - return stringref(start, _cur - start); - } -}; - - -class Parser -{ - std::vector<TensorType::Dimension> _dimensions; - vespalib::string _dimensionName; - vespalib::string _dimensionSizeStr; - bool _parseError; - bool _denseDim; - bool _sparseDim; - - void parseDenseDimension(Tokenizer &tok) { - tok.step(); // step over open bracket - _dimensionSizeStr = tok.getNumber(); - if (_dimensionSizeStr.empty() || (tok.cur() != ']')) { - _parseError = true; // no close bracket or empty dim size - return; - } - tok.step(); // step over close bracket - long dimensionSize = atol(_dimensionSizeStr.c_str()); - if (dimensionSize <= 0) { - _parseError = true; // bad dim size - } else { - _dimensions.emplace_back(_dimensionName, dimensionSize); - _denseDim = true; - } - } - - void parseSparseDimension(Tokenizer &tok) - { - tok.step(); // step over open brace - if (tok.cur() != '}') { - _parseError = true; // no close brace - } else { - tok.step(); // step over close brace - _dimensions.emplace_back(_dimensionName); - _sparseDim = true; - } - } - - void parseDimension(Tokenizer &tok) { - _dimensionName = tok.getDimensionName(); - if (_dimensionName.empty()) { - _parseError = true; // no dimension name - } else if (tok.cur() == '[') { - parseDenseDimension(tok); - } else if (tok.cur() == '{') { - parseSparseDimension(tok); - } else { - _parseError = true; // no open brace or bracket - } - } -public: - - Parser() - : _dimensions(), - _dimensionName(), - _dimensionSizeStr(), - _parseError(false), - _denseDim(false), - _sparseDim(false) - { - } - - void parse(Tokenizer &tok) { - tok.skipSpace(); - _dimensionName = tok.getDimensionName(); - if (_dimensionName != "tensor" || tok.cur() != '(') { - _parseError = true; // doesn't start with tensor and left parenthesis - } else { - tok.step(); // step over left parentesis - } - while (!_parseError && tok.cur() != ')') { - parseDimension(tok); - if (tok.cur() == ',') { - tok.step(); - tok.skipSpace(); - } else if (tok.cur() != ')') { - _parseError = true; // no comma between dimensions - } - } - if (_parseError) { - } else if (tok.cur() != ')') { - _parseError = true; // no right parenthesis at end - } else { - tok.step(); // step over right parenthesis - tok.skipSpace(); - if (!tok.atEnd()) { - _parseError = true; // more unparsed data - } - } - } - - bool invalid() const { return (_parseError || (_denseDim && _sparseDim)); } - bool sparse() const { return _sparseDim; } - std::vector<TensorType::Dimension> &dimensions() { return _dimensions; } -}; - -} // namespace vespalib::tensor::tensor_type::<anonymous> - - -TensorType -fromSpec(const vespalib::string &str) -{ - Tokenizer tok(str); - Parser parser; - parser.parse(tok); - if (parser.invalid()) { - return TensorType::invalid(); - } - - if (parser.sparse()) { - std::vector<vespalib::string> dimensions; - for (const auto &dim : parser.dimensions()) { - dimensions.emplace_back(dim.name); - } - return TensorType::sparse(dimensions); - } else { - return TensorType::dense(parser.dimensions()); - } -} - - -vespalib::string -toSpec(const TensorType &type) -{ - asciistream os; - size_t cnt = 0; - switch (type.type()) { - case TensorType::Type::INVALID: - os << "invalid"; - break; - case TensorType::Type::NUMBER: - os << "number"; - break; - case TensorType::Type::SPARSE: - os << "tensor("; - for (const auto &d: type.dimensions()) { - if (cnt++ > 0) { - os << ","; - } - os << d.name << "{}"; - } - os << ")"; - break; - case TensorType::Type::DENSE: - os << "tensor("; - for (const auto &d: type.dimensions()) { - if (cnt++ > 0) { - os << ","; - } - os << d.name << "[" << d.size << "]"; - } - os << ")"; - break; - } - return os.str(); -} - -} // namespace vespalib::tensor::tensor_type -} // namespace vespalib::tensor -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/tensor/tensor_type_spec.h b/vespalib/src/vespa/vespalib/tensor/tensor_type_spec.h deleted file mode 100644 index 0eb713a8a36..00000000000 --- a/vespalib/src/vespa/vespalib/tensor/tensor_type_spec.h +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "tensor_type.h" - -namespace vespalib { -namespace tensor { -namespace tensor_type { - -TensorType fromSpec(const vespalib::string &str); -vespalib::string toSpec(const TensorType &type); - -} // namespace vespalib::tensor::tensor_type -} // namespace vespalib::tensor -} // namespace vespalib |