diff options
author | Geir Storli <geirst@yahoo-inc.com> | 2016-09-15 09:28:36 +0200 |
---|---|---|
committer | Geir Storli <geirst@yahoo-inc.com> | 2016-09-15 09:28:36 +0200 |
commit | 0d15262410021732ea150f057cf3213751db4d98 (patch) | |
tree | 93c2eb27d171dd2aa274670ee15c812a92e5eff1 | |
parent | 6f3115944901cc50c604f6b165724958dddb51f2 (diff) |
Change to use eval::ValueType instead of tensor::TensorType.
41 files changed, 362 insertions, 303 deletions
diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp index f3604493f01..80e45f3a977 100644 --- a/document/src/tests/documentupdatetestcase.cpp +++ b/document/src/tests/documentupdatetestcase.cpp @@ -27,7 +27,6 @@ using namespace document::config_builder; using vespalib::tensor::Tensor; -using vespalib::tensor::TensorType; using vespalib::tensor::TensorCells; using vespalib::tensor::TensorDimensions; diff --git a/searchcommon/src/tests/attribute/config/attribute_config_test.cpp b/searchcommon/src/tests/attribute/config/attribute_config_test.cpp index 3a7994ee39b..50522aae3b9 100644 --- a/searchcommon/src/tests/attribute/config/attribute_config_test.cpp +++ b/searchcommon/src/tests/attribute/config/attribute_config_test.cpp @@ -7,7 +7,7 @@ using search::attribute::Config; using search::attribute::BasicType; using search::attribute::CollectionType; -using vespalib::tensor::TensorType; +using vespalib::eval::ValueType; struct Fixture @@ -38,7 +38,7 @@ TEST_F("test default attribute config", Fixture) EXPECT_TRUE(!f._config.getEnableOnlyBitVector()); EXPECT_TRUE(!f._config.getIsFilter()); EXPECT_TRUE(!f._config.fastAccess()); - EXPECT_TRUE(!f._config.tensorType().is_valid()); + EXPECT_TRUE(f._config.tensorType().is_error()); } TEST_F("test integer weightedset attribute config", @@ -54,7 +54,7 @@ TEST_F("test integer weightedset attribute config", EXPECT_TRUE(!f._config.getEnableOnlyBitVector()); EXPECT_TRUE(!f._config.getIsFilter()); EXPECT_TRUE(!f._config.fastAccess()); - EXPECT_TRUE(!f._config.tensorType().is_valid()); + EXPECT_TRUE(f._config.tensorType().is_error()); } @@ -76,21 +76,20 @@ TEST("test operator== on attribute config for tensor type") Config cfg2(BasicType::Type::TENSOR); Config cfg3(BasicType::Type::TENSOR); - TensorType dense_x = TensorType::fromSpec("tensor(x[10])"); - TensorType sparse_x = TensorType::fromSpec("tensor(x{})"); + ValueType dense_x = ValueType::from_spec("tensor(x[10])"); + ValueType sparse_x = ValueType::from_spec("tensor(x{})"); - // invalid tensors are not equal - EXPECT_TRUE(cfg1 != cfg2); - EXPECT_TRUE(cfg2 != cfg3); - EXPECT_TRUE(cfg1 != cfg3); + EXPECT_TRUE(cfg1 == cfg2); + EXPECT_TRUE(cfg2 == cfg3); + EXPECT_TRUE(cfg1 == cfg3); cfg1.setTensorType(dense_x); cfg3.setTensorType(dense_x); EXPECT_EQUAL(dense_x, cfg1.tensorType()); EXPECT_EQUAL(dense_x, cfg3.tensorType()); - EXPECT_TRUE(cfg1.tensorType().is_valid()); - EXPECT_TRUE(!cfg2.tensorType().is_valid()); - EXPECT_TRUE(cfg3.tensorType().is_valid()); + EXPECT_TRUE(!cfg1.tensorType().is_error()); + EXPECT_TRUE(cfg2.tensorType().is_error()); + EXPECT_TRUE(!cfg3.tensorType().is_error()); EXPECT_TRUE(cfg1 != cfg2); EXPECT_TRUE(cfg2 != cfg3); @@ -98,7 +97,7 @@ TEST("test operator== on attribute config for tensor type") cfg3.setTensorType(sparse_x); EXPECT_EQUAL(sparse_x, cfg3.tensorType()); - EXPECT_TRUE(cfg3.tensorType().is_valid()); + EXPECT_TRUE(!cfg3.tensorType().is_error()); EXPECT_TRUE(cfg1 != cfg3); } diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.cpp b/searchcommon/src/vespa/searchcommon/attribute/config.cpp index e160d7b9222..17f758770be 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/config.cpp +++ b/searchcommon/src/vespa/searchcommon/attribute/config.cpp @@ -21,7 +21,7 @@ Config::Config() : _lower_bound(LLONG_MIN), _upper_bound(LLONG_MAX), _dense_posting_list_threshold(0.4), - _tensorType(vespalib::tensor::TensorType::invalid()) + _tensorType(vespalib::eval::ValueType::error_type()) { } @@ -42,7 +42,7 @@ Config::Config(BasicType bt, _lower_bound(LLONG_MIN), _upper_bound(LLONG_MAX), _dense_posting_list_threshold(0.4), - _tensorType(vespalib::tensor::TensorType::invalid()) + _tensorType(vespalib::eval::ValueType::error_type()) { } diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.h b/searchcommon/src/vespa/searchcommon/attribute/config.h index dee684763a0..b346799b2ea 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/config.h +++ b/searchcommon/src/vespa/searchcommon/attribute/config.h @@ -5,7 +5,7 @@ #include <vespa/searchcommon/attribute/basictype.h> #include <vespa/searchcommon/attribute/collectiontype.h> #include <vespa/searchcommon/common/growstrategy.h> -#include <vespa/vespalib/tensor/tensor_type.h> +#include <vespa/vespalib/eval/value_type.h> namespace search { namespace attribute { @@ -31,7 +31,7 @@ public: int64_t lower_bound() const { return _lower_bound; } int64_t upper_bound() const { return _upper_bound; } double dense_posting_list_threshold() const { return _dense_posting_list_threshold; } - vespalib::tensor::TensorType tensorType() const { return _tensorType; } + vespalib::eval::ValueType tensorType() const { return _tensorType; } /** * Check if attribute posting list can consist of a bitvector in @@ -73,7 +73,7 @@ public: void setBounds(int64_t lower, int64_t upper) { _lower_bound = lower; _upper_bound = upper; } void setDensePostingListThreshold(double v) { _dense_posting_list_threshold = v; } - void setTensorType(const vespalib::tensor::TensorType &tensorType_in) { + void setTensorType(const vespalib::eval::ValueType &tensorType_in) { _tensorType = tensorType_in; } @@ -148,7 +148,7 @@ private: int64_t _lower_bound; int64_t _upper_bound; double _dense_posting_list_threshold; - vespalib::tensor::TensorType _tensorType; + vespalib::eval::ValueType _tensorType; }; } // namespace attribute } // namespace search diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index 53dffaf9765..7702ac10b48 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -51,8 +51,8 @@ using search::TuneFileAttributes; using search::index::DummyFileHeaderContext; using search::predicate::PredicateIndex; using search::predicate::PredicateHash; +using vespalib::eval::ValueType; using vespalib::tensor::Tensor; -using vespalib::tensor::TensorType; using vespalib::tensor::TensorCells; using vespalib::tensor::TensorDimensions; @@ -573,7 +573,7 @@ createTensor(const TensorCells &cells, const TensorDimensions &dimensions) { AttributeVector::SP createTensorAttribute(Fixture &f) { AVConfig cfg(AVBasicType::TENSOR); - cfg.setTensorType(TensorType::fromSpec("tensor(x{},y{})")); + cfg.setTensorType(ValueType::from_spec("tensor(x{},y{})")); return f._m->addAttribute("a1", cfg, createSerialNum); } diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index 6fc11401299..5dc7b335106 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -32,29 +32,30 @@ LOG_SETUP("feedview_test"); #include <vespa/vespalib/util/blockingthreadstackexecutor.h> #include <mutex> -using documentapi::RemoveDocumentReply; -using namespace proton; + using document::BucketId; using document::DataType; using document::Document; using document::DocumentId; using document::DocumentUpdate; using documentapi::DocumentProtocol; +using documentapi::RemoveDocumentReply; +using fastos::TimeStamp; +using namespace proton; using proton::matching::SessionManager; -using search::index::DocBuilder; -using search::index::Schema; using search::AttributeVector; using search::CacheStats; using search::DocumentMetaData; using search::SearchableStats; +using search::index::DocBuilder; +using search::index::Schema; using searchcorespi::IndexSearchable; using storage::spi::BucketChecksum; using storage::spi::BucketInfo; using storage::spi::PartitionId; using storage::spi::Timestamp; using storage::spi::UpdateResult; -using vespalib::tensor::TensorType; -using fastos::TimeStamp; +using vespalib::eval::ValueType; typedef SearchableFeedView::SerialNum SerialNum; typedef search::DocumentIdT DocumentIdT; @@ -294,7 +295,7 @@ struct MyAttributeWriter : public IAttributeWriter _attrMap["a2"] = search::AttributeFactory::createAttribute("test2", cfg2); search::attribute::Config cfg3(search::attribute::BasicType::TENSOR); - cfg3.setTensorType(TensorType::fromSpec("tensor(x[10])")); + cfg3.setTensorType(ValueType::from_spec("tensor(x[10])")); _attrMap["a3"] = search::AttributeFactory::createAttribute("test3", cfg3); } diff --git a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp index bf247668843..24b1d657d2f 100644 --- a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp +++ b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp @@ -17,7 +17,6 @@ using namespace config; using namespace vespa::config::search; using namespace search; using namespace search::attribute; -using vespalib::tensor::TensorType; using std::shared_ptr; typedef BasicType BT; @@ -298,7 +297,7 @@ AttributeManagerTest::testConfigConvert() a.datatype = CACA::TENSOR; a.tensortype = "tensor(x[5])"; AttributeVector::Config out = ConfigConverter::convert(a); - EXPECT_EQUAL("tensor(x[5])", out.tensorType().toSpec()); + EXPECT_EQUAL("tensor(x[5])", out.tensorType().to_spec()); } } diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 73319a30888..ab8715a29dc 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -12,11 +12,11 @@ LOG_SETUP("tensorattribute_test"); using search::attribute::TensorAttribute; using search::AttributeGuard; using search::AttributeVector; +using vespalib::eval::ValueType; using vespalib::tensor::Tensor; using vespalib::tensor::TensorCells; using vespalib::tensor::TensorDimensions; using vespalib::tensor::TensorFactory; -using vespalib::tensor::TensorType; using vespalib::tensor::SimpleTensorBuilder; namespace vespalib { @@ -49,7 +49,7 @@ struct Fixture _tensorAttr(), _attr() { - _cfg.setTensorType(TensorType::fromSpec(typeSpec)); + _cfg.setTensorType(ValueType::from_spec(typeSpec)); _tensorAttr = std::make_shared<TensorAttribute>(_name, _cfg); _attr = _tensorAttr; _attr->addReservedDoc(); @@ -219,7 +219,7 @@ TEST_F("Require that tensor attribute can provide empty tensor of correct type", const TensorAttribute &tensorAttr = *f._tensorAttr; Tensor::UP emptyTensor = tensorAttr.getEmptyTensor(); EXPECT_EQUAL(emptyTensor->getType(), tensorAttr.getConfig().tensorType()); - EXPECT_EQUAL(emptyTensor->getType(), TensorType::fromSpec("tensor(x[10])")); + EXPECT_EQUAL(emptyTensor->getType(), ValueType::from_spec("tensor(x[10])")); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/features/constant/constant_test.cpp b/searchlib/src/tests/features/constant/constant_test.cpp index 3fae07f908a..6cae122d8c8 100644 --- a/searchlib/src/tests/features/constant/constant_test.cpp +++ b/searchlib/src/tests/features/constant/constant_test.cpp @@ -74,7 +74,7 @@ struct ExecFixture const TensorDimensions &dimensions) { Tensor::UP tensor = createTensor(cells, dimensions); - ValueType type(tensor->getType().as_value_type()); + ValueType type(tensor->getType()); test.getIndexEnv().addConstantValue(name, std::move(type), std::make_unique<TensorValue>(std::move(tensor))); diff --git a/searchlib/src/tests/features/tensor/tensor_test.cpp b/searchlib/src/tests/features/tensor/tensor_test.cpp index bcb984c692b..6b3b09bcec1 100644 --- a/searchlib/src/tests/features/tensor/tensor_test.cpp +++ b/searchlib/src/tests/features/tensor/tensor_test.cpp @@ -28,12 +28,12 @@ using search::attribute::TensorAttribute; using search::AttributeVector; using vespalib::eval::Function; using vespalib::eval::Value; +using vespalib::eval::ValueType; using vespalib::tensor::DenseTensorCells; using vespalib::tensor::Tensor; using vespalib::tensor::TensorCells; using vespalib::tensor::TensorDimensions; using vespalib::tensor::TensorFactory; -using vespalib::tensor::TensorType; typedef search::attribute::Config AVC; typedef search::attribute::BasicType AVBT; @@ -75,7 +75,7 @@ struct ExecFixture AttributeVector::SP createTensorAttribute(const vespalib::string &attrName, const vespalib::string &type) { addAttributeField(attrName); AVC config(AVBT::TENSOR, AVCT::SINGLE); - config.setTensorType(TensorType::fromSpec(type)); + config.setTensorType(ValueType::from_spec(type)); return AttributeFactory::createAttribute(attrName, config); } void setAttributeTensorType(const vespalib::string &attrName, const vespalib::string &type) { diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index cc223a4fada..00aaf73c5df 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -617,7 +617,7 @@ AttributeVector::createSaveTargetConfig() const getConfig().collectionType().asString(), getConfig().basicType().type() == BasicType::Type::TENSOR ? - getConfig().tensorType().toSpec() : + getConfig().tensorType().to_spec() : "", hasMultiValue(), hasWeightedSetType(), diff --git a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp index 250cad79357..6fd23528450 100644 --- a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp @@ -13,7 +13,7 @@ namespace { using search::attribute::CollectionType; using search::attribute::BasicType; -using vespalib::tensor::TensorType; +using vespalib::eval::ValueType; typedef std::map<AttributesConfig::Attribute::Datatype, BasicType::Type> DataTypeMap; typedef std::map<AttributesConfig::Attribute::Collectiontype, CollectionType::Type> CollectionTypeMap; @@ -74,7 +74,11 @@ ConfigConverter::convert(const AttributesConfig::Attribute & cfg) retval.setBounds(cfg.lowerbound, cfg.upperbound); retval.setDensePostingListThreshold(cfg.densepostinglistthreshold); if (retval.basicType().type() == BasicType::Type::TENSOR) { - retval.setTensorType(TensorType::fromSpec(cfg.tensortype)); + if (!cfg.tensortype.empty()) { + retval.setTensorType(ValueType::from_spec(cfg.tensortype)); + } else { + retval.setTensorType(ValueType::tensor_type({})); + } } return retval; } diff --git a/searchlib/src/vespa/searchlib/features/attributefeature.cpp b/searchlib/src/vespa/searchlib/features/attributefeature.cpp index 5f3310c4b53..e0290b1a73e 100644 --- a/searchlib/src/vespa/searchlib/features/attributefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/attributefeature.cpp @@ -30,7 +30,6 @@ using search::attribute::WeightedIntegerContent; using search::attribute::WeightedFloatContent; using search::fef::FeatureExecutor; using search::features::util::ConstCharPtr; -using vespalib::tensor::TensorType; using vespalib::eval::ValueType; using search::fef::FeatureType; @@ -281,7 +280,7 @@ AttributeBlueprint::AttributeBlueprint() : search::fef::Blueprint("attribute"), _attrName(), _extra(), - _tensorType(TensorType::number()) + _tensorType(ValueType::double_type()) { } @@ -303,10 +302,10 @@ AttributeBlueprint::setup(const search::fef::IIndexEnvironment & env, } vespalib::string attrType = type::Attribute::lookup(env.getProperties(), _attrName); if (!attrType.empty()) { - _tensorType = TensorType::fromSpec(attrType); + _tensorType = ValueType::from_spec(attrType); } FeatureType output_type = _tensorType.is_tensor() - ? FeatureType::object(_tensorType.as_value_type()) + ? FeatureType::object(_tensorType) : FeatureType::number(); describeOutput("value", "The value of a single value attribute, " "the value at the given index of an array attribute, " @@ -386,7 +385,7 @@ createAttributeExecutor(const IAttributeVector *attribute, const vespalib::strin search::fef::FeatureExecutor::LP createTensorAttributeExecutor(const IAttributeVector *attribute, const vespalib::string &attrName, - const TensorType &tensorType) + const ValueType &tensorType) { if (attribute == NULL) { LOG(warning, "The attribute vector '%s' was not found in the attribute manager." @@ -409,8 +408,8 @@ createTensorAttributeExecutor(const IAttributeVector *attribute, const vespalib: LOG(warning, "The tensor attribute '%s' has tensor type '%s'," " while the feature executor expects type '%s'. Returning empty tensor.", tensorAttribute->getName().c_str(), - tensorAttribute->getConfig().tensorType().toSpec().c_str(), - tensorType.toSpec().c_str()); + tensorAttribute->getConfig().tensorType().to_spec().c_str(), + tensorType.to_spec().c_str()); return ConstantTensorExecutor::createEmpty(tensorType); } return FeatureExecutor::LP(new TensorFromTensorAttributeExecutor(tensorAttribute)); diff --git a/searchlib/src/vespa/searchlib/features/attributefeature.h b/searchlib/src/vespa/searchlib/features/attributefeature.h index c25b2b558b1..0946dab3516 100644 --- a/searchlib/src/vespa/searchlib/features/attributefeature.h +++ b/searchlib/src/vespa/searchlib/features/attributefeature.h @@ -4,8 +4,7 @@ #include <vespa/searchlib/fef/blueprint.h> #include <vespa/searchlib/fef/featureexecutor.h> -#include <vespa/vespalib/tensor/tensor_type.h> - +#include <vespa/vespalib/eval/value_type.h> namespace search { namespace features { @@ -21,7 +20,7 @@ class AttributeBlueprint : public search::fef::Blueprint { private: vespalib::string _attrName; // the name of the attribute vector vespalib::string _extra; // the index or key - vespalib::tensor::TensorType _tensorType; + vespalib::eval::ValueType _tensorType; public: /** diff --git a/searchlib/src/vespa/searchlib/features/queryfeature.cpp b/searchlib/src/vespa/searchlib/features/queryfeature.cpp index 32763ef6ef2..c150f158d79 100644 --- a/searchlib/src/vespa/searchlib/features/queryfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/queryfeature.cpp @@ -24,7 +24,6 @@ using namespace search::fef; using namespace search::fef::indexproperties; using vespalib::tensor::DefaultTensor; using vespalib::tensor::TensorBuilder; -using vespalib::tensor::TensorType; using vespalib::eval::ValueType; using search::fef::FeatureType; @@ -64,7 +63,7 @@ QueryBlueprint::QueryBlueprint() : _key(), _key2(), _defaultValue(0), - _tensorType(TensorType::number()) + _valueType(ValueType::double_type()) { } @@ -101,10 +100,10 @@ QueryBlueprint::setup(const IIndexEnvironment &env, } vespalib::string queryFeatureType = type::QueryFeature::lookup(env.getProperties(), _key); if (!queryFeatureType.empty()) { - _tensorType = TensorType::fromSpec(queryFeatureType); + _valueType = ValueType::from_spec(queryFeatureType); } - FeatureType output_type = _tensorType.is_tensor() - ? FeatureType::object(_tensorType.as_value_type()) + FeatureType output_type = _valueType.is_tensor() + ? FeatureType::object(_valueType) : FeatureType::number(); describeOutput("out", "The value looked up in query properties using the given key.", output_type); @@ -116,7 +115,7 @@ namespace { FeatureExecutor::LP createTensorExecutor(const search::fef::IQueryEnvironment &env, const vespalib::string &queryKey, - const TensorType &tensorType) + const ValueType &valueType) { search::fef::Property prop = env.getProperties().lookup(queryKey); if (prop.found() && !prop.get().empty()) { @@ -125,14 +124,14 @@ createTensorExecutor(const search::fef::IQueryEnvironment &env, vespalib::nbostream stream(value.data(), value.size()); vespalib::tensor::TypedBinaryFormat::deserialize(stream, tensorBuilder); vespalib::tensor::Tensor::UP tensor = tensorBuilder.build(); - if (tensor->getType() != tensorType) { - vespalib::tensor::TensorMapper mapper(tensorType); + if (tensor->getType() != valueType) { + vespalib::tensor::TensorMapper mapper(valueType); vespalib::tensor::Tensor::UP mappedTensor = mapper.map(*tensor); tensor = std::move(mappedTensor); } return ConstantTensorExecutor::create(std::move(tensor)); } - return ConstantTensorExecutor::createEmpty(tensorType); + return ConstantTensorExecutor::createEmpty(valueType); } } @@ -140,8 +139,8 @@ createTensorExecutor(const search::fef::IQueryEnvironment &env, FeatureExecutor::LP QueryBlueprint::createExecutor(const IQueryEnvironment &env) const { - if (_tensorType.is_tensor()) { - return createTensorExecutor(env, _key, _tensorType); + if (_valueType.is_tensor()) { + return createTensorExecutor(env, _key, _valueType); } else { std::vector<feature_t> values; Property p = env.getProperties().lookup(_key); diff --git a/searchlib/src/vespa/searchlib/features/queryfeature.h b/searchlib/src/vespa/searchlib/features/queryfeature.h index fa3194b30c5..b6dd4e77308 100644 --- a/searchlib/src/vespa/searchlib/features/queryfeature.h +++ b/searchlib/src/vespa/searchlib/features/queryfeature.h @@ -19,7 +19,7 @@ private: vespalib::string _key; // 'foo' vespalib::string _key2; // '$foo' feature_t _defaultValue; - vespalib::tensor::TensorType _tensorType; + vespalib::eval::ValueType _valueType; public: /** diff --git a/searchlib/src/vespa/searchlib/fef/test/as_tensor.cpp b/searchlib/src/vespa/searchlib/fef/test/as_tensor.cpp index f71973469fb..4f604bb9f89 100644 --- a/searchlib/src/vespa/searchlib/fef/test/as_tensor.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/as_tensor.cpp @@ -10,8 +10,8 @@ #include <iostream> using vespalib::eval::Function; +using vespalib::eval::ValueType; using vespalib::tensor::DefaultTensorEngine; -using vespalib::tensor::TensorType; using vespalib::tensor::TensorMapper; namespace search { @@ -35,7 +35,7 @@ bool AsTensor::operator==(const Tensor &rhs) const { AsEmptyTensor::AsEmptyTensor(const vespalib::string &type) : AsTensor("{ }"), - mappedTensor(TensorMapper(TensorType::fromSpec(type)).map(*tensor)) + mappedTensor(TensorMapper(ValueType::from_spec(type)).map(*tensor)) { } diff --git a/vespalib/src/tests/tensor/compact_tensor_builder/compact_tensor_builder_test.cpp b/vespalib/src/tests/tensor/compact_tensor_builder/compact_tensor_builder_test.cpp index 3d9f7f36242..e8b5d3e141c 100644 --- a/vespalib/src/tests/tensor/compact_tensor_builder/compact_tensor_builder_test.cpp +++ b/vespalib/src/tests/tensor/compact_tensor_builder/compact_tensor_builder_test.cpp @@ -52,7 +52,7 @@ TEST("require that dimensions are extracted") EXPECT_EQUAL("a", dims[0]); EXPECT_EQUAL("b", dims[1]); EXPECT_EQUAL("c", dims[2]); - EXPECT_EQUAL("tensor(a{},b{},c{})", compactTensor.getType().toSpec()); + EXPECT_EQUAL("tensor(a{},b{},c{})", compactTensor.getType().to_spec()); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/tensor/compact_tensor_v2_builder/compact_tensor_v2_builder_test.cpp b/vespalib/src/tests/tensor/compact_tensor_v2_builder/compact_tensor_v2_builder_test.cpp index 18630967d33..7dc1e6c1117 100644 --- a/vespalib/src/tests/tensor/compact_tensor_v2_builder/compact_tensor_v2_builder_test.cpp +++ b/vespalib/src/tests/tensor/compact_tensor_v2_builder/compact_tensor_v2_builder_test.cpp @@ -73,7 +73,7 @@ TEST("require that dimensions are extracted") EXPECT_EQUAL("a", dims[0]); EXPECT_EQUAL("b", dims[1]); EXPECT_EQUAL("c", dims[2]); - EXPECT_EQUAL("tensor(a{},b{},c{})", compactTensor.getType().toSpec()); + EXPECT_EQUAL("tensor(a{},b{},c{})", compactTensor.getType().to_spec()); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/tensor/dense_tensor_builder/dense_tensor_builder_test.cpp b/vespalib/src/tests/tensor/dense_tensor_builder/dense_tensor_builder_test.cpp index 09c1e31ef2f..8478d46e1f4 100644 --- a/vespalib/src/tests/tensor/dense_tensor_builder/dense_tensor_builder_test.cpp +++ b/vespalib/src/tests/tensor/dense_tensor_builder/dense_tensor_builder_test.cpp @@ -202,7 +202,7 @@ TEST_F("require that dimensions are sorted", Fixture) assertTensor({{"x", 5}, {"y", 3}}, {10, 11, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, denseTensor); - EXPECT_EQUAL("tensor(x[5],y[3])", denseTensor.getType().toSpec()); + EXPECT_EQUAL("tensor(x[5],y[3])", denseTensor.getType().to_spec()); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/tensor/dense_tensor_operations/dense_tensor_operations_test.cpp b/vespalib/src/tests/tensor/dense_tensor_operations/dense_tensor_operations_test.cpp index 8264fa74e17..f64043e8988 100644 --- a/vespalib/src/tests/tensor/dense_tensor_operations/dense_tensor_operations_test.cpp +++ b/vespalib/src/tests/tensor/dense_tensor_operations/dense_tensor_operations_test.cpp @@ -52,7 +52,7 @@ public: } }; -const Tensor &eval_tensor(function::Node &function_ir, const TensorFunction::Input &input) { +const Tensor &eval_tensor_checked(function::Node &function_ir, const TensorFunction::Input &input) { ASSERT_TRUE(function_ir.type().is_tensor()); TensorFunction &function = function_ir; // compile step const Tensor &result = function.eval(input).as_tensor; @@ -61,18 +61,20 @@ const Tensor &eval_tensor(function::Node &function_ir, const TensorFunction::Inp } const Tensor &eval_tensor_unchecked(function::Node &function_ir, const TensorFunction::Input &input) { - ASSERT_TRUE(function_ir.type().is_tensor()); TensorFunction &function = function_ir; // compile step return function.eval(input).as_tensor; } -const Tensor &eval_tensor_unchecked_allow_invalid(function::Node &function_ir, const TensorFunction::Input &input) { - TensorFunction &function = function_ir; // compile step - return function.eval(input).as_tensor; +const Tensor &eval_tensor(function::Node &function_ir, const TensorFunction::Input &input, bool check_types) { + if (check_types) { + return eval_tensor_checked(function_ir, input); + } else { + return eval_tensor_unchecked(function_ir, input); + } } double eval_number(function::Node &function_ir, const TensorFunction::Input &input) { - ASSERT_TRUE(function_ir.type().is_number()); + ASSERT_TRUE(function_ir.type().is_double()); TensorFunction &function = function_ir; // compile step return function.eval(input).as_double; } @@ -108,46 +110,46 @@ struct Fixture } return _builder.build(); } - void assertAddImpl(const Tensor &exp, const Tensor &lhs, const Tensor &rhs) { + void assertAddImpl(const Tensor &exp, const Tensor &lhs, const Tensor &rhs, bool check_types) { MyInput input; function::Node_UP ir = function::add(function::input(lhs.getType(), input.add(lhs)), function::input(rhs.getType(), input.add(rhs))); - EXPECT_EQUAL(exp, eval_tensor(*ir, input)); + EXPECT_EQUAL(exp, eval_tensor(*ir, input, check_types)); } void assertAdd(const DenseTensorCells &exp, - const DenseTensorCells &lhs, const DenseTensorCells &rhs) { - assertAddImpl(*createTensor(exp), *createTensor(lhs), *createTensor(rhs)); + const DenseTensorCells &lhs, const DenseTensorCells &rhs, bool check_types = true) { + assertAddImpl(*createTensor(exp), *createTensor(lhs), *createTensor(rhs), check_types); } - void assertSubtractImpl(const Tensor &exp, const Tensor &lhs, const Tensor &rhs) { + void assertSubtractImpl(const Tensor &exp, const Tensor &lhs, const Tensor &rhs, bool check_types) { MyInput input; function::Node_UP ir = function::subtract(function::input(lhs.getType(), input.add(lhs)), function::input(rhs.getType(), input.add(rhs))); - EXPECT_EQUAL(exp, eval_tensor(*ir, input)); + EXPECT_EQUAL(exp, eval_tensor(*ir, input, check_types)); } void assertSubtract(const DenseTensorCells &exp, const DenseTensorCells &lhs, - const DenseTensorCells &rhs) { - assertSubtractImpl(*createTensor(exp), *createTensor(lhs), *createTensor(rhs)); + const DenseTensorCells &rhs, bool check_types = true) { + assertSubtractImpl(*createTensor(exp), *createTensor(lhs), *createTensor(rhs), check_types); } - void assertMinImpl(const Tensor &exp, const Tensor &lhs, const Tensor &rhs) { + void assertMinImpl(const Tensor &exp, const Tensor &lhs, const Tensor &rhs, bool check_types) { MyInput input; function::Node_UP ir = function::min(function::input(lhs.getType(), input.add(lhs)), function::input(rhs.getType(), input.add(rhs))); - EXPECT_EQUAL(exp, eval_tensor(*ir, input)); + EXPECT_EQUAL(exp, eval_tensor(*ir, input, check_types)); } void assertMin(const DenseTensorCells &exp, const DenseTensorCells &lhs, - const DenseTensorCells &rhs) { - assertMinImpl(*createTensor(exp), *createTensor(lhs), *createTensor(rhs)); + const DenseTensorCells &rhs, bool check_types = true) { + assertMinImpl(*createTensor(exp), *createTensor(lhs), *createTensor(rhs), check_types); } - void assertMaxImpl(const Tensor &exp, const Tensor &lhs, const Tensor &rhs) { + void assertMaxImpl(const Tensor &exp, const Tensor &lhs, const Tensor &rhs, bool check_types) { MyInput input; function::Node_UP ir = function::max(function::input(lhs.getType(), input.add(lhs)), function::input(rhs.getType(), input.add(rhs))); - EXPECT_EQUAL(exp, eval_tensor(*ir, input)); + EXPECT_EQUAL(exp, eval_tensor(*ir, input, check_types)); } void assertMax(const DenseTensorCells &exp, const DenseTensorCells &lhs, - const DenseTensorCells &rhs) { - assertMaxImpl(*createTensor(exp), *createTensor(lhs), *createTensor(rhs)); + const DenseTensorCells &rhs, bool check_types = true) { + assertMaxImpl(*createTensor(exp), *createTensor(lhs), *createTensor(rhs), check_types); } void assertSumImpl(double exp, const Tensor &tensor) { MyInput input; @@ -157,52 +159,52 @@ struct Fixture void assertSum(double exp, const DenseTensorCells &cells) { assertSumImpl(exp, *createTensor(cells)); } - void assertMatchImpl(const Tensor &exp, const Tensor &lhs, const Tensor &rhs) { + void assertMatchImpl(const Tensor &exp, const Tensor &lhs, const Tensor &rhs, bool check_types) { MyInput input; function::Node_UP ir = function::match(function::input(lhs.getType(), input.add(lhs)), function::input(rhs.getType(), input.add(rhs))); - EXPECT_EQUAL(exp, eval_tensor(*ir, input)); + EXPECT_EQUAL(exp, eval_tensor(*ir, input, check_types)); } void assertMatch(const DenseTensorCells &exp, const DenseTensorCells &lhs, - const DenseTensorCells &rhs) { - assertMatchImpl(*createTensor(exp), *createTensor(lhs), *createTensor(rhs)); + const DenseTensorCells &rhs, bool check_types = true) { + assertMatchImpl(*createTensor(exp), *createTensor(lhs), *createTensor(rhs), check_types); } void assertApplyImpl(const Tensor &exp, const Tensor &tensor, const CellFunction &func) { MyInput input; function::Node_UP ir = function::apply(function::input(tensor.getType(), input.add(tensor)), input.add(func)); - EXPECT_EQUAL(exp, eval_tensor(*ir, input)); + EXPECT_EQUAL(exp, eval_tensor_checked(*ir, input)); } void assertApply(const DenseTensorCells &exp, const DenseTensorCells &arg, const CellFunction &func) { assertApplyImpl(*createTensor(exp), *createTensor(arg), func); } - void assertDimensionSumImpl(const Tensor &exp, const Tensor &tensor, const vespalib::string &dimension) { + void assertDimensionSumImpl(const Tensor &exp, const Tensor &tensor, const vespalib::string &dimension, bool check_types) { MyInput input; function::Node_UP ir = function::dimension_sum(function::input(tensor.getType(), input.add(tensor)), dimension); - if (!ir->type().is_valid()) { + if (ir->type().is_error()) { // According to the ir, it is not allowed to sum over a // non-existing dimension. The current implementation // allows this, resulting in a tensor with no cells and // with all dimensions not sliced. - EXPECT_EQUAL(exp, eval_tensor_unchecked_allow_invalid(*ir, input)); // UNCHECKED_ALLOW_INVALID + EXPECT_EQUAL(exp, eval_tensor_unchecked(*ir, input)); } else { - EXPECT_EQUAL(exp, eval_tensor(*ir, input)); + EXPECT_EQUAL(exp, eval_tensor(*ir, input, check_types)); } } void assertDimensionSum(const DenseTensorCells &exp, const DenseTensorCells &arg, - const vespalib::string &dimension) { - assertDimensionSumImpl(*createTensor(exp), *createTensor(arg), dimension); + const vespalib::string &dimension, bool check_types = true) { + assertDimensionSumImpl(*createTensor(exp), *createTensor(arg), dimension, check_types); } - void assertMultiplyImpl(const Tensor &exp, const Tensor &lhs, const Tensor &rhs) { + void assertMultiplyImpl(const Tensor &exp, const Tensor &lhs, const Tensor &rhs, bool check_types) { MyInput input; function::Node_UP ir = function::multiply(function::input(lhs.getType(), input.add(lhs)), function::input(rhs.getType(), input.add(rhs))); - EXPECT_EQUAL(exp, eval_tensor(*ir, input)); + EXPECT_EQUAL(exp, eval_tensor(*ir, input, check_types)); } void assertMultiply(const DenseTensorCells &exp, - const DenseTensorCells &lhs, const DenseTensorCells &rhs) { - assertMultiplyImpl(*createTensor(exp), *createTensor(lhs), *createTensor(rhs)); + const DenseTensorCells &lhs, const DenseTensorCells &rhs, bool check_types = true) { + assertMultiplyImpl(*createTensor(exp), *createTensor(lhs), *createTensor(rhs), check_types); } }; @@ -213,7 +215,7 @@ template <typename FixtureType> void testTensorAdd(FixtureType &f) { - f.assertAdd({},{},{}); + f.assertAdd({},{},{}, false); f.assertAdd({ {{{"x",0}}, 8} }, { {{{"x",0}}, 3} }, { {{{"x",0}}, 5} }); @@ -232,7 +234,7 @@ template <typename FixtureType> void testTensorSubtract(FixtureType &f) { - f.assertSubtract({},{},{}); + f.assertSubtract({},{},{}, false); f.assertSubtract({ {{{"x",0}}, -2} }, { {{{"x",0}}, 3} }, { {{{"x",0}}, 5} }); @@ -251,7 +253,7 @@ template <typename FixtureType> void testTensorMin(FixtureType &f) { - f.assertMin({},{},{}); + f.assertMin({},{},{}, false); f.assertMin({ {{{"x",0}}, 3} }, { {{{"x",0}}, 3} }, { {{{"x",0}}, 5} }); @@ -270,7 +272,7 @@ template <typename FixtureType> void testTensorMax(FixtureType &f) { - f.assertMax({},{},{}); + f.assertMax({},{},{}, false); f.assertMax({ {{{"x",0}}, 5} }, { {{{"x",0}}, 3} }, { {{{"x",0}}, 5} }); @@ -300,7 +302,7 @@ template <typename FixtureType> void testTensorMatch(FixtureType &f) { - f.assertMatch({}, {}, {}); + f.assertMatch({}, {}, {}, false); f.assertMatch({ {{{"x",0}}, 15} }, { {{{"x",0}}, 3} }, { {{{"x",0}}, 5} }); @@ -328,7 +330,7 @@ template <typename FixtureType> void testTensorMultiply(FixtureType &f) { - f.assertMultiply({}, {}, {}); + f.assertMultiply({}, {}, {}, false); f.assertMultiply({ {{{"x",0}}, 15} }, { {{{"x",0}}, 3} }, { {{{"x",0}}, 5} }); @@ -446,7 +448,7 @@ testTensorSumDimension(FixtureType &f) "y"); f.assertDimensionSum({ {{}, 3} }, { {{{"x",0}}, 3} }, - "x"); + "x", false); } template <typename FixtureType> diff --git a/vespalib/src/tests/tensor/simple_tensor_builder/simple_tensor_builder_test.cpp b/vespalib/src/tests/tensor/simple_tensor_builder/simple_tensor_builder_test.cpp index 1181d56b12b..d43e1606d26 100644 --- a/vespalib/src/tests/tensor/simple_tensor_builder/simple_tensor_builder_test.cpp +++ b/vespalib/src/tests/tensor/simple_tensor_builder/simple_tensor_builder_test.cpp @@ -46,7 +46,7 @@ TEST("require that dimensions are extracted") EXPECT_EQUAL("a", dims[0]); EXPECT_EQUAL("b", dims[1]); EXPECT_EQUAL("c", dims[2]); - EXPECT_EQUAL("tensor(a{},b{},c{})", simpleTensor.getType().toSpec()); + EXPECT_EQUAL("tensor(a{},b{},c{})", simpleTensor.getType().to_spec()); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/tensor/tensor_function/tensor_function_test.cpp b/vespalib/src/tests/tensor/tensor_function/tensor_function_test.cpp index 4d64ad74209..6c38b18eaa6 100644 --- a/vespalib/src/tests/tensor/tensor_function/tensor_function_test.cpp +++ b/vespalib/src/tests/tensor/tensor_function/tensor_function_test.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/tensor/tensor_function.h> using namespace vespalib::tensor; +using vespalib::eval::ValueType; // Evaluation of tensor functions is tested in the 'tensor operations' // test. This test checks type resolving and will be extended with @@ -15,156 +16,152 @@ using namespace vespalib::tensor; // must have the same size. function::Node_UP invalid_value() { - return function::input(TensorType::invalid(), 0); + return function::input(ValueType::error_type(), 0); } function::Node_UP number_value() { - return function::sum(function::input(TensorType::sparse({}), 0)); + return function::sum(function::input(ValueType::tensor_type({}), 0)); } function::Node_UP sparse_value(const std::vector<vespalib::string> &arg) { - return function::input(TensorType::sparse(arg), 0); + std::vector<ValueType::Dimension> dimensions; + std::copy(arg.begin(), arg.end(), std::back_inserter(dimensions)); + return function::input(ValueType::tensor_type(dimensions), 0); } -function::Node_UP dense_value(std::vector<TensorType::Dimension> arg) { - return function::input(TensorType::dense(std::move(arg)), 0); +function::Node_UP dense_value(std::vector<ValueType::Dimension> arg) { + return function::input(ValueType::tensor_type(std::move(arg)), 0); } TensorAddress address(const TensorAddress::Elements &elems) { return TensorAddress(elems); } +ValueType sparse_type(const std::vector<vespalib::string> &dimensions_in) { + std::vector<ValueType::Dimension> dimensions; + std::copy(dimensions_in.begin(), dimensions_in.end(), std::back_inserter(dimensions)); + return ValueType::tensor_type(dimensions); +} + +ValueType dense_type(const std::vector<ValueType::Dimension> &dimensions_in) { + return ValueType::tensor_type(dimensions_in); +} + TEST("require that helper functions produce appropriate types") { - EXPECT_TRUE(!invalid_value()->type().is_valid()); - EXPECT_EQUAL(number_value()->type(), TensorType::number()); - EXPECT_EQUAL(sparse_value({"x", "y"})->type(), TensorType::sparse({"x", "y"})); - EXPECT_EQUAL(dense_value({{"x", 10}})->type(), TensorType::dense({{"x", 10}})); + EXPECT_TRUE(invalid_value()->type().is_error()); + EXPECT_EQUAL(number_value()->type(), ValueType::double_type()); + EXPECT_EQUAL(sparse_value({"x", "y"})->type(), sparse_type({"x", "y"})); + EXPECT_EQUAL(dense_value({{"x", 10}})->type(), dense_type({{"x", 10}})); } TEST("require that input tensors preserves type") { - EXPECT_EQUAL(TensorType::sparse({"x", "y"}), - function::input(TensorType::sparse({"x", "y"}), 0)->type()); - EXPECT_EQUAL(TensorType::dense({{"x", 10}}), - function::input(TensorType::dense({{"x", 10}}), 0)->type()); + EXPECT_EQUAL(sparse_type({"x", "y"}), + function::input(sparse_type({"x", "y"}), 0)->type()); + EXPECT_EQUAL(dense_type({{"x", 10}}), + function::input(dense_type({{"x", 10}}), 0)->type()); } TEST("require that input tensors with non-tensor types are invalid") { - EXPECT_TRUE(!function::input(TensorType::invalid(), 0)->type().is_valid()); - EXPECT_TRUE(!function::input(TensorType::number(), 0)->type().is_valid()); + EXPECT_TRUE(function::input(ValueType::error_type(), 0)->type().is_error()); } TEST("require that sum of tensor gives number as result") { - EXPECT_EQUAL(TensorType::number(), function::sum(sparse_value({}))->type()); - EXPECT_EQUAL(TensorType::number(), function::sum(dense_value({}))->type()); + EXPECT_EQUAL(ValueType::double_type(), function::sum(sparse_value({}))->type()); + EXPECT_EQUAL(ValueType::double_type(), function::sum(dense_value({}))->type()); } TEST("require that dimension sum removes the summed dimension") { - EXPECT_EQUAL(TensorType::sparse({"x", "y"}), + EXPECT_EQUAL(sparse_type({"x", "y"}), function::dimension_sum(sparse_value({"x", "y", "z"}), "z")->type()); - EXPECT_EQUAL(TensorType::dense({{"y", 10}}), + EXPECT_EQUAL(dense_type({{"y", 10}}), function::dimension_sum(dense_value({{"x", 10}, {"y", 10}}), "x")->type()); } TEST("require that dimension sum over non-existing dimension is invalid") { - EXPECT_TRUE(!function::dimension_sum(sparse_value({"x", "y", "z"}), "w")->type().is_valid()); - EXPECT_TRUE(!function::dimension_sum(dense_value({{"x", 10}, {"y", 10}}), "z")->type().is_valid()); + EXPECT_TRUE(function::dimension_sum(sparse_value({"x", "y", "z"}), "w")->type().is_error()); + EXPECT_TRUE(function::dimension_sum(dense_value({{"x", 10}, {"y", 10}}), "z")->type().is_error()); } TEST("require that apply preserves tensor type") { - EXPECT_EQUAL(TensorType::sparse({"x", "y"}), + EXPECT_EQUAL(sparse_type({"x", "y"}), function::apply(sparse_value({"x", "y"}), 0)->type()); - EXPECT_EQUAL(TensorType::dense({{"x", 10}}), + EXPECT_EQUAL(dense_type({{"x", 10}}), function::apply(dense_value({{"x", 10}}), 0)->type()); } TEST("require that tensor add result has union of input dimensions") { - EXPECT_EQUAL(TensorType::sparse({"x", "y", "z"}), + EXPECT_EQUAL(sparse_type({"x", "y", "z"}), function::add(sparse_value({"x", "y"}), sparse_value({"y", "z"}))->type()); - EXPECT_EQUAL(TensorType::sparse({{"x", 10}, {"y", 10}, {"z", 10}}), - function::add(sparse_value({{"x", 10}, {"y", 10}}), - sparse_value({{"y", 10}, {"z", 10}}))->type()); + EXPECT_EQUAL(dense_type({{"x", 10}, {"y", 10}, {"z", 10}}), + function::add(dense_value({{"x", 10}, {"y", 10}}), + dense_value({{"y", 10}, {"z", 10}}))->type()); } TEST("require that tensor subtract result has union of input dimensions") { - EXPECT_EQUAL(TensorType::sparse({"x", "y", "z"}), + EXPECT_EQUAL(sparse_type({"x", "y", "z"}), function::subtract(sparse_value({"x", "y"}), sparse_value({"y", "z"}))->type()); - EXPECT_EQUAL(TensorType::sparse({{"x", 10}, {"y", 10}, {"z", 10}}), - function::subtract(sparse_value({{"x", 10}, {"y", 10}}), - sparse_value({{"y", 10}, {"z", 10}}))->type()); + EXPECT_EQUAL(dense_type({{"x", 10}, {"y", 10}, {"z", 10}}), + function::subtract(dense_value({{"x", 10}, {"y", 10}}), + dense_value({{"y", 10}, {"z", 10}}))->type()); } TEST("require that tensor multiply result has union of input dimensions") { - EXPECT_EQUAL(TensorType::sparse({"x", "y", "z"}), + EXPECT_EQUAL(sparse_type({"x", "y", "z"}), function::multiply(sparse_value({"x", "y"}), sparse_value({"y", "z"}))->type()); - EXPECT_EQUAL(TensorType::sparse({{"x", 10}, {"y", 10}, {"z", 10}}), - function::multiply(sparse_value({{"x", 10}, {"y", 10}}), - sparse_value({{"y", 10}, {"z", 10}}))->type()); + EXPECT_EQUAL(dense_type({{"x", 10}, {"y", 10}, {"z", 10}}), + function::multiply(dense_value({{"x", 10}, {"y", 10}}), + dense_value({{"y", 10}, {"z", 10}}))->type()); } TEST("require that tensor min result has union of input dimensions") { - EXPECT_EQUAL(TensorType::sparse({"x", "y", "z"}), + EXPECT_EQUAL(sparse_type({"x", "y", "z"}), function::min(sparse_value({"x", "y"}), sparse_value({"y", "z"}))->type()); - EXPECT_EQUAL(TensorType::sparse({{"x", 10}, {"y", 10}, {"z", 10}}), - function::min(sparse_value({{"x", 10}, {"y", 10}}), - sparse_value({{"y", 10}, {"z", 10}}))->type()); + EXPECT_EQUAL(dense_type({{"x", 10}, {"y", 10}, {"z", 10}}), + function::min(dense_value({{"x", 10}, {"y", 10}}), + dense_value({{"y", 10}, {"z", 10}}))->type()); } TEST("require that tensor max result has union of input dimensions") { - EXPECT_EQUAL(TensorType::sparse({"x", "y", "z"}), + EXPECT_EQUAL(sparse_type({"x", "y", "z"}), function::max(sparse_value({"x", "y"}), sparse_value({"y", "z"}))->type()); - EXPECT_EQUAL(TensorType::sparse({{"x", 10}, {"y", 10}, {"z", 10}}), - function::max(sparse_value({{"x", 10}, {"y", 10}}), - sparse_value({{"y", 10}, {"z", 10}}))->type()); + EXPECT_EQUAL(dense_type({{"x", 10}, {"y", 10}, {"z", 10}}), + function::max(dense_value({{"x", 10}, {"y", 10}}), + dense_value({{"y", 10}, {"z", 10}}))->type()); } TEST("require that tensor match result has intersection of input dimensions") { - EXPECT_EQUAL(TensorType::sparse({"y"}), + EXPECT_EQUAL(sparse_type({"y"}), function::match(sparse_value({"x", "y"}), sparse_value({"y", "z"}))->type()); - EXPECT_EQUAL(TensorType::sparse({{"y", 10}}), - function::match(sparse_value({{"x", 10}, {"y", 10}}), - sparse_value({{"y", 10}, {"z", 10}}))->type()); -} - -TEST("require that sparse and dense tensors cannot be directly combined") { - EXPECT_TRUE(!function::add(sparse_value({}), dense_value({}))->type().is_valid()); - EXPECT_TRUE(!function::add(sparse_value({}), dense_value({}))->type().is_valid()); - EXPECT_TRUE(!function::subtract(sparse_value({}), dense_value({}))->type().is_valid()); - EXPECT_TRUE(!function::subtract(sparse_value({}), dense_value({}))->type().is_valid()); - EXPECT_TRUE(!function::multiply(sparse_value({}), dense_value({}))->type().is_valid()); - EXPECT_TRUE(!function::multiply(sparse_value({}), dense_value({}))->type().is_valid()); - EXPECT_TRUE(!function::min(sparse_value({}), dense_value({}))->type().is_valid()); - EXPECT_TRUE(!function::min(sparse_value({}), dense_value({}))->type().is_valid()); - EXPECT_TRUE(!function::max(sparse_value({}), dense_value({}))->type().is_valid()); - EXPECT_TRUE(!function::max(sparse_value({}), dense_value({}))->type().is_valid()); - EXPECT_TRUE(!function::match(sparse_value({}), dense_value({}))->type().is_valid()); - EXPECT_TRUE(!function::match(sparse_value({}), dense_value({}))->type().is_valid()); + EXPECT_EQUAL(dense_type({{"y", 10}}), + function::match(dense_value({{"x", 10}, {"y", 10}}), + dense_value({{"y", 10}, {"z", 10}}))->type()); } TEST("require that tensor operations on non-tensor types are invalid") { - EXPECT_TRUE(!function::sum(invalid_value())->type().is_valid()); - EXPECT_TRUE(!function::sum(number_value())->type().is_valid()); - EXPECT_TRUE(!function::dimension_sum(invalid_value(), "x")->type().is_valid()); - EXPECT_TRUE(!function::dimension_sum(number_value(), "x")->type().is_valid()); - EXPECT_TRUE(!function::apply(invalid_value(), 0)->type().is_valid()); - EXPECT_TRUE(!function::apply(number_value(), 0)->type().is_valid()); - EXPECT_TRUE(!function::add(invalid_value(), invalid_value())->type().is_valid()); - EXPECT_TRUE(!function::add(number_value(), number_value())->type().is_valid()); - EXPECT_TRUE(!function::subtract(invalid_value(), invalid_value())->type().is_valid()); - EXPECT_TRUE(!function::subtract(number_value(), number_value())->type().is_valid()); - EXPECT_TRUE(!function::multiply(invalid_value(), invalid_value())->type().is_valid()); - EXPECT_TRUE(!function::multiply(number_value(), number_value())->type().is_valid()); - EXPECT_TRUE(!function::min(invalid_value(), invalid_value())->type().is_valid()); - EXPECT_TRUE(!function::min(number_value(), number_value())->type().is_valid()); - EXPECT_TRUE(!function::max(invalid_value(), invalid_value())->type().is_valid()); - EXPECT_TRUE(!function::max(number_value(), number_value())->type().is_valid()); - EXPECT_TRUE(!function::match(invalid_value(), invalid_value())->type().is_valid()); - EXPECT_TRUE(!function::match(number_value(), number_value())->type().is_valid()); + EXPECT_TRUE(function::sum(invalid_value())->type().is_error()); + EXPECT_TRUE(function::sum(number_value())->type().is_error()); + EXPECT_TRUE(function::dimension_sum(invalid_value(), "x")->type().is_error()); + EXPECT_TRUE(function::dimension_sum(number_value(), "x")->type().is_error()); + EXPECT_TRUE(function::apply(invalid_value(), 0)->type().is_error()); + EXPECT_TRUE(function::apply(number_value(), 0)->type().is_error()); + EXPECT_TRUE(function::add(invalid_value(), invalid_value())->type().is_error()); + EXPECT_TRUE(function::add(number_value(), number_value())->type().is_error()); + EXPECT_TRUE(function::subtract(invalid_value(), invalid_value())->type().is_error()); + EXPECT_TRUE(function::subtract(number_value(), number_value())->type().is_error()); + EXPECT_TRUE(function::multiply(invalid_value(), invalid_value())->type().is_error()); + EXPECT_TRUE(function::multiply(number_value(), number_value())->type().is_error()); + EXPECT_TRUE(function::min(invalid_value(), invalid_value())->type().is_error()); + EXPECT_TRUE(function::min(number_value(), number_value())->type().is_error()); + EXPECT_TRUE(function::max(invalid_value(), invalid_value())->type().is_error()); + EXPECT_TRUE(function::max(number_value(), number_value())->type().is_error()); + EXPECT_TRUE(function::match(invalid_value(), invalid_value())->type().is_error()); + EXPECT_TRUE(function::match(number_value(), number_value())->type().is_error()); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp b/vespalib/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp index 190ac0bf015..21e964b95e1 100644 --- a/vespalib/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp +++ b/vespalib/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp @@ -18,6 +18,7 @@ #include <vespa/vespalib/tensor/default_tensor.h> #include <ostream> +using vespalib::eval::ValueType; using namespace vespalib::tensor; namespace vespalib { @@ -58,6 +59,28 @@ struct TensorTFromBuilder<CompactTensorV2Builder> { template <typename BuilderType> using TensorTFromBuilder_t = typename TensorTFromBuilder<BuilderType>::TensorT; +bool +hasOnlyMappedDimensions(const ValueType &type) +{ + for (const auto &dim : type.dimensions()) { + if (!dim.is_mapped()) { + return false; + } + } + return true; +} + +bool +hasOnlyIndexedDimensions(const ValueType &type) +{ + for (const auto &dim : type.dimensions()) { + if (!dim.is_indexed()) { + return false; + } + } + return true; +} + struct FixtureBase { Tensor::UP createDenseTensor(const DenseTensorCells &cells) { @@ -78,10 +101,10 @@ struct Fixture : public FixtureBase } void assertSparseMapImpl(const Tensor &exp, - const TensorType &tensorType, + const ValueType &tensorType, const Tensor &rhs, bool isDefaultBuilder) { - EXPECT_TRUE(tensorType.type() == TensorType::Type::SPARSE); + EXPECT_TRUE(hasOnlyMappedDimensions(tensorType)); if (isDefaultBuilder) { TensorMapper mapper(tensorType); std::unique_ptr<Tensor> mapped = mapper.map(rhs); @@ -95,10 +118,10 @@ struct Fixture : public FixtureBase } void assertDenseMapImpl(const Tensor &exp, - const TensorType &tensorType, + const ValueType &tensorType, const Tensor &rhs) { - EXPECT_TRUE(tensorType.type() == TensorType::Type::DENSE); + EXPECT_TRUE(hasOnlyIndexedDimensions(tensorType)); TensorMapper mapper(tensorType); std::unique_ptr<Tensor> mapped = mapper.map(rhs); EXPECT_TRUE(!!mapped); @@ -113,7 +136,7 @@ struct Fixture : public FixtureBase const TensorDimensions &rhsDimensions) { assertSparseMapImpl(*createTensor(expTensor, expDimensions), - TensorType::fromSpec(typeSpec), + ValueType::from_spec(typeSpec), *createTensor(rhsTensor, rhsDimensions), defaultBuilder<BuilderType>()); } @@ -125,7 +148,7 @@ struct Fixture : public FixtureBase const TensorDimensions &rhsDimensions) { assertDenseMapImpl(*createDenseTensor(expTensor), - TensorType::fromSpec(typeSpec), + ValueType::from_spec(typeSpec), *createTensor(rhsTensor, rhsDimensions)); } }; diff --git a/vespalib/src/tests/tensor/tensor_operations/tensor_operations_test.cpp b/vespalib/src/tests/tensor/tensor_operations/tensor_operations_test.cpp index b079c4f64c3..3948a6d68a6 100644 --- a/vespalib/src/tests/tensor/tensor_operations/tensor_operations_test.cpp +++ b/vespalib/src/tests/tensor/tensor_operations/tensor_operations_test.cpp @@ -54,7 +54,7 @@ public: } }; -const Tensor &eval_tensor(function::Node &function_ir, const TensorFunction::Input &input) { +const Tensor &eval_tensor_checked(function::Node &function_ir, const TensorFunction::Input &input) { ASSERT_TRUE(function_ir.type().is_tensor()); TensorFunction &function = function_ir; // compile step const Tensor &result = function.eval(input).as_tensor; @@ -63,13 +63,20 @@ const Tensor &eval_tensor(function::Node &function_ir, const TensorFunction::Inp } const Tensor &eval_tensor_unchecked(function::Node &function_ir, const TensorFunction::Input &input) { - ASSERT_TRUE(function_ir.type().is_tensor()); TensorFunction &function = function_ir; // compile step return function.eval(input).as_tensor; } +const Tensor &eval_tensor(function::Node &function_ir, const TensorFunction::Input &input, bool check_types) { + if (check_types) { + return eval_tensor_checked(function_ir, input); + } else { + return eval_tensor_unchecked(function_ir, input); + } +} + double eval_number(function::Node &function_ir, const TensorFunction::Input &input) { - ASSERT_TRUE(function_ir.type().is_number()); + ASSERT_TRUE(function_ir.type().is_double()); TensorFunction &function = function_ir; // compile step return function.eval(input).as_double; } @@ -108,41 +115,41 @@ struct Fixture EXPECT_NOT_EQUAL(*createTensor(lhs, lhsDimensions), *createTensor(rhs, rhsDimensions)); } - void assertAddImpl(const Tensor &exp, const Tensor &lhs, const Tensor &rhs) { + void assertAddImpl(const Tensor &exp, const Tensor &lhs, const Tensor &rhs, bool check_types) { MyInput input; function::Node_UP ir = function::add(function::input(lhs.getType(), input.add(lhs)), function::input(rhs.getType(), input.add(rhs))); - EXPECT_EQUAL(exp, eval_tensor(*ir, input)); + EXPECT_EQUAL(exp, eval_tensor(*ir, input, check_types)); } - void assertAdd(const TensorCells &exp, const TensorCells &lhs, const TensorCells &rhs) { - assertAddImpl(*createTensor(exp), *createTensor(lhs), *createTensor(rhs)); + void assertAdd(const TensorCells &exp, const TensorCells &lhs, const TensorCells &rhs, bool check_types = true) { + assertAddImpl(*createTensor(exp), *createTensor(lhs), *createTensor(rhs), check_types); } - void assertSubtractImpl(const Tensor &exp, const Tensor &lhs, const Tensor &rhs) { + void assertSubtractImpl(const Tensor &exp, const Tensor &lhs, const Tensor &rhs, bool check_types) { MyInput input; function::Node_UP ir = function::subtract(function::input(lhs.getType(), input.add(lhs)), function::input(rhs.getType(), input.add(rhs))); - EXPECT_EQUAL(exp, eval_tensor(*ir, input)); + EXPECT_EQUAL(exp, eval_tensor(*ir, input, check_types)); } - void assertSubtract(const TensorCells &exp, const TensorCells &lhs, const TensorCells &rhs) { - assertSubtractImpl(*createTensor(exp), *createTensor(lhs), *createTensor(rhs)); + void assertSubtract(const TensorCells &exp, const TensorCells &lhs, const TensorCells &rhs, bool check_types = true) { + assertSubtractImpl(*createTensor(exp), *createTensor(lhs), *createTensor(rhs), check_types); } - void assertMinImpl(const Tensor &exp, const Tensor &lhs, const Tensor &rhs) { + void assertMinImpl(const Tensor &exp, const Tensor &lhs, const Tensor &rhs, bool check_types) { MyInput input; function::Node_UP ir = function::min(function::input(lhs.getType(), input.add(lhs)), function::input(rhs.getType(), input.add(rhs))); - EXPECT_EQUAL(exp, eval_tensor(*ir, input)); + EXPECT_EQUAL(exp, eval_tensor(*ir, input, check_types)); } - void assertMin(const TensorCells &exp, const TensorCells &lhs, const TensorCells &rhs) { - assertMinImpl(*createTensor(exp), *createTensor(lhs), *createTensor(rhs)); + void assertMin(const TensorCells &exp, const TensorCells &lhs, const TensorCells &rhs, bool check_types = true) { + assertMinImpl(*createTensor(exp), *createTensor(lhs), *createTensor(rhs), check_types); } - void assertMaxImpl(const Tensor &exp, const Tensor &lhs, const Tensor &rhs) { + void assertMaxImpl(const Tensor &exp, const Tensor &lhs, const Tensor &rhs, bool check_types) { MyInput input; function::Node_UP ir = function::max(function::input(lhs.getType(), input.add(lhs)), function::input(rhs.getType(), input.add(rhs))); - EXPECT_EQUAL(exp, eval_tensor(*ir, input)); + EXPECT_EQUAL(exp, eval_tensor(*ir, input, check_types)); } - void assertMax(const TensorCells &exp, const TensorCells &lhs, const TensorCells &rhs) { - assertMaxImpl(*createTensor(exp), *createTensor(lhs), *createTensor(rhs)); + void assertMax(const TensorCells &exp, const TensorCells &lhs, const TensorCells &rhs, bool check_types = true) { + assertMaxImpl(*createTensor(exp), *createTensor(lhs), *createTensor(rhs), check_types); } void assertSumImpl(double exp, const Tensor &tensor) { MyInput input; @@ -170,18 +177,18 @@ struct Fixture const TensorCells &lhs, const TensorCells &rhs) { assertMatchImpl(*createTensor(expTensor, expDimensions), *createTensor(lhs), *createTensor(rhs)); } - void assertMultiplyImpl(const Tensor &exp, const Tensor &lhs, const Tensor &rhs) { + void assertMultiplyImpl(const Tensor &exp, const Tensor &lhs, const Tensor &rhs, bool check_types) { MyInput input; function::Node_UP ir = function::multiply(function::input(lhs.getType(), input.add(lhs)), function::input(rhs.getType(), input.add(rhs))); - EXPECT_EQUAL(exp, eval_tensor(*ir, input)); + EXPECT_EQUAL(exp, eval_tensor(*ir, input, check_types)); } - void assertMultiply(const TensorCells &exp, const TensorCells &lhs, const TensorCells &rhs) { - assertMultiplyImpl(*createTensor(exp), *createTensor(lhs), *createTensor(rhs)); + void assertMultiply(const TensorCells &exp, const TensorCells &lhs, const TensorCells &rhs, bool check_types = true) { + assertMultiplyImpl(*createTensor(exp), *createTensor(lhs), *createTensor(rhs), check_types); } void assertMultiply(const TensorCells &expTensor, const TensorDimensions &expDimensions, const TensorCells &lhs, const TensorCells &rhs) { - assertMultiplyImpl(*createTensor(expTensor, expDimensions), *createTensor(lhs), *createTensor(rhs)); + assertMultiplyImpl(*createTensor(expTensor, expDimensions), *createTensor(lhs), *createTensor(rhs), true); } void assertMultiplyImpl(const Tensor &exp, const Tensor &arg1, const Tensor &arg2, const Tensor &arg3) { MyInput input; @@ -189,7 +196,7 @@ struct Fixture function::multiply(function::input(arg1.getType(), input.add(arg1)), function::input(arg2.getType(), input.add(arg2))), function::input(arg3.getType(), input.add(arg3))); - EXPECT_EQUAL(exp, eval_tensor(*ir, input)); + EXPECT_EQUAL(exp, eval_tensor_checked(*ir, input)); } void assertMultiply(const TensorCells &expTensor, const TensorDimensions &expDimensions, const TensorCells &arg1, const TensorCells &arg2, const TensorCells &arg3) { @@ -198,7 +205,7 @@ struct Fixture void assertApplyImpl(const Tensor &exp, const Tensor &tensor, const CellFunction &func) { MyInput input; function::Node_UP ir = function::apply(function::input(tensor.getType(), input.add(tensor)), input.add(func)); - EXPECT_EQUAL(exp, eval_tensor(*ir, input)); + EXPECT_EQUAL(exp, eval_tensor_checked(*ir, input)); } void assertApply(const TensorCells &exp, const TensorCells &arg, const CellFunction &func) { assertApplyImpl(*createTensor(exp), *createTensor(arg), func); @@ -206,7 +213,7 @@ struct Fixture void assertDimensionSumImpl(const Tensor &exp, const Tensor &tensor, const vespalib::string &dimension) { MyInput input; function::Node_UP ir = function::dimension_sum(function::input(tensor.getType(), input.add(tensor)), dimension); - EXPECT_EQUAL(exp, eval_tensor(*ir, input)); + EXPECT_EQUAL(exp, eval_tensor_checked(*ir, input)); } void assertDimensionSum(const TensorCells &exp, const TensorCells &arg, const vespalib::string &dimension) { @@ -250,7 +257,7 @@ template <typename FixtureType> void testTensorAdd(FixtureType &f) { - f.assertAdd({},{},{}); + f.assertAdd({},{},{}, false); f.assertAdd({ {{{"x","1"}}, 3}, {{{"x","2"}}, 5} }, { {{{"x","1"}}, 3} }, { {{{"x","2"}}, 5} }); @@ -293,7 +300,7 @@ template <typename FixtureType> void testTensorSubtract(FixtureType &f) { - f.assertSubtract({},{},{}); + f.assertSubtract({},{},{}, false); f.assertSubtract({ {{{"x","1"}}, 3}, {{{"x","2"}}, -5} }, { {{{"x","1"}}, 3} }, { {{{"x","2"}}, 5} }); @@ -336,7 +343,7 @@ template <typename FixtureType> void testTensorMin(FixtureType &f) { - f.assertMin({},{},{}); + f.assertMin({},{},{}, false); f.assertMin({ {{{"x","1"}}, 3}, {{{"x","2"}}, 5} }, { {{{"x","1"}}, 3} }, { {{{"x","2"}}, 5} }); @@ -379,7 +386,7 @@ template <typename FixtureType> void testTensorMax(FixtureType &f) { - f.assertMax({},{},{}); + f.assertMax({},{},{}, false); f.assertMax({ {{{"x","1"}}, 3}, {{{"x","2"}}, 5} }, { {{{"x","1"}}, 3} }, { {{{"x","2"}}, 5} }); @@ -488,7 +495,7 @@ template <typename FixtureType> void testTensorMultiply(FixtureType &f) { - f.assertMultiply({}, {}, {}); + f.assertMultiply({}, {}, {}, false); f.assertMultiply({}, {"x"}, { {{{"x","1"}}, 3} }, { {{{"x","2"}}, 5} }); diff --git a/vespalib/src/vespa/vespalib/tensor/compact/compact_tensor.cpp b/vespalib/src/vespa/vespalib/tensor/compact/compact_tensor.cpp index c2fa7b762af..735967e95af 100644 --- a/vespalib/src/vespa/vespalib/tensor/compact/compact_tensor.cpp +++ b/vespalib/src/vespa/vespalib/tensor/compact/compact_tensor.cpp @@ -65,10 +65,12 @@ CompactTensor::combineDimensionsWith(const CompactTensor &rhs) const return result; } -TensorType +eval::ValueType CompactTensor::getType() const { - return TensorType::sparse(_dimensions); + std::vector<eval::ValueType::Dimension> dimensions; + std::copy(_dimensions.begin(), _dimensions.end(), std::back_inserter(dimensions)); + return eval::ValueType::tensor_type(dimensions); } double diff --git a/vespalib/src/vespa/vespalib/tensor/compact/compact_tensor.h b/vespalib/src/vespa/vespalib/tensor/compact/compact_tensor.h index 2f32baa9b4c..71431d36461 100644 --- a/vespalib/src/vespa/vespalib/tensor/compact/compact_tensor.h +++ b/vespalib/src/vespa/vespalib/tensor/compact/compact_tensor.h @@ -42,7 +42,7 @@ public: bool operator==(const CompactTensor &rhs) const; Dimensions combineDimensionsWith(const CompactTensor &rhs) const; - virtual TensorType getType() const override; + virtual eval::ValueType getType() const override; virtual double sum() const override; virtual Tensor::UP add(const Tensor &arg) const override; virtual Tensor::UP subtract(const Tensor &arg) const override; diff --git a/vespalib/src/vespa/vespalib/tensor/compact/compact_tensor_v2.cpp b/vespalib/src/vespa/vespalib/tensor/compact/compact_tensor_v2.cpp index 48962791e59..55f86b6b214 100644 --- a/vespalib/src/vespa/vespalib/tensor/compact/compact_tensor_v2.cpp +++ b/vespalib/src/vespa/vespalib/tensor/compact/compact_tensor_v2.cpp @@ -67,10 +67,12 @@ CompactTensorV2::combineDimensionsWith(const CompactTensorV2 &rhs) const return result; } -TensorType +eval::ValueType CompactTensorV2::getType() const { - return TensorType::sparse(_dimensions); + std::vector<eval::ValueType::Dimension> dimensions; + std::copy(_dimensions.begin(), _dimensions.end(), std::back_inserter(dimensions)); + return eval::ValueType::tensor_type(dimensions); } double diff --git a/vespalib/src/vespa/vespalib/tensor/compact/compact_tensor_v2.h b/vespalib/src/vespa/vespalib/tensor/compact/compact_tensor_v2.h index d45dd4a0004..d7a3f3e4e5c 100644 --- a/vespalib/src/vespa/vespalib/tensor/compact/compact_tensor_v2.h +++ b/vespalib/src/vespa/vespalib/tensor/compact/compact_tensor_v2.h @@ -42,7 +42,7 @@ public: bool operator==(const CompactTensorV2 &rhs) const; Dimensions combineDimensionsWith(const CompactTensorV2 &rhs) const; - virtual TensorType getType() const override; + virtual eval::ValueType getType() const override; virtual double sum() const override; virtual Tensor::UP add(const Tensor &arg) const override; virtual Tensor::UP subtract(const Tensor &arg) const override; diff --git a/vespalib/src/vespa/vespalib/tensor/default_tensor_engine.cpp b/vespalib/src/vespa/vespalib/tensor/default_tensor_engine.cpp index bf76a7bd9e3..6b38d66d62c 100644 --- a/vespalib/src/vespa/vespalib/tensor/default_tensor_engine.cpp +++ b/vespalib/src/vespa/vespalib/tensor/default_tensor_engine.cpp @@ -23,7 +23,7 @@ DefaultTensorEngine::type_of(const Tensor &tensor) const { assert(&tensor.engine() == this); const tensor::Tensor &my_tensor = static_cast<const tensor::Tensor &>(tensor); - return my_tensor.getType().as_value_type(); + return my_tensor.getType(); } bool diff --git a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor.cpp b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor.cpp index 1f1b318d9a2..5d2388cd3b9 100644 --- a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor.cpp +++ b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor.cpp @@ -177,16 +177,16 @@ DenseTensor::operator==(const DenseTensor &rhs) const (_cells == rhs._cells); } -TensorType +eval::ValueType DenseTensor::getType() const { - std::vector<TensorType::Dimension> dimensions; + std::vector<eval::ValueType::Dimension> dimensions; dimensions.reserve(_dimensionsMeta.size()); for (const auto &dimensionMeta : _dimensionsMeta) { dimensions.emplace_back(dimensionMeta.dimension(), dimensionMeta.size()); } - return TensorType::dense(dimensions); + return eval::ValueType::tensor_type(dimensions); } double diff --git a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor.h b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor.h index 7b4adc8e7a4..73d9c26c408 100644 --- a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor.h +++ b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor.h @@ -89,7 +89,7 @@ public: bool operator==(const DenseTensor &rhs) const; CellsIterator cellsIterator() const { return CellsIterator(_dimensionsMeta, _cells); } - virtual TensorType getType() const override; + virtual eval::ValueType getType() const override; virtual double sum() const override; virtual Tensor::UP add(const Tensor &arg) const override; virtual Tensor::UP subtract(const Tensor &arg) const override; diff --git a/vespalib/src/vespa/vespalib/tensor/serialization/compact_binary_format.cpp b/vespalib/src/vespa/vespalib/tensor/serialization/compact_binary_format.cpp index f2871025f58..48b5cb971d6 100644 --- a/vespalib/src/vespa/vespalib/tensor/serialization/compact_binary_format.cpp +++ b/vespalib/src/vespa/vespalib/tensor/serialization/compact_binary_format.cpp @@ -22,7 +22,7 @@ vespalib::string undefinedLabel(""); void writeTensorAddress(nbostream &output, - const TensorType &type, + const eval::ValueType &type, const TensorAddress &value) { auto elemItr = value.elements().cbegin(); @@ -44,7 +44,7 @@ class CompactBinaryFormatSerializer : public TensorVisitor { uint32_t _numCells; nbostream _cells; - TensorType _type; + eval::ValueType _type; public: CompactBinaryFormatSerializer(); @@ -56,7 +56,7 @@ public: CompactBinaryFormatSerializer::CompactBinaryFormatSerializer() : _numCells(0u), _cells(), - _type(TensorType::invalid()) + _type(eval::ValueType::error_type()) { } diff --git a/vespalib/src/vespa/vespalib/tensor/serialization/slime_binary_format.cpp b/vespalib/src/vespa/vespalib/tensor/serialization/slime_binary_format.cpp index 1e746c2a9a8..0bdcbe2c124 100644 --- a/vespalib/src/vespa/vespalib/tensor/serialization/slime_binary_format.cpp +++ b/vespalib/src/vespa/vespalib/tensor/serialization/slime_binary_format.cpp @@ -77,7 +77,7 @@ SlimeBinaryFormatSerializer::visit(const TensorAddress &address, void SlimeBinaryFormatSerializer::serialize(const Tensor &tensor) { - TensorType type(tensor.getType()); + eval::ValueType type(tensor.getType()); for (const auto & dimension : type.dimensions()) { _dimensions.addString(Memory(dimension.name)); } diff --git a/vespalib/src/vespa/vespalib/tensor/simple/simple_tensor.cpp b/vespalib/src/vespa/vespalib/tensor/simple/simple_tensor.cpp index cd1dd935fe8..e4c617824a7 100644 --- a/vespalib/src/vespa/vespalib/tensor/simple/simple_tensor.cpp +++ b/vespalib/src/vespa/vespalib/tensor/simple/simple_tensor.cpp @@ -40,10 +40,12 @@ SimpleTensor::combineDimensionsWith(const SimpleTensor &rhs) const return result; } -TensorType +eval::ValueType SimpleTensor::getType() const { - return TensorType::sparse(_dimensions); + std::vector<eval::ValueType::Dimension> dimensions; + std::copy(_dimensions.begin(), _dimensions.end(), std::back_inserter(dimensions)); + return eval::ValueType::tensor_type(dimensions); } double diff --git a/vespalib/src/vespa/vespalib/tensor/simple/simple_tensor.h b/vespalib/src/vespa/vespalib/tensor/simple/simple_tensor.h index 0525649ff57..aaa3c274f12 100644 --- a/vespalib/src/vespa/vespalib/tensor/simple/simple_tensor.h +++ b/vespalib/src/vespa/vespalib/tensor/simple/simple_tensor.h @@ -39,7 +39,7 @@ public: bool operator==(const SimpleTensor &rhs) const; Dimensions combineDimensionsWith(const SimpleTensor &rhs) const; - virtual TensorType getType() const override; + virtual eval::ValueType getType() const override; virtual double sum() const override; virtual Tensor::UP add(const Tensor &arg) const override; virtual Tensor::UP subtract(const Tensor &arg) const override; diff --git a/vespalib/src/vespa/vespalib/tensor/tensor.h b/vespalib/src/vespa/vespalib/tensor/tensor.h index a38c1c6bfcc..4128a27d9a7 100644 --- a/vespalib/src/vespa/vespalib/tensor/tensor.h +++ b/vespalib/src/vespa/vespalib/tensor/tensor.h @@ -4,9 +4,9 @@ #include "cell_function.h" #include "tensor_address.h" -#include "tensor_type.h" #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/eval/tensor.h> +#include <vespa/vespalib/eval/value_type.h> namespace vespalib { namespace tensor { @@ -27,7 +27,7 @@ struct Tensor : public eval::Tensor Tensor(); virtual ~Tensor() {} - virtual TensorType getType() const = 0; + virtual eval::ValueType getType() const = 0; virtual double sum() const = 0; virtual Tensor::UP add(const Tensor &arg) const = 0; virtual Tensor::UP subtract(const Tensor &arg) const = 0; diff --git a/vespalib/src/vespa/vespalib/tensor/tensor_function.cpp b/vespalib/src/vespa/vespalib/tensor/tensor_function.cpp index a89dd3c22de..2c47a8e4c03 100644 --- a/vespalib/src/vespa/vespalib/tensor/tensor_function.cpp +++ b/vespalib/src/vespa/vespalib/tensor/tensor_function.cpp @@ -2,6 +2,7 @@ #include <vespa/fastos/fastos.h> #include "tensor_function.h" +#include <vespa/vespalib/eval/value_type.h> namespace vespalib { namespace tensor { @@ -16,10 +17,10 @@ namespace { class FunctionBase : public Node { private: - TensorType _type; + eval::ValueType _type; protected: - explicit FunctionBase(const TensorType &type_in) : _type(type_in) {} - const TensorType &type() const override { return _type; } + explicit FunctionBase(const eval::ValueType &type_in) : _type(type_in) {} + const eval::ValueType &type() const override { return _type; } // helper function used to unwrap tensor value from eval result static const Tensor &eval_tensor(Node &node, const Input &input) { @@ -37,7 +38,7 @@ class TensorCache : public FunctionBase private: Tensor::UP _my_result; protected: - explicit TensorCache(const TensorType &type_in) + explicit TensorCache(const eval::ValueType &type_in) : FunctionBase(type_in), _my_result() {} const Tensor &store_tensor(Tensor::UP result) { _my_result = std::move(result); @@ -55,16 +56,16 @@ class InputTensor : public FunctionBase private: size_t _tensor_id; - static TensorType infer_type(const TensorType &type_in) { + static eval::ValueType infer_type(const eval::ValueType &type_in) { if (type_in.is_tensor()) { return type_in; } else { - return TensorType::invalid(); + return eval::ValueType::error_type(); } } public: - InputTensor(const TensorType &type_in, size_t tensor_id) + InputTensor(const eval::ValueType &type_in, size_t tensor_id) : FunctionBase(infer_type(type_in)), _tensor_id(tensor_id) {} Result eval(const Input &input) override { return input.get_tensor(_tensor_id); @@ -81,11 +82,11 @@ class Sum : public FunctionBase private: Node_UP _child; - static TensorType infer_type(const TensorType &child_type) { + static eval::ValueType infer_type(const eval::ValueType &child_type) { if (child_type.is_tensor()) { - return TensorType::number(); + return eval::ValueType::double_type(); } else { - return TensorType::invalid(); + return eval::ValueType::error_type(); } } @@ -110,7 +111,7 @@ private: Node_UP _child; vespalib::string _dimension; - static TensorType infer_type(const TensorType &child_type, const vespalib::string &dimension) { + static eval::ValueType infer_type(const eval::ValueType &child_type, const vespalib::string &dimension) { return child_type.remove_dimensions({dimension}); } @@ -135,11 +136,11 @@ private: Node_UP _child; size_t _cell_function_id; - static TensorType infer_type(const TensorType &child_type) { + static eval::ValueType infer_type(const eval::ValueType &child_type) { if (child_type.is_tensor()) { return child_type; } else { - return TensorType::invalid(); + return eval::ValueType::error_type(); } } @@ -165,7 +166,7 @@ private: Node_UP _lhs; Node_UP _rhs; - static TensorType infer_type(const TensorType &lhs_type, const TensorType &rhs_type) { + static eval::ValueType infer_type(const eval::ValueType &lhs_type, const eval::ValueType &rhs_type) { return lhs_type.add_dimensions_from(rhs_type); } @@ -191,7 +192,7 @@ private: Node_UP _lhs; Node_UP _rhs; - static TensorType infer_type(const TensorType &lhs_type, const TensorType &rhs_type) { + static eval::ValueType infer_type(const eval::ValueType &lhs_type, const eval::ValueType &rhs_type) { return lhs_type.add_dimensions_from(rhs_type); } @@ -217,7 +218,7 @@ private: Node_UP _lhs; Node_UP _rhs; - static TensorType infer_type(const TensorType &lhs_type, const TensorType &rhs_type) { + static eval::ValueType infer_type(const eval::ValueType &lhs_type, const eval::ValueType &rhs_type) { return lhs_type.add_dimensions_from(rhs_type); } @@ -243,7 +244,7 @@ private: Node_UP _lhs; Node_UP _rhs; - static TensorType infer_type(const TensorType &lhs_type, const TensorType &rhs_type) { + static eval::ValueType infer_type(const eval::ValueType &lhs_type, const eval::ValueType &rhs_type) { return lhs_type.add_dimensions_from(rhs_type); } @@ -269,7 +270,7 @@ private: Node_UP _lhs; Node_UP _rhs; - static TensorType infer_type(const TensorType &lhs_type, const TensorType &rhs_type) { + static eval::ValueType infer_type(const eval::ValueType &lhs_type, const eval::ValueType &rhs_type) { return lhs_type.add_dimensions_from(rhs_type); } @@ -295,7 +296,7 @@ private: Node_UP _lhs; Node_UP _rhs; - static TensorType infer_type(const TensorType &lhs_type, const TensorType &rhs_type) { + static eval::ValueType infer_type(const eval::ValueType &lhs_type, const eval::ValueType &rhs_type) { return lhs_type.keep_dimensions_in(rhs_type); } @@ -314,7 +315,7 @@ public: } // namespace vespalib::tensor::function::<unnamed> -Node_UP input(const TensorType &type, size_t tensor_id) { +Node_UP input(const eval::ValueType &type, size_t tensor_id) { return std::make_unique<InputTensor>(type, tensor_id); } diff --git a/vespalib/src/vespa/vespalib/tensor/tensor_function.h b/vespalib/src/vespa/vespalib/tensor/tensor_function.h index 1bc55b30206..f60fc6ac881 100644 --- a/vespalib/src/vespa/vespalib/tensor/tensor_function.h +++ b/vespalib/src/vespa/vespalib/tensor/tensor_function.h @@ -86,12 +86,12 @@ struct Node : public TensorFunction * * @return tensor operation result type. **/ - virtual const TensorType &type() const = 0; + virtual const eval::ValueType &type() const = 0; }; using Node_UP = std::unique_ptr<Node>; -Node_UP input(const TensorType &type, size_t tensor_id); +Node_UP input(const eval::ValueType &type, size_t tensor_id); Node_UP sum(Node_UP child); Node_UP dimension_sum(Node_UP child, const vespalib::string &dimension); Node_UP apply(Node_UP child, size_t cell_function_id); diff --git a/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp b/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp index 8f34beec535..3cce3357aa9 100644 --- a/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp +++ b/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp @@ -1,17 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> + #include "tensor_mapper.h" #include "tensor.h" #include "tensor_visitor.h" #include <vespa/vespalib/tensor/simple/direct_simple_tensor_builder.h> #include <vespa/vespalib/tensor/compact/direct_compact_tensor_v2_builder.h> #include <vespa/vespalib/tensor/compact/direct_compact_tensor_builder.h> -#include <vespa/vespalib/tensor/compact/compact_tensor_address_ref.h> #include <vespa/vespalib/tensor/dense/dense_tensor.h> #include "tensor_address_element_iterator.h" #include "default_tensor.h" +using vespalib::eval::ValueType; + namespace vespalib { namespace tensor { @@ -20,11 +22,11 @@ namespace { class SparseTensorMapperBase { protected: - static TensorDimensions mapDimensions(const TensorType &type); + static TensorDimensions mapDimensions(const ValueType &type); }; TensorDimensions -SparseTensorMapperBase::mapDimensions(const TensorType &type) +SparseTensorMapperBase::mapDimensions(const ValueType &type) { TensorDimensions dimensions; dimensions.reserve(type.dimensions().size()); @@ -46,19 +48,19 @@ class SparseTensorMapper : public TensorVisitor, public SparseTensorMapperBase void mapAddress(const TensorAddress &address); virtual void visit(const TensorAddress &address, double value) override; - SparseTensorMapper(const TensorType &type); + SparseTensorMapper(const ValueType &type); ~SparseTensorMapper(); std::unique_ptr<Tensor> build(); public: static std::unique_ptr<Tensor> - map(const Tensor &tensor, const TensorType &type); + map(const Tensor &tensor, const ValueType &type); }; template <class TensorT> SparseTensorMapper<TensorT>:: -SparseTensorMapper(const TensorType &type) +SparseTensorMapper(const ValueType &type) : TensorVisitor(), SparseTensorMapperBase(), _builder(mapDimensions(type)), @@ -125,7 +127,7 @@ SparseTensorMapper<TensorT>::visit(const TensorAddress &address, double value) template <class TensorT> std::unique_ptr<Tensor> SparseTensorMapper<TensorT>::map(const Tensor &tensor, - const TensorType &type) + const ValueType &type) { SparseTensorMapper<TensorT> mapper(type); tensor.accept(mapper); @@ -144,16 +146,16 @@ class DenseTensorMapper : public TensorVisitor uint32_t mapAddressToIndex(const TensorAddress &address); virtual void visit(const TensorAddress &address, double value) override; - DenseTensorMapper(const TensorType &type); + DenseTensorMapper(const ValueType &type); ~DenseTensorMapper(); std::unique_ptr<Tensor> build(); public: static std::unique_ptr<Tensor> - map(const Tensor &tensor, const TensorType &type); + map(const Tensor &tensor, const ValueType &type); }; -DenseTensorMapper::DenseTensorMapper(const TensorType &type) +DenseTensorMapper::DenseTensorMapper(const ValueType &type) : _dimensionsMeta(), _cells() { @@ -225,16 +227,38 @@ DenseTensorMapper::visit(const TensorAddress &address, double value) } std::unique_ptr<Tensor> -DenseTensorMapper::map(const Tensor &tensor, const TensorType &type) +DenseTensorMapper::map(const Tensor &tensor, const ValueType &type) { DenseTensorMapper mapper(type); tensor.accept(mapper); return mapper.build(); } +bool +hasOnlyMappedDimensions(const ValueType &type) +{ + for (const auto &dim : type.dimensions()) { + if (!dim.is_mapped()) { + return false; + } + } + return true; +} + +bool +hasOnlyIndexedDimensions(const ValueType &type) +{ + for (const auto &dim : type.dimensions()) { + if (!dim.is_indexed()) { + return false; + } + } + return true; +} + } // namespace vespalib::tensor::<anonymous> -TensorMapper::TensorMapper(const TensorType &type) +TensorMapper::TensorMapper(const ValueType &type) : _type(type) { } @@ -245,30 +269,30 @@ TensorMapper::~TensorMapper() template <typename TensorT> std::unique_ptr<Tensor> -TensorMapper::mapToSparse(const Tensor &tensor, const TensorType &type) +TensorMapper::mapToSparse(const Tensor &tensor, const ValueType &type) { - assert(type.type() == TensorType::Type::SPARSE); + assert(hasOnlyMappedDimensions(type)); return SparseTensorMapper<TensorT>::map(tensor, type); } std::unique_ptr<Tensor> -TensorMapper::mapToDense(const Tensor &tensor, const TensorType &type) +TensorMapper::mapToDense(const Tensor &tensor, const ValueType &type) { - assert(type.type() == TensorType::Type::DENSE); + assert(hasOnlyIndexedDimensions(type)); return DenseTensorMapper::map(tensor, type); } std::unique_ptr<Tensor> TensorMapper::map(const Tensor &tensor) const { - switch (_type.type()) { - case TensorType::Type::INVALID: - case TensorType::Type::NUMBER: + if (_type.is_tensor()) { + if (hasOnlyMappedDimensions(_type)) { + return mapToSparse<DefaultTensor::type>(tensor, _type); + } else if (hasOnlyIndexedDimensions(_type)) { + return mapToDense(tensor, _type); + } + } else { return std::unique_ptr<Tensor>(); - case TensorType::Type::SPARSE: - return mapToSparse<DefaultTensor::type>(tensor, _type); - case TensorType::Type::DENSE: - return mapToDense(tensor, _type); } abort(); } @@ -276,17 +300,17 @@ TensorMapper::map(const Tensor &tensor) const template std::unique_ptr<Tensor> TensorMapper::mapToSparse<SimpleTensor>(const Tensor &tensor, - const TensorType &type); + const ValueType &type); template std::unique_ptr<Tensor> TensorMapper::mapToSparse<CompactTensor>(const Tensor &tensor, - const TensorType &type); + const ValueType &type); template std::unique_ptr<Tensor> TensorMapper::mapToSparse<CompactTensorV2>(const Tensor &tensor, - const TensorType &type); + const ValueType &type); } // namespace vespalib::tensor } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/tensor/tensor_mapper.h b/vespalib/src/vespa/vespalib/tensor/tensor_mapper.h index 36ce0426c69..e0394fbd3f7 100644 --- a/vespalib/src/vespa/vespalib/tensor/tensor_mapper.h +++ b/vespalib/src/vespa/vespalib/tensor/tensor_mapper.h @@ -2,7 +2,7 @@ #pragma once -#include "tensor_type.h" +#include <vespa/vespalib/eval/value_type.h> namespace vespalib { namespace tensor { @@ -23,17 +23,17 @@ class Tensor; */ class TensorMapper { - TensorType _type; + eval::ValueType _type; public: - TensorMapper(const TensorType &type); + TensorMapper(const eval::ValueType &type); ~TensorMapper(); template <typename TensorT> static std::unique_ptr<Tensor> - mapToSparse(const Tensor &tensor, const TensorType &type); + mapToSparse(const Tensor &tensor, const eval::ValueType &type); static std::unique_ptr<Tensor> - mapToDense(const Tensor &tensor, const TensorType &type); + mapToDense(const Tensor &tensor, const eval::ValueType &type); std::unique_ptr<Tensor> map(const Tensor &tensor) const; }; |