diff options
author | Geir Storli <geirst@verizonmedia.com> | 2019-03-19 08:35:06 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-03-19 08:35:06 +0100 |
commit | 43b44b6d1fba5c5c7eb982716119bf37aeeb7604 (patch) | |
tree | 5c05e5d469ed0309ccfa7230aa37114d6c8f7646 | |
parent | de69949611cb36b688c280333347f39aaec7304b (diff) | |
parent | 2ab63d9c085311f97e28ef481dd910003d47d75d (diff) |
Merge pull request #8823 from vespa-engine/toregge/stop-using-tensor-mapper-in-query-feature-blueprint
Stop using TensorMapper in query QueryBlueprint.
-rw-r--r-- | searchlib/src/tests/features/tensor/tensor_test.cpp | 68 | ||||
-rw-r--r-- | searchlib/src/vespa/searchlib/features/queryfeature.cpp | 20 |
2 files changed, 39 insertions, 49 deletions
diff --git a/searchlib/src/tests/features/tensor/tensor_test.cpp b/searchlib/src/tests/features/tensor/tensor_test.cpp index b097f27342d..abe54906791 100644 --- a/searchlib/src/tests/features/tensor/tensor_test.cpp +++ b/searchlib/src/tests/features/tensor/tensor_test.cpp @@ -16,6 +16,7 @@ #include <vespa/eval/tensor/default_tensor_engine.h> #include <vespa/eval/tensor/serialization/typed_binary_format.h> #include <vespa/eval/tensor/tensor_factory.h> +#include <vespa/eval/tensor/test/test_utils.h> #include <vespa/vespalib/objects/nbostream.h> using search::feature_t; @@ -36,6 +37,7 @@ using vespalib::tensor::Tensor; using vespalib::tensor::TensorCells; using vespalib::tensor::TensorDimensions; using vespalib::tensor::TensorFactory; +using vespalib::tensor::test::makeTensor; using AVC = search::attribute::Config; using AVBT = search::attribute::BasicType; @@ -47,19 +49,8 @@ using CollectionType = FieldInfo::CollectionType; namespace { -Tensor::UP createTensor(const TensorCells &cells, - const TensorDimensions &dimensions) { - vespalib::tensor::DefaultTensor::builder builder; - return TensorFactory::create(cells, dimensions, builder); -} - -Tensor::UP make_tensor(const TensorSpec &spec) { - auto tensor = DefaultTensorEngine::ref().from_spec(spec); - return Tensor::UP(dynamic_cast<Tensor*>(tensor.release())); -} - Tensor::UP make_empty(const vespalib::string &type) { - return make_tensor(TensorSpec(type)); + return makeTensor<Tensor>(TensorSpec(type)); } } @@ -118,21 +109,18 @@ struct ExecFixture TensorAttribute *tensorAttr = dynamic_cast<TensorAttribute *>(attrs[0].get()); - tensorAttr->setTensor(1, *createTensor({ {{{"x", "a"}}, 3}, - {{{"x", "b"}}, 5}, - {{{"x", "c"}}, 7} }, - { "x" })); - + tensorAttr->setTensor(1, *makeTensor<Tensor>(TensorSpec("tensor(x{})") + .add({{"x", "a"}}, 3) + .add({{"x", "b"}}, 5) + .add({{"x", "c"}}, 7))); for (const auto &attr : attrs) { attr->commit(); } } void setQueryTensor(const vespalib::string &tensorName, const vespalib::string &tensorTypeSpec, - const TensorCells &cells, - const TensorDimensions &dimensions) + std::unique_ptr<Tensor> tensor) { - auto tensor = createTensor(cells, dimensions); vespalib::nbostream stream; vespalib::tensor::TypedBinaryFormat::serialize(stream, *tensor); test.getQueryEnv().getProperties().add(tensorName, @@ -143,16 +131,16 @@ struct ExecFixture void setupQueryEnvironment() { setQueryTensor("tensorquery", "tensor(q{})", - { {{{"q", "d"}}, 11 }, - {{{"q", "e"}}, 13 }, - {{{"q", "f"}}, 17 } }, - { "q" }); + makeTensor<Tensor>(TensorSpec("tensor(q{})") + .add({{"q", "d"}}, 11 ) + .add({{"q", "e"}}, 13 ) + .add({{"q", "f"}}, 17 ))); setQueryTensor("mappedtensorquery", "tensor(x[2])", - { {{{"x", "0"},{"y", "0"}}, 11 }, - {{{"x", "0"},{"y", "1"}}, 13 }, - {{{"x", "1"},{"y", "0"}}, 17 } }, - { "x", "y" }); + makeTensor<Tensor>(TensorSpec("tensor(x{},y{})") + .add({{"x", "0"},{"y", "0"}}, 11 ) + .add({{"x", "0"},{"y", "1"}}, 13 ) + .add({{"x", "1"},{"y", "0"}}, 17 ))); setQueryTensorType("null", "tensor(q{})"); } const Tensor &extractTensor(uint32_t docid) { @@ -168,19 +156,19 @@ struct ExecFixture TEST_F("require that tensor attribute can be extracted as tensor in attribute feature", ExecFixture("attribute(tensorattr)")) { - EXPECT_EQUAL(*make_tensor(TensorSpec("tensor(x{})") - .add({{"x", "b"}}, 5) - .add({{"x", "c"}}, 7) - .add({{"x", "a"}}, 3)), f.execute()); + EXPECT_EQUAL(*makeTensor<Tensor>(TensorSpec("tensor(x{})") + .add({{"x", "b"}}, 5) + .add({{"x", "c"}}, 7) + .add({{"x", "a"}}, 3)), f.execute()); } TEST_F("require that tensor from query can be extracted as tensor in query feature", ExecFixture("query(tensorquery)")) { - EXPECT_EQUAL(*make_tensor(TensorSpec("tensor(q{})") - .add({{"q", "f"}}, 17) - .add({{"q", "d"}}, 11) - .add({{"q", "e"}}, 13)), f.execute()); + EXPECT_EQUAL(*makeTensor<Tensor>(TensorSpec("tensor(q{})") + .add({{"q", "f"}}, 17) + .add({{"q", "d"}}, 11) + .add({{"q", "e"}}, 13)), f.execute()); } TEST_F("require that empty tensor is created if attribute does not exists", @@ -206,11 +194,11 @@ TEST_F("require that empty tensor with correct type is created if document has n EXPECT_EQUAL(*make_empty("tensor(x{})"), f.execute(2)); } -TEST_F("require that tensor from query is mapped", +TEST_F("require that wrong tensor type from query tensor gives empty tensor", ExecFixture("query(mappedtensorquery)")) { - EXPECT_EQUAL(*make_tensor(TensorSpec("tensor(x[2])") - .add({{"x", 0}}, 24) - .add({{"x", 1}}, 17)), f.execute()); + EXPECT_EQUAL(*makeTensor<Tensor>(TensorSpec("tensor(x[2])") + .add({{"x", 0}}, 0) + .add({{"x", 1}}, 0)), f.execute()); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchlib/features/queryfeature.cpp b/searchlib/src/vespa/searchlib/features/queryfeature.cpp index 3e52546e841..c5488581d29 100644 --- a/searchlib/src/vespa/searchlib/features/queryfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/queryfeature.cpp @@ -4,22 +4,24 @@ #include "utils.h" #include "valuefeature.h" +#include <vespa/document/datatype/tensor_data_type.h> #include <vespa/searchlib/features/constant_tensor_executor.h> #include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/feature_type.h> #include <vespa/vespalib/objects/nbostream.h> -#include <vespa/eval/tensor/default_tensor.h> -#include <vespa/eval/tensor/tensor_mapper.h> #include <vespa/eval/tensor/serialization/typed_binary_format.h> +#include <vespa/eval/tensor/tensor.h> #include <vespa/eval/eval/value_type.h> #include <vespa/vespalib/locale/c.h> +#include <vespa/log/log.h> +LOG_SETUP(".features.queryfeature"); + using namespace search::fef; using namespace search::fef::indexproperties; -using vespalib::tensor::DefaultTensor; -using vespalib::tensor::TensorBuilder; +using document::TensorDataType; using vespalib::eval::ValueType; using search::fef::FeatureType; @@ -121,11 +123,11 @@ createTensorExecutor(const search::fef::IQueryEnvironment &env, if (prop.found() && !prop.get().empty()) { const vespalib::string &value = prop.get(); vespalib::nbostream stream(value.data(), value.size()); - vespalib::tensor::Tensor::UP tensor = vespalib::tensor::TypedBinaryFormat::deserialize(stream); - if (tensor->type() != valueType) { - vespalib::tensor::TensorMapper mapper(valueType); - vespalib::tensor::Tensor::UP mappedTensor = mapper.map(*tensor); - tensor = std::move(mappedTensor); + auto tensor = vespalib::tensor::TypedBinaryFormat::deserialize(stream); + if (!TensorDataType::isAssignableType(valueType, tensor->type())) { + LOG(warning, "Query feature type is '%s' but other tensor type is '%s'", + valueType.to_spec().c_str(), tensor->type().to_spec().c_str()); + return ConstantTensorExecutor::createEmpty(valueType, stash); } return ConstantTensorExecutor::create(std::move(tensor), stash); } |