diff options
Diffstat (limited to 'searchlib/src/tests/features/tensor/tensor_test.cpp')
| -rw-r--r-- | searchlib/src/tests/features/tensor/tensor_test.cpp | 237 |
1 files changed, 237 insertions, 0 deletions
diff --git a/searchlib/src/tests/features/tensor/tensor_test.cpp b/searchlib/src/tests/features/tensor/tensor_test.cpp new file mode 100644 index 00000000000..caceea0f47b --- /dev/null +++ b/searchlib/src/tests/features/tensor/tensor_test.cpp @@ -0,0 +1,237 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/eval/function.h> + +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/searchlib/attribute/tensorattribute.h> +#include <vespa/searchlib/features/setup.h> +#include <vespa/searchlib/fef/test/indexenvironment.h> +#include <vespa/searchlib/fef/test/indexenvironmentbuilder.h> +#include <vespa/searchlib/fef/test/queryenvironment.h> +#include <vespa/searchlib/fef/test/ftlib.h> +#include <vespa/searchlib/fef/fef.h> +#include <vespa/vespalib/tensor/tensor_factory.h> +#include <vespa/vespalib/tensor/default_tensor.h> +#include <vespa/vespalib/tensor/serialization/typed_binary_format.h> +#include <vespa/searchlib/attribute/tensorattribute.h> +#include <vespa/vespalib/eval/interpreted_function.h> +#include <vespa/vespalib/tensor/default_tensor_engine.h> + +using search::feature_t; +using namespace search::fef; +using namespace search::fef::indexproperties; +using namespace search::fef::test; +using namespace search::features; +using search::AttributeFactory; +using search::attribute::TensorAttribute; +using search::AttributeVector; +using vespalib::eval::Value; +using vespalib::eval::Function; +using vespalib::tensor::Tensor; +using vespalib::tensor::TensorCells; +using vespalib::tensor::DenseTensorCells; +using vespalib::tensor::TensorDimensions; +using vespalib::tensor::TensorFactory; +using vespalib::tensor::TensorType; +using vespalib::eval::InterpretedFunction; +using vespalib::tensor::DefaultTensorEngine; + +typedef search::attribute::Config AVC; +typedef search::attribute::BasicType AVBT; +typedef search::attribute::CollectionType AVCT; +typedef search::AttributeVector::SP AttributePtr; +typedef FtTestApp FTA; + +namespace +{ + +Tensor::UP createTensor(const TensorCells &cells, + const TensorDimensions &dimensions) { + vespalib::tensor::DefaultTensor::builder builder; + return TensorFactory::create(cells, dimensions, builder); +} + +} + +struct ExecFixture +{ + BlueprintFactory factory; + FtFeatureTest test; + ExecFixture(const vespalib::string &feature) + : factory(), + test(factory, feature) + { + setup_search_features(factory); + setupAttributeVectors(); + setupQueryEnvironment(); + ASSERT_TRUE(test.setup()); + } + void addAttributeField(const vespalib::string &attrName) { + test.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, attrName); + } + AttributeVector::SP createStringAttribute(const vespalib::string &attrName) { + addAttributeField(attrName); + return AttributeFactory::createAttribute(attrName, AVC(AVBT::STRING, AVCT::SINGLE)); + } + AttributeVector::SP createTensorAttribute(const vespalib::string &attrName, const vespalib::string &type) { + addAttributeField(attrName); + AVC config(AVBT::TENSOR, AVCT::SINGLE); + config.setTensorType(TensorType::fromSpec(type)); + return AttributeFactory::createAttribute(attrName, config); + } + void setAttributeTensorType(const vespalib::string &attrName, const vespalib::string &type) { + type::Attribute::set(test.getIndexEnv().getProperties(), attrName, type); + } + void setQueryTensorType(const vespalib::string &queryFeatureName, const vespalib::string &type) { + type::QueryFeature::set(test.getIndexEnv().getProperties(), queryFeatureName, type); + } + void setupAttributeVectors() { + std::vector<AttributePtr> attrs; + attrs.push_back(createTensorAttribute("tensorattr", "tensor(x{})")); + attrs.push_back(createStringAttribute("singlestr")); + attrs.push_back(createTensorAttribute("wrongtype", "tensor(y{})")); + addAttributeField("null"); + setAttributeTensorType("tensorattr", "tensor(x{})"); + setAttributeTensorType("wrongtype", "tensor(x{})"); + setAttributeTensorType("null", "tensor(x{})"); + + for (const auto &attr : attrs) { + attr->addReservedDoc(); + attr->addDocs(2); + attr->clearDoc(1); + attr->clearDoc(2); + attr->commit(); + test.getIndexEnv().getAttributeManager().add(attr); + } + + TensorAttribute *tensorAttr = + dynamic_cast<TensorAttribute *>(attrs[0].get()); + + tensorAttr->setTensor(1, *createTensor({ {{{"x", "a"}}, 3}, + {{{"x", "b"}}, 5}, + {{{"x", "c"}}, 7} }, + { "x" })); + + for (const auto &attr : attrs) { + attr->commit(); + } + } + void setQueryTensor(const vespalib::string &tensorName, + const vespalib::string &tensorTypeSpec, + const TensorCells &cells, + const TensorDimensions &dimensions) + { + auto tensor = createTensor(cells, dimensions); + vespalib::nbostream stream; + vespalib::tensor::TypedBinaryFormat::serialize(stream, *tensor); + test.getQueryEnv().getProperties().add(tensorName, + vespalib::stringref(stream.peek(), stream.size())); + setQueryTensorType(tensorName, tensorTypeSpec); + } + + void setupQueryEnvironment() { + setQueryTensor("tensorquery", + "tensor(q{})", + { {{{"q", "d"}}, 11 }, + {{{"q", "e"}}, 13 }, + {{{"q", "f"}}, 17 } }, + { "q" }); + setQueryTensor("mappedtensorquery", + "tensor(x[2])", + { {{{"x", "0"},{"y", "0"}}, 11 }, + {{{"x", "0"},{"y", "1"}}, 13 }, + {{{"x", "1"},{"y", "0"}}, 17 } }, + { "x", "y" }); + setQueryTensorType("null", "tensor(q{})"); + } + const Tensor &extractTensor() { + const Value::CREF *value = test.resolveObjectFeature(); + ASSERT_TRUE(value != nullptr); + ASSERT_TRUE(value->get().is_tensor()); + return static_cast<const Tensor &>(*value->get().as_tensor()); + } + const Tensor &execute(uint32_t docId = 1) { + test.executeOnly(docId); + return extractTensor(); + } +}; + +struct AsTensor { + InterpretedFunction ifun; + InterpretedFunction::Context ctx; + const Value *result; + explicit AsTensor(const vespalib::string &expr) + : ifun(DefaultTensorEngine::ref(), Function::parse(expr)), ctx(), result(&ifun.eval(ctx)) + { + ASSERT_TRUE(result->is_tensor()); + } + bool operator==(const Tensor &rhs) const { return static_cast<const Tensor &>(*result->as_tensor()).equals(rhs); } +}; + +std::ostream &operator<<(std::ostream &os, const AsTensor &my_tensor) { + os << my_tensor.result->as_tensor(); + return os; +} + +TEST_F("require that tensor attribute can be extracted as tensor in attribute feature", + ExecFixture("attribute(tensorattr)")) +{ + EXPECT_EQUAL(AsTensor("{ {x:b}:5, {x:c}:7, {x:a}:3 }"), f.execute()); +} + +TEST_F("require that tensor from query can be extracted as tensor in query feature", + ExecFixture("query(tensorquery)")) +{ + EXPECT_EQUAL(AsTensor("{ {q:f}:17, {q:d}:11, {q:e}:13 }"), f.execute()); +} + +TEST_F("require that empty tensor is created if attribute does not exists", + ExecFixture("attribute(null)")) +{ + EXPECT_EQUAL(AsTensor("{ }"), f.execute()); +} + +TEST_F("require that empty tensor is created if tensor type is wrong", + ExecFixture("attribute(wrongtype)")) +{ + EXPECT_EQUAL(AsTensor("{ }"), f.execute()); +} + +TEST_F("require that empty tensor is created if query parameter is not found", + ExecFixture("query(null)")) +{ + EXPECT_EQUAL(AsTensor("{ }"), f.execute()); +} + +TEST_F("require that empty tensor is created if document has no tensor", + ExecFixture("attribute(tensorattr)")) { + EXPECT_EQUAL(AsTensor("{ }"), f.execute(2)); +} + +struct AsDenseTensor { + Tensor::UP tensor; + explicit AsDenseTensor(const DenseTensorCells &cells) + : tensor(TensorFactory::createDense(cells)) + { + ASSERT_TRUE(!!tensor); + } + bool operator==(const Tensor &rhs) const { return tensor->equals(rhs); } +}; + + +std::ostream &operator<<(std::ostream &os, const AsDenseTensor &my_tensor) { + os << *my_tensor.tensor; + return os; +} + +TEST_F("require that tensor from query is mapped", + ExecFixture("query(mappedtensorquery)")) { + EXPECT_EQUAL(AsDenseTensor({ {{{"x", 0}}, 24}, + {{{"x", 1}}, 17} }), + f.execute()); +} + + +TEST_MAIN() { TEST_RUN_ALL(); } |