diff options
author | Harald Musum <musum@verizonmedia.com> | 2021-01-01 19:13:11 +0100 |
---|---|---|
committer | Harald Musum <musum@verizonmedia.com> | 2021-01-01 19:13:11 +0100 |
commit | 9ab7a397bacbae98501137a349786d7461a8aa29 (patch) | |
tree | 2c009df1401e94beb3acaa4e79e1dc22e7abeade /searchlib | |
parent | 3838f731e99e2a24871b3a90eb40d18579a56e40 (diff) | |
parent | 65df61f0c3d23ae95dbe8cb4296cc4c2d3e7fb6e (diff) |
Merge branch 'master' into revert-14062-revert-14057-hmusum/upgrade-to-curator-4
Diffstat (limited to 'searchlib')
82 files changed, 1036 insertions, 526 deletions
diff --git a/searchlib/src/apps/docstore/documentstoreinspect.cpp b/searchlib/src/apps/docstore/documentstoreinspect.cpp index 2526ce456ae..8a3bb6b247b 100644 --- a/searchlib/src/apps/docstore/documentstoreinspect.cpp +++ b/searchlib/src/apps/docstore/documentstoreinspect.cpp @@ -71,7 +71,7 @@ DocumentStoreInspectApp::Main() if (cmd == "dumpidxfile") { vespalib::string idxfile; if (_argc >= 4) { - if (_argv[2] == vespalib::string("--idxfile")) { + if (vespalib::string(_argv[2]) == vespalib::string("--idxfile")) { idxfile = _argv[3]; dumpIdxFile(idxfile); } else { diff --git a/searchlib/src/apps/tests/btreestress_test.cpp b/searchlib/src/apps/tests/btreestress_test.cpp index 0fb356001f2..37fc6b26cc3 100644 --- a/searchlib/src/apps/tests/btreestress_test.cpp +++ b/searchlib/src/apps/tests/btreestress_test.cpp @@ -202,19 +202,19 @@ TEST_F("Test single threaded lower_bound reader during updates", Fixture) { uint32_t cnt = 1000000; f._reportWork = true; - f._writer.execute(makeLambdaTask([=]() { f.writeWork(cnt); })); - f._readers.execute(makeLambdaTask([=]() { f.readWork(); })); + f._writer.execute(makeLambdaTask([this, cnt]() { f.writeWork(cnt); })); + f._readers.execute(makeLambdaTask([this]() { f.readWork(); })); } TEST_F("Test multithreaded lower_bound reader during updates", Fixture) { uint32_t cnt = 1000000; f._reportWork = true; - f._writer.execute(makeLambdaTask([=]() { f.writeWork(cnt); })); - f._readers.execute(makeLambdaTask([=]() { f.readWork(); })); - f._readers.execute(makeLambdaTask([=]() { f.readWork(); })); - f._readers.execute(makeLambdaTask([=]() { f.readWork(); })); - f._readers.execute(makeLambdaTask([=]() { f.readWork(); })); + f._writer.execute(makeLambdaTask([this, cnt]() { f.writeWork(cnt); })); + f._readers.execute(makeLambdaTask([this]() { f.readWork(); })); + f._readers.execute(makeLambdaTask([this]() { f.readWork(); })); + f._readers.execute(makeLambdaTask([this]() { f.readWork(); })); + f._readers.execute(makeLambdaTask([this]() { f.readWork(); })); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index 3f848b45cc3..e8590999bc1 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -371,9 +371,9 @@ Fixture::stressTest(uint32_t writeCnt) LOG(info, "starting stress test, 1 write thread, %u read threads, %u writes", readThreads, writeCnt); - _writer.execute(makeLambdaTask([=]() { writeWork(writeCnt); })); + _writer.execute(makeLambdaTask([this, writeCnt]() { writeWork(writeCnt); })); for (uint32_t i = 0; i < readThreads; ++i) { - _readers.execute(makeLambdaTask([=]() { readWork(); })); + _readers.execute(makeLambdaTask([this]() { readWork(); })); } } diff --git a/searchlib/src/apps/vespa-ranking-expression-analyzer/vespa-ranking-expression-analyzer.cpp b/searchlib/src/apps/vespa-ranking-expression-analyzer/vespa-ranking-expression-analyzer.cpp index 51f243e1c37..f2a9ee2932f 100644 --- a/searchlib/src/apps/vespa-ranking-expression-analyzer/vespa-ranking-expression-analyzer.cpp +++ b/searchlib/src/apps/vespa-ranking-expression-analyzer/vespa-ranking-expression-analyzer.cpp @@ -4,13 +4,12 @@ #include <vespa/eval/eval/llvm/compiled_function.h> #include <vespa/eval/eval/interpreted_function.h> #include <vespa/eval/eval/call_nodes.h> +#include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/operator_nodes.h> #include <vespa/vespalib/util/benchmark_timer.h> #include <vespa/eval/eval/vm_forest.h> #include <vespa/eval/eval/fast_forest.h> #include <vespa/eval/eval/llvm/deinline_forest.h> -#include <vespa/eval/eval/engine_or_factory.h> -#include <vespa/eval/tensor/default_tensor_engine.h> #include <vespa/vespalib/io/mapped_file_input.h> #include <vespa/eval/eval/param_usage.h> #include <vespa/fastos/app.h> @@ -154,7 +153,7 @@ struct FunctionInfo { size_t get_path_len(const TreeList &trees) const { size_t path = 0; for (const Node *tree: trees) { - InterpretedFunction ifun(EngineOrFactory::get(), *tree, NodeTypes()); + InterpretedFunction ifun(FastValueBuilderFactory::get(), *tree, NodeTypes()); InterpretedFunction::Context ctx(ifun); SimpleParams fun_params(params); ifun.eval(ctx, fun_params); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/tensor/TensorConformanceTest.java b/searchlib/src/test/java/com/yahoo/searchlib/tensor/TensorConformanceTest.java deleted file mode 100644 index 61aec069c72..00000000000 --- a/searchlib/src/test/java/com/yahoo/searchlib/tensor/TensorConformanceTest.java +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchlib.tensor; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.searchlib.rankingexpression.evaluation.BooleanValue; -import com.yahoo.searchlib.rankingexpression.evaluation.DoubleCompatibleValue; -import com.yahoo.searchlib.rankingexpression.evaluation.MapContext; -import com.yahoo.searchlib.rankingexpression.evaluation.StringValue; -import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; -import com.yahoo.searchlib.rankingexpression.evaluation.Value; -import com.yahoo.searchlib.rankingexpression.parser.ParseException; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.serialization.TypedBinaryFormat; -import org.junit.Assert; -import org.junit.Test; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Optional; - -import static org.junit.Assert.assertEquals; - -public class TensorConformanceTest { - - private static String testPath = "eval/src/apps/tensor_conformance/test_spec.json"; - - @Test - public void testConformance() throws IOException { - File testSpec = new File(testPath); - if (!testSpec.exists()) { - testSpec = new File("../" + testPath); - } - int count = 0; - List<Integer> failList = new ArrayList<>(); - - try(BufferedReader br = new BufferedReader(new FileReader(testSpec))) { - String test = br.readLine(); - while (test != null) { - boolean success = testCase(test, count); - if (!success) { - failList.add(count); - } - test = br.readLine(); - count++; - } - } - assertEquals(failList.size() + " conformance test fails: " + failList, 0, failList.size()); - } - - private boolean testCase(String test, int count) { - try { - ObjectMapper mapper = new ObjectMapper(); - JsonNode node = mapper.readTree(test); - - if (node.has("num_tests")) { - Assert.assertEquals(node.get("num_tests").asInt(), count); - return true; - } - if (!node.has("expression")) { - return true; // ignore - } - - String expression = node.get("expression").asText(); - MapContext context = getInput(node.get("inputs")); - Tensor expect = getTensor(node.get("result").get("expect").asText()); - Tensor result = evaluate(expression, context); - boolean equals = Tensor.equals(result, expect); - if (!equals) { - System.out.println(count + " : Tensors not equal. Result: " + result.toString() + " Expected: " + expect.toString() + " -> expression \"" + expression + "\""); - } else if (! result.type().valueType().equals(expect.type().valueType())) { - System.out.println(count + " : Tensor cell value types not equal. Result: " + result.type() + " Expected: " + expect.type() + " -> expression \"" + expression + "\""); - equals = false; - } - return equals; - - } catch (Exception e) { - System.out.println(count + " : " + e.toString()); - } - return false; - } - - private Tensor evaluate(String expression, MapContext context) throws ParseException { - Value value = new RankingExpression(expression).evaluate(context); - if (!(value instanceof TensorValue)) { - throw new IllegalArgumentException("Result is not a tensor"); - } - return ((TensorValue)value).asTensor(); - } - - private MapContext getInput(JsonNode inputs) { - MapContext context = new MapContext(); - for (Iterator<String> i = inputs.fieldNames(); i.hasNext(); ) { - String name = i.next(); - String value = inputs.get(name).asText(); - Tensor tensor = getTensor(value); - context.put(name, new TensorValue(tensor)); - } - return context; - } - - private Tensor getTensor(String binaryRepresentation) { - byte[] bin = getBytes(binaryRepresentation); - return TypedBinaryFormat.decode(Optional.empty(), GrowableByteBuffer.wrap(bin)); - } - - private byte[] getBytes(String binaryRepresentation) { - return parseHexValue(binaryRepresentation.substring(2)); - } - - private byte[] parseHexValue(String s) { - final int len = s.length(); - byte[] bytes = new byte[len/2]; - for (int i = 0; i < len; i += 2) { - int c1 = hexValue(s.charAt(i)) << 4; - int c2 = hexValue(s.charAt(i + 1)); - bytes[i/2] = (byte)(c1 + c2); - } - return bytes; - } - - private int hexValue(Character c) { - if (c >= 'a' && c <= 'f') { - return c - 'a' + 10; - } else if (c >= 'A' && c <= 'F') { - return c - 'A' + 10; - } else if (c >= '0' && c <= '9') { - return c - '0'; - } - throw new IllegalArgumentException("Hex contains illegal characters"); - } - -} - diff --git a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp index 1d6f68c5802..2d30f89015d 100644 --- a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp +++ b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp @@ -1,9 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/eval/eval/engine_or_factory.h> #include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/test/value_compare.h> -#include <vespa/eval/tensor/default_tensor_engine.h> #include <vespa/searchcommon/attribute/search_context_params.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/tensor/i_tensor_attribute.h> @@ -16,10 +16,10 @@ using search::tensor::TensorAttribute; using vespalib::eval::Value; using vespalib::eval::ValueType; using vespalib::eval::TensorSpec; -using vespalib::eval::EngineOrFactory; +using vespalib::eval::SimpleValue; Value::UP createTensor(const TensorSpec &spec) { - return EngineOrFactory::get().from_spec(spec); + return SimpleValue::from_spec(spec); } namespace search::attribute { diff --git a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp index 2bb55ac6977..c622ccae679 100644 --- a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp +++ b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp @@ -123,7 +123,7 @@ public: _attr->shrinkLidSpace(); } void clearDocs(uint32_t lidLow, uint32_t lidLimit) { - _mvMapping->clearDocs(lidLow, lidLimit, [=](uint32_t docId) { _attr->clearDoc(docId); }); + _mvMapping->clearDocs(lidLow, lidLimit, [this](uint32_t docId) { _attr->clearDoc(docId); }); } size_t getTotalValueCnt() const { return _mvMapping->getTotalValueCnt(); } diff --git a/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp b/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp index b23277bee21..51b4f1d760d 100644 --- a/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/eval/eval/engine_or_factory.h> -#include <vespa/eval/tensor/default_tensor_engine.h> -#include <vespa/eval/tensor/dense/dense_tensor_view.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/value_codec.h> #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchlib/attribute/attribute_blueprint_factory.h> #include <vespa/searchlib/attribute/attribute_read_guard.h> @@ -51,7 +51,6 @@ using search::queryeval::SearchIterator; using std::string; using std::vector; using vespalib::eval::TensorSpec; -using vespalib::eval::EngineOrFactory; using vespalib::eval::Value; using vespalib::eval::ValueType; using namespace search::attribute; @@ -358,7 +357,7 @@ expect_nearest_neighbor_blueprint(const vespalib::string& attribute_tensor_type_ auto result = f.create_blueprint(); const auto& nearest = downcast<const NearestNeighborBlueprint>(*result); EXPECT_EQ(attribute_tensor_type_spec, nearest.get_attribute_tensor().getTensorType().to_spec()); - EXPECT_EQ(converted_query_tensor, EngineOrFactory::get().to_spec(nearest.get_query_tensor())); + EXPECT_EQ(converted_query_tensor, spec_from_value(nearest.get_query_tensor())); EXPECT_EQ(7u, nearest.get_target_num_hits()); } diff --git a/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp b/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp index 98cb8c0485e..e3d84683d70 100644 --- a/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp +++ b/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp @@ -65,7 +65,7 @@ using queryeval::SimpleResult; class DocSet : public std::set<uint32_t> { public: - DocSet(); + DocSet() noexcept; ~DocSet(); DocSet(const uint32_t *b, const uint32_t *e) : std::set<uint32_t>(b, e) {} DocSet & put(const uint32_t &v) { @@ -74,7 +74,7 @@ public: } }; -DocSet::DocSet() = default; +DocSet::DocSet() noexcept = default; DocSet::~DocSet() = default; template <typename V, typename T> diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index daa85c91b2c..e1bd47af358 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -1,7 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/document/base/exceptions.h> -#include <vespa/eval/eval/engine_or_factory.h> +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value.h> #include <vespa/eval/eval/test/value_compare.h> #include <vespa/fastos/file.h> @@ -16,7 +17,7 @@ #include <vespa/searchlib/tensor/nearest_neighbor_index.h> #include <vespa/searchlib/tensor/nearest_neighbor_index_factory.h> #include <vespa/searchlib/tensor/nearest_neighbor_index_saver.h> -#include <vespa/searchlib/tensor/serialized_tensor_attribute.h> +#include <vespa/searchlib/tensor/serialized_fast_value_attribute.h> #include <vespa/searchlib/tensor/tensor_attribute.h> #include <vespa/searchlib/test/directory_handler.h> #include <vespa/searchlib/util/fileutil.h> @@ -40,7 +41,7 @@ using search::tensor::DefaultNearestNeighborIndexFactory; using search::tensor::DenseTensorAttribute; using search::tensor::DirectTensorAttribute; using search::tensor::DocVectorAccess; -using search::tensor::SerializedTensorAttribute; +using search::tensor::SerializedFastValueAttribute; using search::tensor::HnswIndex; using search::tensor::HnswNode; using search::tensor::NearestNeighborIndex; @@ -52,7 +53,7 @@ using vespalib::eval::TensorSpec; using vespalib::eval::CellType; using vespalib::eval::ValueType; using vespalib::eval::Value; -using vespalib::eval::EngineOrFactory; +using vespalib::eval::SimpleValue; using DoubleVector = std::vector<double>; using generation_t = vespalib::GenerationHandler::generation_t; @@ -62,7 +63,7 @@ vespalib::string denseSpec("tensor(x[2],y[3])"); vespalib::string vec_2d_spec("tensor(x[2])"); Value::UP createTensor(const TensorSpec &spec) { - return EngineOrFactory::get().from_spec(spec); + return SimpleValue::from_spec(spec); } TensorSpec @@ -344,7 +345,7 @@ struct Fixture { } else if (_traits.use_direct_tensor_attribute) { return std::make_shared<DirectTensorAttribute>(_name, _cfg); } else { - return std::make_shared<SerializedTensorAttribute>(_name, _cfg); + return std::make_shared<SerializedFastValueAttribute>(_name, _cfg); } } @@ -904,7 +905,7 @@ public: } std::unique_ptr<Value> createDenseTensor(const TensorSpec &spec) { - return EngineOrFactory::get().from_spec(spec); + return SimpleValue::from_spec(spec); } std::unique_ptr<NearestNeighborBlueprint> make_blueprint(double brute_force_limit = 0.05) { diff --git a/searchlib/src/tests/engine/proto_rpc_adapter/proto_rpc_adapter_test.cpp b/searchlib/src/tests/engine/proto_rpc_adapter/proto_rpc_adapter_test.cpp index 3dbe0d00881..1cd91329912 100644 --- a/searchlib/src/tests/engine/proto_rpc_adapter/proto_rpc_adapter_test.cpp +++ b/searchlib/src/tests/engine/proto_rpc_adapter/proto_rpc_adapter_test.cpp @@ -6,7 +6,9 @@ #include <vespa/searchlib/engine/searchapi.h> #include <vespa/searchlib/engine/docsumapi.h> #include <vespa/searchlib/engine/monitorapi.h> -#include <vespa/fnet/frt/frt.h> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/target.h> +#include <vespa/fnet/frt/rpcrequest.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/slime/binary_format.h> #include <thread> @@ -83,7 +85,7 @@ struct ProtoRpcAdapterTest : ::testing::Test { FRT_Target *connect() { return server.supervisor().GetTarget(server.supervisor().GetListenPort()); } - ~ProtoRpcAdapterTest() = default; + ~ProtoRpcAdapterTest() override = default; }; //----------------------------------------------------------------------------- diff --git a/searchlib/src/tests/features/constant/constant_test.cpp b/searchlib/src/tests/features/constant/constant_test.cpp index e513e20a35c..140c93125b0 100644 --- a/searchlib/src/tests/features/constant/constant_test.cpp +++ b/searchlib/src/tests/features/constant/constant_test.cpp @@ -6,8 +6,8 @@ #include <vespa/searchlib/fef/test/ftlib.h> #include <vespa/searchlib/fef/test/indexenvironment.h> #include <vespa/eval/eval/function.h> +#include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> -#include <vespa/eval/eval/engine_or_factory.h> #include <vespa/eval/eval/value.h> #include <vespa/eval/eval/test/value_compare.h> @@ -16,18 +16,18 @@ using namespace search::fef; using namespace search::fef::indexproperties; using namespace search::fef::test; using namespace search::features; -using vespalib::eval::Function; -using vespalib::eval::Value; using vespalib::eval::DoubleValue; +using vespalib::eval::Function; +using vespalib::eval::SimpleValue; using vespalib::eval::TensorSpec; +using vespalib::eval::Value; using vespalib::eval::ValueType; -using vespalib::eval::EngineOrFactory; namespace { Value::UP make_tensor(const TensorSpec &spec) { - return EngineOrFactory::get().from_spec(spec); + return SimpleValue::from_spec(spec); } } diff --git a/searchlib/src/tests/features/imported_dot_product/imported_dot_product_test.cpp b/searchlib/src/tests/features/imported_dot_product/imported_dot_product_test.cpp index 3a755704c0d..3c9cf209484 100644 --- a/searchlib/src/tests/features/imported_dot_product/imported_dot_product_test.cpp +++ b/searchlib/src/tests/features/imported_dot_product/imported_dot_product_test.cpp @@ -6,9 +6,10 @@ #include <vespa/searchlib/fef/test/ftlib.h> #include <vespa/searchlib/fef/test/rankresult.h> #include <vespa/searchlib/fef/test/dummy_dependency_handler.h> -#include <vespa/eval/eval/engine_or_factory.h> +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/value_codec.h> #include <vespa/vespalib/objects/nbostream.h> -#include <vespa/eval/tensor/dense/dense_tensor.h> using namespace search; using namespace search::attribute; @@ -17,6 +18,9 @@ using namespace search::fef; using namespace search::fef::test; using namespace search::index; +using vespalib::eval::SimpleValue; +using vespalib::eval::TensorSpec; + template <typename T> std::unique_ptr<fef::Anything> create_param(const vespalib::string& param) { Properties props; @@ -109,12 +113,18 @@ struct ArrayFixture : FixtureBase { template <typename ExpectedType> void check_prepare_state_output(const vespalib::eval::Value & tensor, const ExpectedType & expected) { vespalib::nbostream os; - vespalib::eval::EngineOrFactory::get().encode(tensor, os); + encode_value(tensor, os); vespalib::string input_vector(os.data(), os.size()); check_prepare_state_output(".tensor", input_vector, expected); } template <typename ExpectedType> + void check_prepare_state_output(const TensorSpec & spec, const ExpectedType & expected) { + auto value = SimpleValue::from_spec(spec); + check_prepare_state_output(*value, expected); + } + + template <typename ExpectedType> void check_prepare_state_output(const vespalib::string& input_vector, const ExpectedType & expected) { check_prepare_state_output("", input_vector, expected); } @@ -196,25 +206,25 @@ TEST_F("prepareSharedState emits double vector for double imported attribute", A TEST_F("prepareSharedState handles tensor as float from tensor for double imported attribute", ArrayFixture) { f.setup_float_mappings(BasicType::DOUBLE); - vespalib::tensor::DenseTensor<float> tensor(vespalib::eval::ValueType::from_spec("tensor<float>(x[3])"), {10.1, 20.2, 30.3}); + auto tensor = TensorSpec::from_expr("tensor<float>(x[3]):[10.1,20.2,30.3]"); f.template check_prepare_state_output(tensor, dotproduct::ArrayParam<double>({10.1, 20.2, 30.3})); } TEST_F("prepareSharedState handles tensor as double from tensor for double imported attribute", ArrayFixture) { f.setup_float_mappings(BasicType::DOUBLE); - vespalib::tensor::DenseTensor<double> tensor(vespalib::eval::ValueType::from_spec("tensor(x[3])"), {10.1, 20.2, 30.3}); + auto tensor = TensorSpec::from_expr("tensor(x[3]):[10.1,20.2,30.3]"); f.template check_prepare_state_output(tensor, dotproduct::ArrayParam<double>({10.1, 20.2, 30.3})); } TEST_F("prepareSharedState handles tensor as float from tensor for float imported attribute", ArrayFixture) { f.setup_float_mappings(BasicType::FLOAT); - vespalib::tensor::DenseTensor<float> tensor(vespalib::eval::ValueType::from_spec("tensor<float>(x[3])"), {10.1, 20.2, 30.3}); + auto tensor = TensorSpec::from_expr("tensor<float>(x[3]):[10.1,20.2,30.3]"); f.template check_prepare_state_output(tensor, dotproduct::ArrayParam<float>({10.1, 20.2, 30.3})); } TEST_F("prepareSharedState handles tensor as double from tensor for float imported attribute", ArrayFixture) { f.setup_float_mappings(BasicType::FLOAT); - vespalib::tensor::DenseTensor<double> tensor(vespalib::eval::ValueType::from_spec("tensor(x[3])"), {10.1, 20.2, 30.3}); + auto tensor = TensorSpec::from_expr("tensor(x[3]):[10.1,20.2,30.3]"); f.template check_prepare_state_output(tensor, dotproduct::ArrayParam<float>({10.1, 20.2, 30.3})); } diff --git a/searchlib/src/tests/features/tensor/tensor_test.cpp b/searchlib/src/tests/features/tensor/tensor_test.cpp index 116b4ed2bb5..53049c4a385 100644 --- a/searchlib/src/tests/features/tensor/tensor_test.cpp +++ b/searchlib/src/tests/features/tensor/tensor_test.cpp @@ -11,12 +11,12 @@ #include <vespa/searchlib/fef/test/queryenvironment.h> #include <vespa/searchlib/tensor/tensor_attribute.h> #include <vespa/searchlib/tensor/direct_tensor_attribute.h> -#include <vespa/eval/eval/engine_or_factory.h> #include <vespa/eval/eval/function.h> +#include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/value_codec.h> #include <vespa/eval/eval/test/value_compare.h> -#include <vespa/eval/tensor/test/test_utils.h> #include <vespa/vespalib/objects/nbostream.h> using search::feature_t; @@ -28,12 +28,12 @@ using search::AttributeFactory; using search::tensor::TensorAttribute; using search::tensor::DirectTensorAttribute; using search::AttributeVector; -using vespalib::eval::EngineOrFactory; using vespalib::eval::Function; +using vespalib::eval::SimpleValue; +using vespalib::eval::TensorSpec; using vespalib::eval::Value; using vespalib::eval::ValueType; -using vespalib::eval::TensorSpec; -using vespalib::tensor::test::makeTensor; +using vespalib::eval::spec_from_value; using AVC = search::attribute::Config; using AVBT = search::attribute::BasicType; @@ -46,7 +46,7 @@ namespace { Value::UP make_empty(const vespalib::string &type) { - return EngineOrFactory::get().from_spec(TensorSpec(type)); + return SimpleValue::from_spec(TensorSpec(type)); } } @@ -113,10 +113,10 @@ struct ExecFixture DirectTensorAttribute *directAttr = dynamic_cast<DirectTensorAttribute *>(attrs[1].get()); - auto doc_tensor = makeTensor<Value>(TensorSpec("tensor(x{})") - .add({{"x", "a"}}, 3) - .add({{"x", "b"}}, 5) - .add({{"x", "c"}}, 7)); + auto doc_tensor = SimpleValue::from_spec(TensorSpec("tensor(x{})") + .add({{"x", "a"}}, 3) + .add({{"x", "b"}}, 5) + .add({{"x", "c"}}, 7)); tensorAttr->setTensor(1, *doc_tensor); directAttr->set_tensor(1, std::move(doc_tensor)); @@ -129,7 +129,7 @@ struct ExecFixture std::unique_ptr<Value> tensor) { vespalib::nbostream stream; - EngineOrFactory::get().encode(*tensor, stream); + encode_value(*tensor, stream); test.getQueryEnv().getProperties().add(tensorName, vespalib::stringref(stream.peek(), stream.size())); setQueryTensorType(tensorName, tensorTypeSpec); @@ -138,16 +138,16 @@ struct ExecFixture void setupQueryEnvironment() { setQueryTensor("tensorquery", "tensor(q{})", - makeTensor<Value>(TensorSpec("tensor(q{})") - .add({{"q", "d"}}, 11 ) - .add({{"q", "e"}}, 13 ) - .add({{"q", "f"}}, 17 ))); + SimpleValue::from_spec(TensorSpec("tensor(q{})") + .add({{"q", "d"}}, 11 ) + .add({{"q", "e"}}, 13 ) + .add({{"q", "f"}}, 17 ))); setQueryTensor("mappedtensorquery", "tensor(x[2])", - makeTensor<Value>(TensorSpec("tensor(x{},y{})") - .add({{"x", "0"},{"y", "0"}}, 11 ) - .add({{"x", "0"},{"y", "1"}}, 13 ) - .add({{"x", "1"},{"y", "0"}}, 17 ))); + SimpleValue::from_spec(TensorSpec("tensor(x{},y{})") + .add({{"x", "0"},{"y", "0"}}, 11 ) + .add({{"x", "0"},{"y", "1"}}, 13 ) + .add({{"x", "1"},{"y", "0"}}, 17 ))); setQueryTensorType("null", "tensor(q{})"); } const Value &extractTensor(uint32_t docid) { @@ -163,28 +163,28 @@ struct ExecFixture TEST_F("require that tensor attribute can be extracted as tensor in attribute feature", ExecFixture("attribute(tensorattr)")) { - EXPECT_EQUAL(*makeTensor<Value>(TensorSpec("tensor(x{})") - .add({{"x", "b"}}, 5) - .add({{"x", "c"}}, 7) - .add({{"x", "a"}}, 3)), f.execute()); + EXPECT_EQUAL(TensorSpec("tensor(x{})") + .add({{"x", "b"}}, 5) + .add({{"x", "c"}}, 7) + .add({{"x", "a"}}, 3), spec_from_value(f.execute())); } TEST_F("require that direct tensor attribute can be extracted in attribute feature", ExecFixture("attribute(directattr)")) { - EXPECT_EQUAL(*makeTensor<Value>(TensorSpec("tensor(x{})") - .add({{"x", "b"}}, 5) - .add({{"x", "c"}}, 7) - .add({{"x", "a"}}, 3)), f.execute()); + EXPECT_EQUAL(TensorSpec("tensor(x{})") + .add({{"x", "b"}}, 5) + .add({{"x", "c"}}, 7) + .add({{"x", "a"}}, 3), spec_from_value(f.execute())); } TEST_F("require that tensor from query can be extracted as tensor in query feature", ExecFixture("query(tensorquery)")) { - EXPECT_EQUAL(*makeTensor<Value>(TensorSpec("tensor(q{})") - .add({{"q", "f"}}, 17) - .add({{"q", "d"}}, 11) - .add({{"q", "e"}}, 13)), f.execute()); + EXPECT_EQUAL(TensorSpec("tensor(q{})") + .add({{"q", "f"}}, 17) + .add({{"q", "d"}}, 11) + .add({{"q", "e"}}, 13), spec_from_value(f.execute())); } TEST_F("require that empty tensor is created if attribute does not exists", @@ -217,9 +217,9 @@ TEST_F("require that empty tensor with correct type is returned by direct tensor TEST_F("require that wrong tensor type from query tensor gives empty tensor", ExecFixture("query(mappedtensorquery)")) { - EXPECT_EQUAL(*makeTensor<Value>(TensorSpec("tensor(x[2])") - .add({{"x", 0}}, 0) - .add({{"x", 1}}, 0)), f.execute()); + EXPECT_EQUAL(TensorSpec("tensor(x[2])") + .add({{"x", 0}}, 0) + .add({{"x", 1}}, 0), spec_from_value(f.execute())); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/features/tensor_from_labels/tensor_from_labels_test.cpp b/searchlib/src/tests/features/tensor_from_labels/tensor_from_labels_test.cpp index 33fd7c856b3..2e83d2acbf2 100644 --- a/searchlib/src/tests/features/tensor_from_labels/tensor_from_labels_test.cpp +++ b/searchlib/src/tests/features/tensor_from_labels/tensor_from_labels_test.cpp @@ -12,8 +12,9 @@ #include <vespa/searchlib/fef/test/indexenvironment.h> #include <vespa/searchlib/fef/test/indexenvironmentbuilder.h> #include <vespa/searchlib/fef/test/queryenvironment.h> -#include <vespa/eval/eval/engine_or_factory.h> #include <vespa/eval/eval/function.h> +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value.h> #include <vespa/eval/eval/test/value_compare.h> @@ -27,7 +28,7 @@ using search::StringAttribute; using vespalib::eval::Value; using vespalib::eval::Function; using vespalib::eval::TensorSpec; -using vespalib::eval::EngineOrFactory; +using vespalib::eval::SimpleValue; typedef search::attribute::Config AVC; typedef search::attribute::BasicType AVBT; @@ -36,7 +37,7 @@ typedef search::AttributeVector::SP AttributePtr; typedef FtTestApp FTA; Value::UP make_tensor(const TensorSpec &spec) { - return EngineOrFactory::get().from_spec(spec); + return SimpleValue::from_spec(spec); } Value::UP make_empty(const vespalib::string &type) { diff --git a/searchlib/src/tests/features/tensor_from_weighted_set/tensor_from_weighted_set_test.cpp b/searchlib/src/tests/features/tensor_from_weighted_set/tensor_from_weighted_set_test.cpp index 282c8134754..c339d8dae63 100644 --- a/searchlib/src/tests/features/tensor_from_weighted_set/tensor_from_weighted_set_test.cpp +++ b/searchlib/src/tests/features/tensor_from_weighted_set/tensor_from_weighted_set_test.cpp @@ -12,8 +12,8 @@ #include <vespa/searchlib/fef/test/indexenvironment.h> #include <vespa/searchlib/fef/test/indexenvironmentbuilder.h> #include <vespa/searchlib/fef/test/queryenvironment.h> -#include <vespa/eval/eval/engine_or_factory.h> #include <vespa/eval/eval/function.h> +#include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value.h> #include <vespa/eval/eval/test/value_compare.h> @@ -28,7 +28,7 @@ using search::StringAttribute; using vespalib::eval::Value; using vespalib::eval::Function; using vespalib::eval::TensorSpec; -using vespalib::eval::EngineOrFactory; +using vespalib::eval::SimpleValue; typedef search::attribute::Config AVC; typedef search::attribute::BasicType AVBT; @@ -37,7 +37,7 @@ typedef search::AttributeVector::SP AttributePtr; typedef FtTestApp FTA; Value::UP make_tensor(const TensorSpec &spec) { - return EngineOrFactory::get().from_spec(spec); + return SimpleValue::from_spec(spec); } Value::UP make_empty(const vespalib::string &type) { diff --git a/searchlib/src/tests/memoryindex/url_field_inverter/url_field_inverter_test.cpp b/searchlib/src/tests/memoryindex/url_field_inverter/url_field_inverter_test.cpp index 86c58c11c09..cc28a8fd479 100644 --- a/searchlib/src/tests/memoryindex/url_field_inverter/url_field_inverter_test.cpp +++ b/searchlib/src/tests/memoryindex/url_field_inverter/url_field_inverter_test.cpp @@ -227,6 +227,8 @@ struct UrlFieldInverterTest : public ::testing::Test { _inverters[urlField._hostname].get()); } + ~UrlFieldInverterTest() override; + void invertDocument(uint32_t docId, const Document &doc) { _urlInverter->invertField(docId, doc.getValue(url)); } @@ -245,6 +247,8 @@ struct UrlFieldInverterTest : public ::testing::Test { } }; +UrlFieldInverterTest::~UrlFieldInverterTest() = default; + struct SingleInverterTest : public UrlFieldInverterTest { SingleInverterTest() : UrlFieldInverterTest(CollectionType::SINGLE) {} }; diff --git a/searchlib/src/tests/query/stackdumpquerycreator_test.cpp b/searchlib/src/tests/query/stackdumpquerycreator_test.cpp index 7f682b380bf..cb3989387b7 100644 --- a/searchlib/src/tests/query/stackdumpquerycreator_test.cpp +++ b/searchlib/src/tests/query/stackdumpquerycreator_test.cpp @@ -32,9 +32,9 @@ void appendString(RawBuf &buf, const string &s) { } void appendNumTerm(RawBuf &buf, const string &term_string) { - uint8_t typefield = ParseItem::ITEM_NUMTERM | - ParseItem::IF_WEIGHT | - ParseItem::IF_UNIQUEID; + uint8_t typefield = static_cast<uint8_t>(ParseItem::ITEM_NUMTERM) | + static_cast<uint8_t>(ParseItem::IF_WEIGHT) | + static_cast<uint8_t>(ParseItem::IF_UNIQUEID); buf.append(typefield); buf.appendCompressedNumber(2); // weight buf.appendCompressedPositiveNumber(42); // id diff --git a/searchlib/src/tests/query/streaming_query_test.cpp b/searchlib/src/tests/query/streaming_query_test.cpp index 94851c72886..dbd186fdcb5 100644 --- a/searchlib/src/tests/query/streaming_query_test.cpp +++ b/searchlib/src/tests/query/streaming_query_test.cpp @@ -226,7 +226,7 @@ public: virtual bool getRewriteFloatTerms() const override { return true; } }; -const char TERM_UNIQ = ParseItem::ITEM_TERM | ParseItem::IF_UNIQUEID; +const char TERM_UNIQ = static_cast<char>(ParseItem::ITEM_TERM) | static_cast<char>(ParseItem::IF_UNIQUEID); TEST("e is not rewritten even if allowed") { const char term[6] = {TERM_UNIQ, 3, 1, 'c', 1, 'e'}; diff --git a/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_bench.cpp b/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_bench.cpp index e1d0e034a89..10e03238e81 100644 --- a/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_bench.cpp +++ b/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_bench.cpp @@ -118,8 +118,8 @@ Test::Main() return -1; } _type = _argv[1]; - _strict = _argv[2] == vespalib::string("strict"); - _optimize = _argv[3] == vespalib::string("optimize"); + _strict = vespalib::string(_argv[2]) == vespalib::string("strict"); + _optimize = vespalib::string(_argv[3]) == vespalib::string("optimize"); _numSearch = strtoul(_argv[4], NULL, 0); _numDocs = strtoul(_argv[5], NULL, 0); for (int i(6); i < _argc; i++) { diff --git a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp index 23cb3831b6d..ad450a91f33 100644 --- a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp +++ b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp @@ -3,7 +3,8 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/eval/eval/engine_or_factory.h> +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/tensor_spec.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/common/feature.h> #include <vespa/searchlib/fef/matchdata.h> @@ -27,7 +28,7 @@ using vespalib::eval::Value; using vespalib::eval::ValueType; using vespalib::eval::CellType; using vespalib::eval::TensorSpec; -using vespalib::eval::EngineOrFactory; +using vespalib::eval::SimpleValue; using search::tensor::DistanceFunction; using search::attribute::DistanceMetric; @@ -41,7 +42,7 @@ DistanceFunction::UP euclid_d = search::tensor::make_distance_function(DistanceM DistanceFunction::UP euclid_f = search::tensor::make_distance_function(DistanceMetric::Euclidean, CellType::FLOAT); std::unique_ptr<Value> createTensor(const TensorSpec &spec) { - return EngineOrFactory::get().from_spec(spec); + return SimpleValue::from_spec(spec); } std::unique_ptr<Value> createTensor(const vespalib::string& type_spec, double v1, double v2) { diff --git a/searchlib/src/tests/queryeval/predicate/predicate_search_test.cpp b/searchlib/src/tests/queryeval/predicate/predicate_search_test.cpp index efa01b79580..382e2bdb8ff 100644 --- a/searchlib/src/tests/queryeval/predicate/predicate_search_test.cpp +++ b/searchlib/src/tests/queryeval/predicate/predicate_search_test.cpp @@ -41,6 +41,8 @@ public: { } + ~MyPostingList() override; + bool next(uint32_t doc_id) override { if (_index < _entries.size()) { while (_entries[_index].first <= doc_id) { @@ -70,6 +72,8 @@ public: uint32_t getInterval() const override { return _interval; } }; +MyPostingList::~MyPostingList() = default; + template <int N> vector<PredicatePostingList::UP> make_posting_lists_vector(MyPostingList (&plists)[N]) { diff --git a/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp b/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp index 4ed366a6cc2..b5d4fb3130f 100644 --- a/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp +++ b/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp @@ -3,27 +3,24 @@ LOG_SETUP("dense_tensor_store_test"); #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/tensor/dense_tensor_store.h> -#include <vespa/eval/eval/engine_or_factory.h> +#include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> -#include <vespa/eval/eval/test/value_compare.h> #include <vespa/eval/eval/value.h> #include <vespa/eval/eval/value_type.h> -#include <vespa/eval/tensor/default_tensor_engine.h> -#include <vespa/eval/tensor/dense/mutable_dense_tensor_view.h> +#include <vespa/eval/eval/test/value_compare.h> using search::tensor::DenseTensorStore; +using vespalib::eval::SimpleValue; using vespalib::eval::TensorSpec; using vespalib::eval::Value; using vespalib::eval::ValueType; -using vespalib::eval::EngineOrFactory; -using vespalib::tensor::MutableDenseTensorView; using EntryRef = DenseTensorStore::EntryRef; Value::UP makeTensor(const TensorSpec &spec) { - return EngineOrFactory::get().from_spec(spec); + return SimpleValue::from_spec(spec); } struct Fixture @@ -47,8 +44,8 @@ struct Fixture assertTensorView(ref, *expTensor); } void assertTensorView(EntryRef ref, const Value &expTensor) { - MutableDenseTensorView actTensor(store.type()); - store.getTensor(ref, actTensor); + auto cells = store.get_typed_cells(ref); + vespalib::eval::DenseValueView actTensor(store.type(), cells); EXPECT_EQUAL(expTensor, actTensor); } }; diff --git a/searchlib/src/tests/tensor/direct_tensor_store/direct_tensor_store_test.cpp b/searchlib/src/tests/tensor/direct_tensor_store/direct_tensor_store_test.cpp index a6f4d56425b..f5160e4b879 100644 --- a/searchlib/src/tests/tensor/direct_tensor_store/direct_tensor_store_test.cpp +++ b/searchlib/src/tests/tensor/direct_tensor_store/direct_tensor_store_test.cpp @@ -2,14 +2,15 @@ #include <vespa/searchlib/tensor/direct_tensor_store.h> #include <vespa/vespalib/gtest/gtest.h> -#include <vespa/eval/eval/engine_or_factory.h> +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value.h> #include <vespa/vespalib/datastore/datastore.hpp> using namespace search::tensor; using vespalib::datastore::EntryRef; -using vespalib::eval::EngineOrFactory; +using vespalib::eval::SimpleValue; using vespalib::eval::TensorSpec; using vespalib::eval::Value; using vespalib::eval::ValueType; @@ -40,7 +41,7 @@ public: Value::UP make_tensor(const TensorSpec& spec) { - auto value = EngineOrFactory::get().from_spec(spec); + auto value = SimpleValue::from_spec(spec); return std::make_unique<MockBigTensor>(std::move(value)); } diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index 95fde42724b..dfcbfbbbe2b 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -8,7 +8,6 @@ #include "attribute_blueprint_params.h" #include "document_weight_or_filter_search.h" #include <vespa/eval/eval/value.h> -#include <vespa/eval/tensor/dense/dense_tensor_view.h> #include <vespa/searchlib/common/location.h> #include <vespa/searchlib/common/locationiterators.h> #include <vespa/searchlib/common/matching_elements_fields.h> diff --git a/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp b/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp index 148d18f79ff..ffaec9d0779 100644 --- a/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp +++ b/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp @@ -7,8 +7,10 @@ #include "singlenumericattribute.hpp" #include "singlestringattribute.h" #include "singleboolattribute.h" +#include <vespa/eval/eval/fast_value.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/tensor/serialized_tensor_attribute.h> +#include <vespa/searchlib/tensor/serialized_fast_value_attribute.h> namespace search { @@ -46,7 +48,7 @@ AttributeFactory::createSingleStd(stringref name, const Config & info) if (info.tensorType().is_dense()) { return std::make_shared<tensor::DenseTensorAttribute>(name, info); } else { - return std::make_shared<tensor::SerializedTensorAttribute>(name, info); + return std::make_shared<tensor::SerializedFastValueAttribute>(name, info); } case BasicType::REFERENCE: return std::make_shared<attribute::ReferenceAttribute>(name, info); diff --git a/searchlib/src/vespa/searchlib/attribute/integerbase.cpp b/searchlib/src/vespa/searchlib/attribute/integerbase.cpp index 084bb85c28c..c288d0507d7 100644 --- a/searchlib/src/vespa/searchlib/attribute/integerbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/integerbase.cpp @@ -3,6 +3,7 @@ #include "integerbase.hpp" #include "attributevector.hpp" #include <vespa/document/fieldvalue/fieldvalue.h> +#include <charconv> namespace search { @@ -14,7 +15,7 @@ IntegerAttribute::IntegerAttribute(const vespalib::string & name, const Config & { } -IntegerAttribute::~IntegerAttribute() { } +IntegerAttribute::~IntegerAttribute() = default; uint32_t IntegerAttribute::clearDoc(DocId doc) { @@ -27,14 +28,23 @@ uint32_t IntegerAttribute::clearDoc(DocId doc) return removed; } +namespace { + +// TODO Move to vespalib::to_string and template on value type +vespalib::string +to_string(int64_t v) { + char tmp[32]; + auto res = std::to_chars(tmp, tmp + sizeof(tmp) - 1, v, 10); + return vespalib::string(tmp, res.ptr - tmp); +} + +} uint32_t IntegerAttribute::get(DocId doc, WeightedString * s, uint32_t sz) const { WeightedInt * v = new WeightedInt[sz]; unsigned num(static_cast<const AttributeVector *>(this)->get(doc, v, sz)); for(unsigned i(0); i < num; i++) { - char tmp[32]; - snprintf(tmp, sizeof(tmp), "%" PRId64, v[i].getValue()); - s[i] = WeightedString(tmp, v[i].getWeight()); + s[i] = WeightedString(to_string(v[i].getValue()), v[i].getWeight()); } delete [] v; return num; @@ -49,8 +59,15 @@ uint32_t IntegerAttribute::get(DocId doc, WeightedConstChar * v, uint32_t sz) co } const char * IntegerAttribute::getString(DocId doc, char * s, size_t sz) const { - largeint_t v = getInt(doc); - snprintf(s, sz, "%" PRId64, v); + if (sz > 1) { + largeint_t v = getInt(doc); + auto res = std::to_chars(s, s + sz - 1, v, 10); + if (res.ec == std::errc()) { + res.ptr[0] = 0; + } else { + s[0] = 0; + } + } return s; } uint32_t IntegerAttribute::get(DocId doc, vespalib::string * s, uint32_t sz) const @@ -58,9 +75,7 @@ uint32_t IntegerAttribute::get(DocId doc, vespalib::string * s, uint32_t sz) con largeint_t * v = new largeint_t[sz]; unsigned num(static_cast<const AttributeVector *>(this)->get(doc, v, sz)); for(unsigned i(0); i < num; i++) { - char tmp[32]; - snprintf(tmp, sizeof(tmp), "%" PRId64, v[i]); - s[i] = tmp; + s[i] = to_string(v[i]); } delete [] v; return num; diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp index bafb859a246..1def5dab060 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp @@ -251,7 +251,7 @@ template <typename B, typename M> void MultiValueAttribute<B, M>::clearDocs(DocId lidLow, DocId lidLimit) { - _mvMapping.clearDocs(lidLow, lidLimit, [=](uint32_t docId) { this->clearDoc(docId); }); + _mvMapping.clearDocs(lidLow, lidLimit, [this](uint32_t docId) { this->clearDoc(docId); }); } diff --git a/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h b/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h index a5b570a4864..f311907bd91 100644 --- a/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h +++ b/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h @@ -9,7 +9,7 @@ namespace search { struct NotImplementedAttribute : AttributeVector { using AttributeVector::AttributeVector; - [[noreturn]] virtual void notImplemented() const ; + void notImplemented [[noreturn]] () const; uint32_t getValueCount(DocId) const override; largeint_t getInt(DocId) const override; diff --git a/searchlib/src/vespa/searchlib/common/geo_location.h b/searchlib/src/vespa/searchlib/common/geo_location.h index 261951caf3e..506434e75e0 100644 --- a/searchlib/src/vespa/searchlib/common/geo_location.h +++ b/searchlib/src/vespa/searchlib/common/geo_location.h @@ -20,6 +20,7 @@ struct GeoLocation static constexpr int32_t range_high = std::numeric_limits<int32_t>::max(); static constexpr uint32_t radius_inf = std::numeric_limits<uint32_t>::max(); struct Point { + Point(int32_t x_in, int32_t y_in) : x(x_in), y(y_in) {} const int32_t x; const int32_t y; Point() = delete; @@ -79,7 +80,7 @@ struct GeoLocation int32_t x = 0; int32_t y = 0; vespalib::geo::ZCurve::decode(zcurve_encoded_xy, &x, &y); - return inside_limit(Point{x, y}); + return inside_limit(Point(x, y)); } private: diff --git a/searchlib/src/vespa/searchlib/engine/search_protocol_metrics.h b/searchlib/src/vespa/searchlib/engine/search_protocol_metrics.h index 378a045e32b..9dc609091f0 100644 --- a/searchlib/src/vespa/searchlib/engine/search_protocol_metrics.h +++ b/searchlib/src/vespa/searchlib/engine/search_protocol_metrics.h @@ -2,7 +2,9 @@ #pragma once -#include <vespa/metrics/metrics.h> +#include <vespa/metrics/valuemetric.h> +#include <vespa/metrics/countmetric.h> +#include <vespa/metrics/metricset.h> #include <mutex> namespace search::engine { diff --git a/searchlib/src/vespa/searchlib/expression/resultvector.h b/searchlib/src/vespa/searchlib/expression/resultvector.h index cf2f0730c37..2c10a26dd9a 100644 --- a/searchlib/src/vespa/searchlib/expression/resultvector.h +++ b/searchlib/src/vespa/searchlib/expression/resultvector.h @@ -82,6 +82,7 @@ public: DECLARE_NBO_SERIALIZE; using Vector = std::vector<B>; using BaseType = B; + ~ResultNodeVectorT() override; const Vector & getVector() const { return _result; } Vector & getVector() { return _result; } const ResultNode * find(const ResultNode & key) const override; @@ -109,6 +110,9 @@ private: }; template <typename B, typename C, typename G> +ResultNodeVectorT<B, C, G>::~ResultNodeVectorT() = default; + +template <typename B, typename C, typename G> ResultNodeVector & ResultNodeVectorT<B, C, G>::set(size_t index, const ResultNode & node) { _result[index].set(node); diff --git a/searchlib/src/vespa/searchlib/features/CMakeLists.txt b/searchlib/src/vespa/searchlib/features/CMakeLists.txt index 7bb4849bb87..271aa73ddcb 100644 --- a/searchlib/src/vespa/searchlib/features/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/features/CMakeLists.txt @@ -35,6 +35,7 @@ vespa_add_library(searchlib_features OBJECT matchesfeature.cpp matchfeature.cpp max_reduce_prod_join_replacer.cpp + mutable_dense_value_view.cpp native_dot_product_feature.cpp nativeattributematchfeature.cpp nativefieldmatchfeature.cpp diff --git a/searchlib/src/vespa/searchlib/features/attributefeature.cpp b/searchlib/src/vespa/searchlib/features/attributefeature.cpp index eac69f87587..80d9a305ef4 100644 --- a/searchlib/src/vespa/searchlib/features/attributefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/attributefeature.cpp @@ -469,7 +469,7 @@ createTensorAttributeExecutor(const IAttributeVector *attribute, const vespalib: tensorType.to_spec().c_str()); return ConstantTensorExecutor::createEmpty(tensorType, stash); } - if (tensorAttribute->supports_extract_dense_view()) { + if (tensorAttribute->supports_extract_cells_ref()) { return stash.create<DenseTensorAttributeExecutor>(*tensorAttribute); } if (tensorAttribute->supports_get_tensor_ref()) { diff --git a/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h b/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h index 44e530821a2..f95cbfa4726 100644 --- a/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h +++ b/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h @@ -6,8 +6,8 @@ #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value.h> #include <vespa/eval/eval/value_type.h> -#include <vespa/eval/eval/engine_or_factory.h> -#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/value_codec.h> #include <vespa/vespalib/util/stash.h> namespace search::features { @@ -32,9 +32,9 @@ public: return stash.create<ConstantTensorExecutor>(std::move(tensor)); } static fef::FeatureExecutor &createEmpty(const vespalib::eval::ValueType &valueType, vespalib::Stash &stash) { - const auto engine = vespalib::eval::EngineOrFactory::get(); + const auto &factory = vespalib::eval::FastValueBuilderFactory::get(); auto spec = vespalib::eval::TensorSpec(valueType.to_spec()); - return stash.create<ConstantTensorExecutor>(engine.from_spec(spec)); + return stash.create<ConstantTensorExecutor>(vespalib::eval::value_from_spec(spec, factory)); } static fef::FeatureExecutor &createEmpty(vespalib::Stash &stash) { return createEmpty(vespalib::eval::ValueType::double_type(), stash); diff --git a/searchlib/src/vespa/searchlib/features/dense_tensor_attribute_executor.cpp b/searchlib/src/vespa/searchlib/features/dense_tensor_attribute_executor.cpp index 8f7d82adadd..0de375b7acb 100644 --- a/searchlib/src/vespa/searchlib/features/dense_tensor_attribute_executor.cpp +++ b/searchlib/src/vespa/searchlib/features/dense_tensor_attribute_executor.cpp @@ -4,7 +4,6 @@ #include <vespa/searchlib/tensor/i_tensor_attribute.h> using search::tensor::ITensorAttribute; -using vespalib::tensor::MutableDenseTensorView; namespace search::features { @@ -18,7 +17,7 @@ DenseTensorAttributeExecutor(const ITensorAttribute& attribute) void DenseTensorAttributeExecutor::execute(uint32_t docId) { - _attribute.extract_dense_view(docId, _tensorView); + _tensorView.setCells(_attribute.extract_cells_ref(docId)); outputs().set_object(0, _tensorView); } diff --git a/searchlib/src/vespa/searchlib/features/dense_tensor_attribute_executor.h b/searchlib/src/vespa/searchlib/features/dense_tensor_attribute_executor.h index a8a84447c88..fc09d37f35a 100644 --- a/searchlib/src/vespa/searchlib/features/dense_tensor_attribute_executor.h +++ b/searchlib/src/vespa/searchlib/features/dense_tensor_attribute_executor.h @@ -4,7 +4,7 @@ #include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/eval/eval/value.h> -#include <vespa/eval/tensor/dense/mutable_dense_tensor_view.h> +#include "mutable_dense_value_view.h" namespace search::tensor { class ITensorAttribute; } namespace search::features { @@ -17,7 +17,7 @@ class DenseTensorAttributeExecutor : public fef::FeatureExecutor { private: const search::tensor::ITensorAttribute& _attribute; - vespalib::tensor::MutableDenseTensorView _tensorView; + mutable_value::MutableDenseValueView _tensorView; public: DenseTensorAttributeExecutor(const search::tensor::ITensorAttribute& attribute); diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp index 83beee98634..06f0de631be 100644 --- a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp @@ -10,7 +10,8 @@ #include <vespa/searchlib/attribute/floatbase.h> #include <vespa/searchlib/attribute/multinumericattribute.h> #include <vespa/searchlib/attribute/multienumattribute.h> -#include <vespa/eval/eval/engine_or_factory.h> +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/value_codec.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/stash.h> @@ -19,7 +20,7 @@ LOG_SETUP(".features.dotproduct"); using namespace search::attribute; using namespace search::fef; -using vespalib::eval::EngineOrFactory; +using vespalib::eval::FastValueBuilderFactory; using vespalib::eval::TypedCells; using vespalib::hwaccelrated::IAccelrated; @@ -500,7 +501,7 @@ template <typename T> ArrayParam<T>::ArrayParam(vespalib::nbostream & stream) { using vespalib::typify_invoke; using vespalib::eval::TypifyCellType; - auto tensor = EngineOrFactory::get().decode(stream); + auto tensor = vespalib::eval::decode_value(stream, FastValueBuilderFactory::get()); if (tensor->type().is_dense()) { TypedCells cells = tensor->cells(); typify_invoke<1,TypifyCellType,CopyCellsToVector<T>>(cells.type, cells, values); diff --git a/searchlib/src/vespa/searchlib/features/mutable_dense_value_view.cpp b/searchlib/src/vespa/searchlib/features/mutable_dense_value_view.cpp new file mode 100644 index 00000000000..b0d669e8aaa --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/mutable_dense_value_view.cpp @@ -0,0 +1,14 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "mutable_dense_value_view.h" + +namespace search::features::mutable_value { + +MutableDenseValueView::MutableDenseValueView(const ValueType &type_in) + : _type(type_in), + _cells() +{ + assert(_type.is_dense()); +} + +} diff --git a/searchlib/src/vespa/searchlib/features/mutable_dense_value_view.h b/searchlib/src/vespa/searchlib/features/mutable_dense_value_view.h new file mode 100644 index 00000000000..d4e813ae927 --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/mutable_dense_value_view.h @@ -0,0 +1,33 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/value.h> +#include <cassert> + +namespace search::features::mutable_value { + +using namespace vespalib::eval; + +/** + * A dense tensor with a cells reference that can be modified. + */ +class MutableDenseValueView : public Value { +private: + const ValueType _type; + TypedCells _cells; +public: + MutableDenseValueView(const ValueType &type_in); + void setCells(TypedCells cells_in) { + assert(cells_in.type == _type.cell_type()); + _cells = cells_in; + } + const ValueType &type() const final override { return _type; } + TypedCells cells() const final override { return _cells; } + const Index &index() const final override { return TrivialIndex::get(); } + vespalib::MemoryUsage get_memory_usage() const final override { + return self_memory_usage<MutableDenseValueView>(); + } +}; + +} diff --git a/searchlib/src/vespa/searchlib/features/onnx_feature.cpp b/searchlib/src/vespa/searchlib/features/onnx_feature.cpp index f05ee60076f..87e5ef2a5c2 100644 --- a/searchlib/src/vespa/searchlib/features/onnx_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/onnx_feature.cpp @@ -4,8 +4,7 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/onnx_model.h> #include <vespa/searchlib/fef/featureexecutor.h> -#include <vespa/eval/tensor/dense/dense_tensor_view.h> -#include <vespa/eval/tensor/dense/mutable_dense_tensor_view.h> +#include <vespa/eval/eval/value.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/stash.h> @@ -21,8 +20,7 @@ using search::fef::ParameterList; using vespalib::Stash; using vespalib::eval::ValueType; using vespalib::make_string_short::fmt; -using vespalib::tensor::MutableDenseTensorView; -using vespalib::tensor::Onnx; +using vespalib::eval::Onnx; namespace search::features { diff --git a/searchlib/src/vespa/searchlib/features/onnx_feature.h b/searchlib/src/vespa/searchlib/features/onnx_feature.h index 19c6338d2ee..5a45b26f1f6 100644 --- a/searchlib/src/vespa/searchlib/features/onnx_feature.h +++ b/searchlib/src/vespa/searchlib/features/onnx_feature.h @@ -3,7 +3,7 @@ #pragma once #include <vespa/searchlib/fef/blueprint.h> -#include <vespa/eval/tensor/dense/onnx_wrapper.h> +#include <vespa/eval/onnx/onnx_wrapper.h> namespace search::features { @@ -12,7 +12,7 @@ namespace search::features { **/ class OnnxBlueprint : public fef::Blueprint { private: - using Onnx = vespalib::tensor::Onnx; + using Onnx = vespalib::eval::Onnx; std::unique_ptr<Onnx> _model; Onnx::WireInfo _wire_info; public: diff --git a/searchlib/src/vespa/searchlib/features/queryfeature.cpp b/searchlib/src/vespa/searchlib/features/queryfeature.cpp index 9cded69d5de..c6196fcbc7f 100644 --- a/searchlib/src/vespa/searchlib/features/queryfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/queryfeature.cpp @@ -12,6 +12,8 @@ #include <vespa/searchlib/fef/feature_type.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/eval/eval/value_type.h> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/eval/eval/fast_value.h> #include <vespa/vespalib/locale/c.h> #include <cerrno> @@ -119,7 +121,7 @@ createTensorExecutor(const IQueryEnvironment &env, if (prop.found() && !prop.get().empty()) { const vespalib::string &value = prop.get(); vespalib::nbostream stream(value.data(), value.size()); - auto tensor = vespalib::eval::EngineOrFactory::get().decode(stream); + auto tensor = vespalib::eval::decode_value(stream, vespalib::eval::FastValueBuilderFactory::get()); if (!TensorDataType::isAssignableType(valueType, tensor->type())) { LOG(warning, "Query feature type is '%s' but other tensor type is '%s'", valueType.to_spec().c_str(), tensor->type().to_spec().c_str()); diff --git a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp index c8cc573c0d2..8cba6013627 100644 --- a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp @@ -5,9 +5,8 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/features/rankingexpression/feature_name_extractor.h> -#include <vespa/eval/eval/engine_or_factory.h> -#include <vespa/eval/tensor/default_tensor_engine.h> #include <vespa/eval/eval/param_usage.h> +#include <vespa/eval/eval/fast_value.h> #include <vespa/log/log.h> LOG_SETUP(".features.rankingexpression"); @@ -18,7 +17,7 @@ using vespalib::ConstArrayRef; using vespalib::eval::CompileCache; using vespalib::eval::CompiledFunction; using vespalib::eval::DoubleValue; -using vespalib::eval::EngineOrFactory; +using vespalib::eval::FastValueBuilderFactory; using vespalib::eval::Function; using vespalib::eval::InterpretedFunction; using vespalib::eval::LazyParams; @@ -326,7 +325,7 @@ RankingExpressionBlueprint::setup(const fef::IIndexEnvironment &env, } } } else { - _interpreted_function.reset(new InterpretedFunction(EngineOrFactory::get(), + _interpreted_function.reset(new InterpretedFunction(FastValueBuilderFactory::get(), *rank_function, node_types)); } } diff --git a/searchlib/src/vespa/searchlib/features/tensor_from_attribute_executor.h b/searchlib/src/vespa/searchlib/features/tensor_from_attribute_executor.h index a6d3de54d00..f4a5b0b8d0a 100644 --- a/searchlib/src/vespa/searchlib/features/tensor_from_attribute_executor.h +++ b/searchlib/src/vespa/searchlib/features/tensor_from_attribute_executor.h @@ -3,9 +3,11 @@ #pragma once #include <vespa/searchcommon/attribute/iattributevector.h> +#include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/value.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/eval/tensor/sparse/direct_sparse_tensor_builder.h> + +using vespalib::eval::FastValueBuilderFactory; namespace search::features { @@ -20,6 +22,7 @@ private: const search::attribute::IAttributeVector *_attribute; vespalib::eval::ValueType _type; WeightedBufferType _attrBuffer; + std::vector<vespalib::stringref> _addr_ref; std::unique_ptr<vespalib::eval::Value> _tensor; public: @@ -28,9 +31,11 @@ public: : _attribute(attribute), _type(vespalib::eval::ValueType::tensor_type({{dimension}})), _attrBuffer(), + _addr_ref(), _tensor() { _attrBuffer.allocate(_attribute->getMaxValueCount()); + _addr_ref.reserve(1); } void execute(uint32_t docId) override; }; @@ -40,15 +45,16 @@ void TensorFromAttributeExecutor<WeightedBufferType>::execute(uint32_t docId) { _attrBuffer.fill(*_attribute, docId); - vespalib::tensor::DirectSparseTensorBuilder<double> builder(_type); - builder.reserve(_attrBuffer.size()); - vespalib::tensor::SparseTensorAddressBuilder address; + auto factory = FastValueBuilderFactory::get(); + auto builder = factory.create_value_builder<double>(_type, 1, 1, _attrBuffer.size()); for (size_t i = 0; i < _attrBuffer.size(); ++i) { - address.clear(); - address.add(vespalib::string(_attrBuffer[i].value())); - builder.insertCell(address, _attrBuffer[i].weight(), [](double, double v){ return v; }); + vespalib::string label(_attrBuffer[i].value()); + _addr_ref.clear(); + _addr_ref.push_back(label); + auto cell_array = builder->add_subspace(_addr_ref); + cell_array[0] = _attrBuffer[i].weight(); } - _tensor = builder.build(); + _tensor = builder->build(std::move(builder)); outputs().set_object(0, *_tensor); } diff --git a/searchlib/src/vespa/searchlib/features/tensor_from_labels_feature.cpp b/searchlib/src/vespa/searchlib/features/tensor_from_labels_feature.cpp index e6c5f22dd16..e4f0a010ae2 100644 --- a/searchlib/src/vespa/searchlib/features/tensor_from_labels_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/tensor_from_labels_feature.cpp @@ -8,6 +8,7 @@ #include <vespa/searchlib/fef/feature_type.h> #include <vespa/searchcommon/attribute/attributecontent.h> #include <vespa/searchcommon/attribute/iattributevector.h> +#include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/value_type.h> #include <vespa/log/log.h> @@ -17,8 +18,7 @@ using namespace search::fef; using search::attribute::IAttributeVector; using search::attribute::WeightedConstCharContent; using search::attribute::WeightedStringContent; -using vespalib::tensor::DirectSparseTensorBuilder; -using vespalib::tensor::SparseTensorAddressBuilder; +using vespalib::eval::FastValueBuilderFactory; using vespalib::eval::ValueType; using search::fef::FeatureType; @@ -91,15 +91,16 @@ createQueryExecutor(const search::fef::IQueryEnvironment &env, if (prop.found() && !prop.get().empty()) { std::vector<vespalib::string> vector; ArrayParser::parse(prop.get(), vector); - DirectSparseTensorBuilder<double> tensorBuilder(type); - tensorBuilder.reserve(vector.size()); - SparseTensorAddressBuilder address; + auto factory = FastValueBuilderFactory::get(); + auto builder = factory.create_value_builder<double>(type, 1, 1, vector.size()); + std::vector<vespalib::stringref> addr_ref; for (const auto &elem : vector) { - address.clear(); - address.add(elem); - tensorBuilder.insertCell(address, 1.0, [](double, double v){ return v; }); + addr_ref.clear(); + addr_ref.push_back(elem); + auto cell_array = builder->add_subspace(addr_ref); + cell_array[0] = 1.0; } - return ConstantTensorExecutor::create(tensorBuilder.build(), stash); + return ConstantTensorExecutor::create(builder->build(std::move(builder)), stash); } return ConstantTensorExecutor::createEmpty(type, stash); } diff --git a/searchlib/src/vespa/searchlib/features/tensor_from_weighted_set_feature.cpp b/searchlib/src/vespa/searchlib/features/tensor_from_weighted_set_feature.cpp index 8c290e1f2b0..88309120882 100644 --- a/searchlib/src/vespa/searchlib/features/tensor_from_weighted_set_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/tensor_from_weighted_set_feature.cpp @@ -1,19 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "tensor_from_weighted_set_feature.h" - #include "constant_tensor_executor.h" #include "utils.h" #include "tensor_from_attribute_executor.h" #include "weighted_set_parser.hpp" - #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/feature_type.h> #include <vespa/searchcommon/attribute/attributecontent.h> #include <vespa/searchcommon/attribute/iattributevector.h> -#include <vespa/eval/eval/function.h> +#include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/value_type.h> -#include <vespa/eval/tensor/sparse/direct_sparse_tensor_builder.h> #include <vespa/log/log.h> LOG_SETUP(".features.tensor_from_weighted_set_feature"); @@ -22,8 +19,7 @@ using namespace search::fef; using search::attribute::IAttributeVector; using search::attribute::WeightedConstCharContent; using search::attribute::WeightedStringContent; -using vespalib::tensor::DirectSparseTensorBuilder; -using vespalib::tensor::SparseTensorAddressBuilder; +using vespalib::eval::FastValueBuilderFactory; using vespalib::eval::ValueType; using search::fef::FeatureType; @@ -106,15 +102,17 @@ createQueryExecutor(const search::fef::IQueryEnvironment &env, if (prop.found() && !prop.get().empty()) { WeightedStringVector vector; WeightedSetParser::parse(prop.get(), vector); - DirectSparseTensorBuilder<double> tensorBuilder(type); - tensorBuilder.reserve(vector._data.size()); - SparseTensorAddressBuilder address; + auto factory = FastValueBuilderFactory::get(); + size_t sz = vector._data.size(); + auto builder = factory.create_value_builder<double>(type, 1, 1, sz); + std::vector<vespalib::stringref> addr_ref; for (const auto &elem : vector._data) { - address.clear(); - address.add(elem.value()); - tensorBuilder.insertCell(address, elem.weight(), [](double, double v){ return v; }); + addr_ref.clear(); + addr_ref.push_back(elem.value()); + auto cell_array = builder->add_subspace(addr_ref); + cell_array[0] = elem.weight(); } - return ConstantTensorExecutor::create(tensorBuilder.build(), stash); + return ConstantTensorExecutor::create(builder->build(std::move(builder)), stash); } return ConstantTensorExecutor::createEmpty(type, stash); } diff --git a/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp b/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp index e9f48421fcf..5a60f2e617f 100644 --- a/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp @@ -13,7 +13,7 @@ namespace search::features { TermDistanceExecutor::TermDistanceExecutor(const IQueryEnvironment & env, - [[maybe_unused]] const TermDistanceParams & params) : + const TermDistanceParams & params) : FeatureExecutor(), _termA(env.getTerm(params.termX)), _termB(env.getTerm(params.termY)), diff --git a/searchlib/src/vespa/searchlib/grouping/sketch.h b/searchlib/src/vespa/searchlib/grouping/sketch.h index c105b480a3d..a872e82bedb 100644 --- a/searchlib/src/vespa/searchlib/grouping/sketch.h +++ b/searchlib/src/vespa/searchlib/grouping/sketch.h @@ -100,6 +100,9 @@ struct SparseSketch : Sketch<BucketBits, HashT> { } return true; } + bool operator==(const SparseSketch<BucketBits, HashT>& other) const { + return operator==(static_cast<const SketchType&>(other)); + } void print(std::ostream &out) const override { out << " (" << hash_set.size() << " elements)"; @@ -161,6 +164,9 @@ struct NormalSketch : Sketch<BucketBits, HashT> { } return true; } + bool operator==(const NormalSketch<BucketBits, HashT>& other) const { + return operator==(static_cast<const SketchType&>(other)); + } virtual void print(std::ostream &out) const override { for (size_t i = 0; i < BUCKET_COUNT; ++i) { diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp index 82302e4ab48..9af1ecee224 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp @@ -146,7 +146,7 @@ class QueryNodeConverter : public QueryVisitor { } void visit(Phrase &node) override { - createComplexIntermediate(node, node.getChildren(), (ParseItem::ITEM_PHRASE | ParseItem::IF_WEIGHT)); + createComplexIntermediate(node, node.getChildren(), (static_cast<uint8_t>(ParseItem::ITEM_PHRASE) | static_cast<uint8_t>(ParseItem::IF_WEIGHT))); } template <typename NODE> @@ -173,17 +173,17 @@ class QueryNodeConverter : public QueryVisitor { } void visit(WeightedSetTerm &node) override { - createWeightedSet(node, ParseItem::ITEM_WEIGHTED_SET | ParseItem::IF_WEIGHT); + createWeightedSet(node, static_cast<uint8_t>(ParseItem::ITEM_WEIGHTED_SET) | static_cast<uint8_t>(ParseItem::IF_WEIGHT)); visitNodes(node.getChildren()); } void visit(DotProduct &node) override { - createWeightedSet(node, ParseItem::ITEM_DOT_PRODUCT | ParseItem::IF_WEIGHT); + createWeightedSet(node, static_cast<uint8_t>(ParseItem::ITEM_DOT_PRODUCT) | static_cast<uint8_t>(ParseItem::IF_WEIGHT)); visitNodes(node.getChildren()); } void visit(WandTerm &node) override { - createWeightedSet(node, ParseItem::ITEM_WAND | ParseItem::IF_WEIGHT); + createWeightedSet(node, static_cast<uint8_t>(ParseItem::ITEM_WAND) | static_cast<uint8_t>(ParseItem::IF_WEIGHT)); appendCompressedPositiveNumber(node.getTargetNumHits()); appendDouble(node.getScoreThreshold()); appendDouble(node.getThresholdBoostFactor()); diff --git a/searchlib/src/vespa/searchlib/queryeval/fake_requestcontext.h b/searchlib/src/vespa/searchlib/queryeval/fake_requestcontext.h index c3f9685e47c..72520830d06 100644 --- a/searchlib/src/vespa/searchlib/queryeval/fake_requestcontext.h +++ b/searchlib/src/vespa/searchlib/queryeval/fake_requestcontext.h @@ -4,8 +4,8 @@ #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value.h> -#include <vespa/eval/eval/engine_or_factory.h> -#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/value_codec.h> #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/attribute/attribute_blueprint_params.h> @@ -35,7 +35,7 @@ public: } vespalib::eval::Value::UP get_query_tensor(const vespalib::string& tensor_name) const override { if (_query_tensor && (tensor_name == _query_tensor_name)) { - return vespalib::eval::EngineOrFactory::get().from_spec(*_query_tensor); + return vespalib::eval::value_from_spec(*_query_tensor, vespalib::eval::FastValueBuilderFactory::get()); } return {}; } diff --git a/searchlib/src/vespa/searchlib/queryeval/iterators.cpp b/searchlib/src/vespa/searchlib/queryeval/iterators.cpp index 32f6122558e..f9db8aea4f4 100644 --- a/searchlib/src/vespa/searchlib/queryeval/iterators.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/iterators.cpp @@ -11,4 +11,6 @@ RankedSearchIteratorBase(const fef::TermFieldMatchDataArray &matchData) _needUnpack(1) { } +RankedSearchIteratorBase::~RankedSearchIteratorBase() = default; + } diff --git a/searchlib/src/vespa/searchlib/queryeval/iterators.h b/searchlib/src/vespa/searchlib/queryeval/iterators.h index 0e44a6f6ff6..7adfe4aeba6 100644 --- a/searchlib/src/vespa/searchlib/queryeval/iterators.h +++ b/searchlib/src/vespa/searchlib/queryeval/iterators.h @@ -22,6 +22,7 @@ protected: void incNeedUnpack() { ++_needUnpack; } public: RankedSearchIteratorBase(const fef::TermFieldMatchDataArray &matchData); + ~RankedSearchIteratorBase() override; bool getUnpacked() const { return _needUnpack == 0; } }; diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp index d75cda2e513..d3ecffd1605 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp @@ -4,16 +4,16 @@ #include "nearest_neighbor_blueprint.h" #include "nearest_neighbor_iterator.h" #include "nns_index_iterator.h" +#include <vespa/eval/eval/dense_cells_value.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> -#include <vespa/eval/tensor/dense/dense_tensor.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/tensor/distance_function_factory.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.queryeval.nearest_neighbor_blueprint"); +using vespalib::eval::DenseCellsValue; using vespalib::eval::Value; -using vespalib::tensor::DenseTensor; namespace search::queryeval { @@ -21,7 +21,7 @@ namespace { template<typename LCT, typename RCT> void -convert_cells(std::unique_ptr<Value> &original, vespalib::eval::ValueType want_type) +convert_cells(std::unique_ptr<Value> &original, const vespalib::eval::ValueType &want_type) { auto old_cells = original->cells().typify<LCT>(); std::vector<RCT> new_cells; @@ -30,16 +30,16 @@ convert_cells(std::unique_ptr<Value> &original, vespalib::eval::ValueType want_t RCT conv = value; new_cells.push_back(conv); } - original = std::make_unique<DenseTensor<RCT>>(want_type, std::move(new_cells)); + original = std::make_unique<DenseCellsValue<RCT>>(want_type, std::move(new_cells)); } template<> void -convert_cells<float,float>(std::unique_ptr<Value> &, vespalib::eval::ValueType) {} +convert_cells<float,float>(std::unique_ptr<Value> &, const vespalib::eval::ValueType &) {} template<> void -convert_cells<double,double>(std::unique_ptr<Value> &, vespalib::eval::ValueType) {} +convert_cells<double,double>(std::unique_ptr<Value> &, const vespalib::eval::ValueType &) {} struct ConvertCellsSelector { diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp index 85b7e8f89e8..52814bb2631 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp @@ -5,7 +5,6 @@ using search::tensor::DenseTensorAttribute; using vespalib::ConstArrayRef; -using vespalib::tensor::MutableDenseTensorView; using vespalib::eval::TypedCells; using vespalib::eval::CellType; @@ -36,10 +35,10 @@ public: NearestNeighborImpl(Params params_in) : NearestNeighborIterator(params_in), _lhs(params().queryTensor.cells()), - _fieldTensor(params().tensorAttribute.getTensorType()), _lastScore(0.0) { - assert(is_compatible(_fieldTensor.fast_type(), params().queryTensor.type())); + assert(is_compatible(params().tensorAttribute.getTensorType(), + params().queryTensor.type())); } ~NearestNeighborImpl(); @@ -74,13 +73,11 @@ public: private: double computeDistance(uint32_t docId, double limit) { - params().tensorAttribute.extract_dense_view(docId, _fieldTensor); - auto rhs = _fieldTensor.cells(); + auto rhs = params().tensorAttribute.extract_cells_ref(docId); return params().distanceFunction->calc_with_limit(_lhs, rhs, limit); } TypedCells _lhs; - MutableDenseTensorView _fieldTensor; double _lastScore; }; diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h index 9b8ad6295b3..eec45ea1af0 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h @@ -5,7 +5,6 @@ #include "searchiterator.h" #include "nearest_neighbor_distance_heap.h" #include <vespa/eval/eval/value.h> -#include <vespa/eval/tensor/dense/mutable_dense_tensor_view.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/tensor/distance_function.h> diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt index fac6d015a5f..79b18a57a34 100644 --- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt @@ -25,5 +25,8 @@ vespa_add_library(searchlib_tensor OBJECT tensor_attribute.cpp tensor_deserialize.cpp tensor_store.cpp + serialized_fast_value_attribute.cpp + streamed_value_saver.cpp + streamed_value_store.cpp DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index 8b27dcc1cd4..be127a8be3e 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -5,8 +5,7 @@ #include "nearest_neighbor_index.h" #include "nearest_neighbor_index_saver.h" #include "tensor_attribute.hpp" -#include <vespa/eval/tensor/dense/dense_tensor_view.h> -#include <vespa/eval/tensor/dense/mutable_dense_tensor_view.h> +#include <vespa/eval/eval/value.h> #include <vespa/fastlib/io/bufferedfile.h> #include <vespa/searchlib/attribute/load_utils.h> #include <vespa/searchlib/attribute/readerbase.h> @@ -19,7 +18,6 @@ using search::attribute::LoadUtils; using vespalib::eval::Value; using vespalib::eval::ValueType; using vespalib::slime::ObjectInserter; -using vespalib::tensor::MutableDenseTensorView; namespace search::tensor { @@ -173,14 +171,14 @@ DenseTensorAttribute::getTensor(DocId docId) const return _denseTensorStore.getTensor(ref); } -void -DenseTensorAttribute::extract_dense_view(DocId docId, MutableDenseTensorView &tensor) const +vespalib::eval::TypedCells +DenseTensorAttribute::extract_cells_ref(DocId docId) const { EntryRef ref; if (docId < getCommittedDocIdLimit()) { ref = _refVector[docId]; } - _denseTensorStore.getTensor(ref, tensor); + return _denseTensorStore.get_typed_cells(ref); } bool diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h index e06bbf331ac..55e7b8cb464 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h @@ -8,8 +8,6 @@ #include "tensor_attribute.h" #include <memory> -namespace vespalib::tensor { class MutableDenseTensorView; } - namespace search::tensor { class NearestNeighborIndex; @@ -37,8 +35,8 @@ public: std::unique_ptr<PrepareResult> prepare_set_tensor(DocId docid, const vespalib::eval::Value& tensor) const override; void complete_set_tensor(DocId docid, const vespalib::eval::Value& tensor, std::unique_ptr<PrepareResult> prepare_result) override; std::unique_ptr<vespalib::eval::Value> getTensor(DocId docId) const override; - void extract_dense_view(DocId docId, vespalib::tensor::MutableDenseTensorView &tensor) const override; - bool supports_extract_dense_view() const override { return true; } + vespalib::eval::TypedCells extract_cells_ref(DocId docId) const override; + bool supports_extract_cells_ref() const override { return true; } bool onLoad() override; std::unique_ptr<AttributeSaver> onInitSave(vespalib::stringref fileName) override; void compactWorst() override; diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp index ddbb956838b..bc8362c2643 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp @@ -1,12 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "dense_tensor_store.h" -#include <vespa/eval/tensor/dense/dense_tensor_view.h> -#include <vespa/eval/tensor/dense/mutable_dense_tensor_view.h> +#include <vespa/eval/eval/value.h> #include <vespa/vespalib/datastore/datastore.hpp> using vespalib::datastore::Handle; -using vespalib::tensor::MutableDenseTensorView; using vespalib::eval::Value; using vespalib::eval::ValueType; using CellType = vespalib::eval::CellType; @@ -134,19 +132,7 @@ DenseTensorStore::getTensor(EntryRef ref) const return {}; } vespalib::eval::TypedCells cells_ref(getRawBuffer(ref), _type.cell_type(), getNumCells()); - return std::make_unique<vespalib::tensor::DenseTensorView>(_type, cells_ref); -} - -void -DenseTensorStore::getTensor(EntryRef ref, MutableDenseTensorView &tensor) const -{ - if (!ref.valid()) { - vespalib::eval::TypedCells cells_ref(&_emptySpace[0], _type.cell_type(), getNumCells()); - tensor.setCells(cells_ref); - } else { - vespalib::eval::TypedCells cells_ref(getRawBuffer(ref), _type.cell_type(), getNumCells()); - tensor.setCells(cells_ref); - } + return std::make_unique<vespalib::eval::DenseValueView>(_type, cells_ref); } vespalib::eval::TypedCells diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h index 696a325f813..49e8a585fec 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h @@ -6,7 +6,6 @@ #include <vespa/eval/eval/value_type.h> #include <vespa/eval/eval/typed_cells.h> -namespace vespalib { namespace tensor { class MutableDenseTensorView; }} namespace vespalib::eval { struct Value; } namespace search::tensor { @@ -66,7 +65,6 @@ public: void holdTensor(EntryRef ref) override; EntryRef move(EntryRef ref) override; std::unique_ptr<vespalib::eval::Value> getTensor(EntryRef ref) const; - void getTensor(EntryRef ref, vespalib::tensor::MutableDenseTensorView &tensor) const; vespalib::eval::TypedCells get_typed_cells(EntryRef ref) const; EntryRef setTensor(const vespalib::eval::Value &tensor); // The following method is meant to be used only for unit tests. diff --git a/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp index 62beb25be22..8cda62682d0 100644 --- a/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp @@ -3,7 +3,7 @@ #include "direct_tensor_attribute.h" #include "direct_tensor_saver.h" -#include <vespa/eval/eval/engine_or_factory.h> +#include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/value.h> #include <vespa/fastlib/io/bufferedfile.h> #include <vespa/searchlib/attribute/readerbase.h> @@ -14,7 +14,7 @@ #include "tensor_deserialize.h" #include "tensor_attribute.hpp" -using vespalib::eval::EngineOrFactory; +using vespalib::eval::FastValueBuilderFactory; namespace search::tensor { @@ -73,7 +73,7 @@ DirectTensorAttribute::set_tensor(DocId lid, std::unique_ptr<vespalib::eval::Val void DirectTensorAttribute::setTensor(DocId lid, const vespalib::eval::Value &tensor) { - set_tensor(lid, EngineOrFactory::get().copy(tensor)); + set_tensor(lid, FastValueBuilderFactory::get().copy(tensor)); } std::unique_ptr<vespalib::eval::Value> @@ -86,7 +86,7 @@ DirectTensorAttribute::getTensor(DocId docId) const if (ref.valid()) { auto ptr = _direct_store.get_tensor(ref); if (ptr) { - return EngineOrFactory::get().copy(*ptr); + return FastValueBuilderFactory::get().copy(*ptr); } } std::unique_ptr<vespalib::eval::Value> empty; diff --git a/searchlib/src/vespa/searchlib/tensor/direct_tensor_saver.cpp b/searchlib/src/vespa/searchlib/tensor/direct_tensor_saver.cpp index e67d5d8202b..b4b41187c8f 100644 --- a/searchlib/src/vespa/searchlib/tensor/direct_tensor_saver.cpp +++ b/searchlib/src/vespa/searchlib/tensor/direct_tensor_saver.cpp @@ -3,8 +3,7 @@ #include "direct_tensor_saver.h" #include "direct_tensor_store.h" -#include <vespa/eval/eval/engine_or_factory.h> -#include <vespa/eval/tensor/serialization/typed_binary_format.h> +#include <vespa/eval/eval/value_codec.h> #include <vespa/searchlib/attribute/iattributesavetarget.h> #include <vespa/searchlib/util/bufferwriter.h> #include <vespa/vespalib/objects/nbostream.h> @@ -39,7 +38,7 @@ DirectTensorAttributeSaver::onSave(IAttributeSaveTarget &saveTarget) const vespalib::eval::Value *tensor = _tensorStore.get_tensor(_refs[lid]); if (tensor) { stream.clear(); - vespalib::eval::EngineOrFactory::get().encode(*tensor, stream); + encode_value(*tensor, stream); uint32_t sz = stream.size(); datWriter->write(&sz, sizeof(sz)); datWriter->write(stream.peek(), stream.size()); diff --git a/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h index c962e919d95..360250c869e 100644 --- a/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h @@ -3,10 +3,8 @@ #pragma once #include <memory> +#include <vespa/eval/eval/typed_cells.h> -namespace vespalib::tensor { -class MutableDenseTensorView; -} namespace vespalib::eval { class ValueType; struct Value; } namespace vespalib::slime { struct Inserter; } @@ -21,9 +19,9 @@ public: virtual ~ITensorAttribute() {} virtual std::unique_ptr<vespalib::eval::Value> getTensor(uint32_t docId) const = 0; virtual std::unique_ptr<vespalib::eval::Value> getEmptyTensor() const = 0; - virtual void extract_dense_view(uint32_t docid, vespalib::tensor::MutableDenseTensorView& tensor) const = 0; + virtual vespalib::eval::TypedCells extract_cells_ref(uint32_t docid) const = 0; virtual const vespalib::eval::Value& get_tensor_ref(uint32_t docid) const = 0; - virtual bool supports_extract_dense_view() const = 0; + virtual bool supports_extract_cells_ref() const = 0; virtual bool supports_get_tensor_ref() const = 0; virtual const vespalib::eval::ValueType & getTensorType() const = 0; diff --git a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp index 6a0dbfb9f48..79c8e5a663e 100644 --- a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp +++ b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp @@ -48,10 +48,10 @@ ImportedTensorAttributeVectorReadGuard::getEmptyTensor() const return _target_tensor_attribute.getEmptyTensor(); } -void -ImportedTensorAttributeVectorReadGuard::extract_dense_view(uint32_t docid, vespalib::tensor::MutableDenseTensorView& tensor) const +vespalib::eval::TypedCells +ImportedTensorAttributeVectorReadGuard::extract_cells_ref(uint32_t docid) const { - _target_tensor_attribute.extract_dense_view(getTargetLid(docid), tensor); + return _target_tensor_attribute.extract_cells_ref(getTargetLid(docid)); } const vespalib::eval::Value& diff --git a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h index a3ffc27b153..c55a922487f 100644 --- a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h +++ b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h @@ -32,9 +32,9 @@ public: std::unique_ptr<vespalib::eval::Value> getTensor(uint32_t docId) const override; std::unique_ptr<vespalib::eval::Value> getEmptyTensor() const override; - void extract_dense_view(uint32_t docid, vespalib::tensor::MutableDenseTensorView& tensor) const override; + vespalib::eval::TypedCells extract_cells_ref(uint32_t docid) const override; const vespalib::eval::Value& get_tensor_ref(uint32_t docid) const override; - bool supports_extract_dense_view() const override { return _target_tensor_attribute.supports_extract_dense_view(); } + bool supports_extract_cells_ref() const override { return _target_tensor_attribute.supports_extract_cells_ref(); } bool supports_get_tensor_ref() const override { return _target_tensor_attribute.supports_get_tensor_ref(); } const vespalib::eval::ValueType &getTensorType() const override; void get_state(const vespalib::slime::Inserter& inserter) const override; diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp new file mode 100644 index 00000000000..6e1fb1a0a2f --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp @@ -0,0 +1,234 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "serialized_fast_value_attribute.h" +#include "streamed_value_saver.h" +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/fast_value.hpp> +#include <vespa/eval/streamed/streamed_value_utils.h> +#include <vespa/fastlib/io/bufferedfile.h> +#include <vespa/searchlib/attribute/readerbase.h> +#include <vespa/searchlib/util/fileutil.h> +#include <vespa/vespalib/util/rcuvector.hpp> +#include <vespa/log/log.h> + +LOG_SETUP(".searchlib.tensor.serialized_fast_value_attribute"); + +#include "blob_sequence_reader.h" +#include "tensor_attribute.hpp" + +using namespace vespalib; +using namespace vespalib::eval; + +namespace search::tensor { + +namespace { + +struct ValueBlock : LabelBlock { + TypedCells cells; +}; + +class ValueBlockStream { +private: + const StreamedValueStore::DataFromType &_from_type; + LabelBlockStream _label_block_stream; + const char *_cells_ptr; + + size_t dsss() const { return _from_type.dense_subspace_size; } + auto cell_type() const { return _from_type.cell_type; } +public: + ValueBlock next_block() { + auto labels = _label_block_stream.next_block(); + if (labels) { + TypedCells subspace_cells(_cells_ptr, cell_type(), dsss()); + _cells_ptr += CellTypeUtils::mem_size(cell_type(), dsss()); + return ValueBlock{labels, subspace_cells}; + } else { + TypedCells none(nullptr, cell_type(), 0); + return ValueBlock{labels, none}; + } + } + + ValueBlockStream(const StreamedValueStore::DataFromType &from_type, + const StreamedValueStore::StreamedValueData &from_store) + : _from_type(from_type), + _label_block_stream(from_store.num_subspaces, + from_store.labels_buffer, + from_type.num_mapped_dimensions), + _cells_ptr((const char *)from_store.cells_ref.data) + { + _label_block_stream.reset(); + } + + ~ValueBlockStream(); +}; + +ValueBlockStream::~ValueBlockStream() = default; + +void report_problematic_subspace(size_t idx, + const StreamedValueStore::DataFromType &from_type, + const StreamedValueStore::StreamedValueData &from_store) +{ + LOG(error, "PROBLEM: add_mapping returned same index=%zu twice", idx); + FastValueIndex temp_index(from_type.num_mapped_dimensions, + from_store.num_subspaces); + auto from_start = ValueBlockStream(from_type, from_store); + while (auto redo_block = from_start.next_block()) { + if (idx == temp_index.map.add_mapping(redo_block.address)) { + vespalib::string msg = "Block with address[ "; + for (vespalib::stringref ref : redo_block.address) { + msg.append("'").append(ref).append("' "); + } + msg.append("]"); + LOG(error, "%s maps to subspace %zu", msg.c_str(), idx); + } + } +} + +/** + * This Value implementation is almost exactly like FastValue, but + * instead of owning its type and cells it just has a reference to + * data stored elsewhere. + * XXX: we should find a better name for this, and move it + * (together with the helper classes above) to its own file, + * and add associated unit tests. + **/ +class OnlyFastValueIndex : public Value { +private: + const ValueType &_type; + TypedCells _cells; + FastValueIndex my_index; +public: + OnlyFastValueIndex(const ValueType &type, + const StreamedValueStore::DataFromType &from_type, + const StreamedValueStore::StreamedValueData &from_store) + : _type(type), + _cells(from_store.cells_ref), + my_index(from_type.num_mapped_dimensions, + from_store.num_subspaces) + { + assert(_type.cell_type() == _cells.type); + std::vector<vespalib::stringref> address(from_type.num_mapped_dimensions); + auto block_stream = ValueBlockStream(from_type, from_store); + size_t ss = 0; + while (auto block = block_stream.next_block()) { + size_t idx = my_index.map.add_mapping(block.address); + if (idx != ss) { + report_problematic_subspace(idx, from_type, from_store); + } + ++ss; + } + assert(ss == from_store.num_subspaces); + } + + + ~OnlyFastValueIndex(); + + const ValueType &type() const final override { return _type; } + TypedCells cells() const final override { return _cells; } + const Index &index() const final override { return my_index; } + vespalib::MemoryUsage get_memory_usage() const final override { + auto usage = self_memory_usage<OnlyFastValueIndex>(); + usage.merge(my_index.map.estimate_extra_memory_usage()); + return usage; + } +}; + +OnlyFastValueIndex::~OnlyFastValueIndex() = default; + +} + +SerializedFastValueAttribute::SerializedFastValueAttribute(stringref name, const Config &cfg) + : TensorAttribute(name, cfg, _streamedValueStore), + _tensor_type(cfg.tensorType()), + _streamedValueStore(_tensor_type), + _data_from_type(_tensor_type) +{ +} + + +SerializedFastValueAttribute::~SerializedFastValueAttribute() +{ + getGenerationHolder().clearHoldLists(); + _tensorStore.clearHoldLists(); +} + +void +SerializedFastValueAttribute::setTensor(DocId docId, const vespalib::eval::Value &tensor) +{ + checkTensorType(tensor); + EntryRef ref = _streamedValueStore.store_tensor(tensor); + assert(ref.valid()); + setTensorRef(docId, ref); +} + +std::unique_ptr<Value> +SerializedFastValueAttribute::getTensor(DocId docId) const +{ + EntryRef ref; + if (docId < getCommittedDocIdLimit()) { + ref = _refVector[docId]; + } + if (!ref.valid()) { + return {}; + } + if (auto data_from_store = _streamedValueStore.get_tensor_data(ref)) { + return std::make_unique<OnlyFastValueIndex>(_tensor_type, + _data_from_type, + data_from_store); + } + return {}; +} + +bool +SerializedFastValueAttribute::onLoad() +{ + BlobSequenceReader tensorReader(*this); + if (!tensorReader.hasData()) { + return false; + } + setCreateSerialNum(tensorReader.getCreateSerialNum()); + assert(tensorReader.getVersion() == getVersion()); + uint32_t numDocs(tensorReader.getDocIdLimit()); + _refVector.reset(); + _refVector.unsafe_reserve(numDocs); + vespalib::Array<char> buffer(1024); + for (uint32_t lid = 0; lid < numDocs; ++lid) { + uint32_t tensorSize = tensorReader.getNextSize(); + if (tensorSize != 0) { + if (tensorSize > buffer.size()) { + buffer.resize(tensorSize + 1024); + } + tensorReader.readBlob(&buffer[0], tensorSize); + vespalib::nbostream source(&buffer[0], tensorSize); + EntryRef ref = _streamedValueStore.store_encoded_tensor(source); + _refVector.push_back(ref); + } else { + EntryRef invalid; + _refVector.push_back(invalid); + } + } + setNumDocs(numDocs); + setCommittedDocIdLimit(numDocs); + return true; +} + + +std::unique_ptr<AttributeSaver> +SerializedFastValueAttribute::onInitSave(vespalib::stringref fileName) +{ + vespalib::GenerationHandler::Guard guard(getGenerationHandler(). + takeGuard()); + return std::make_unique<StreamedValueSaver> + (std::move(guard), + this->createAttributeHeader(fileName), + getRefCopy(), + _streamedValueStore); +} + +void +SerializedFastValueAttribute::compactWorst() +{ + doCompactWorst<StreamedValueStore::RefType>(); +} + +} diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.h b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.h new file mode 100644 index 00000000000..a8c1df4913a --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.h @@ -0,0 +1,33 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "tensor_attribute.h" +#include "streamed_value_store.h" + +namespace search::tensor { + +/** + * Attribute vector class storing serialized tensors for all documents in memory. + * + * When fetching a tensor with getTensor(docId) the returned Value + * will have a FastValueIndex (constructed on the fly) for its sparse + * mapping, but refer to a common type, while cells() will refer to + * memory in the serialized store without copying. + * + */ +class SerializedFastValueAttribute : public TensorAttribute { + vespalib::eval::ValueType _tensor_type; + StreamedValueStore _streamedValueStore; // data store for serialized tensors + const StreamedValueStore::DataFromType _data_from_type; +public: + SerializedFastValueAttribute(vespalib::stringref baseFileName, const Config &cfg); + virtual ~SerializedFastValueAttribute(); + virtual void setTensor(DocId docId, const vespalib::eval::Value &tensor) override; + virtual std::unique_ptr<vespalib::eval::Value> getTensor(DocId docId) const override; + virtual bool onLoad() override; + virtual std::unique_ptr<AttributeSaver> onInitSave(vespalib::stringref fileName) override; + virtual void compactWorst() override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_store.cpp index 2b3514fa352..7045c82935c 100644 --- a/searchlib/src/vespa/searchlib/tensor/serialized_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_store.cpp @@ -2,8 +2,8 @@ #include "serialized_tensor_store.h" #include "tensor_deserialize.h" -#include <vespa/eval/eval/engine_or_factory.h> #include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/value_codec.h> #include <vespa/vespalib/datastore/datastore.hpp> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/stringfmt.h> @@ -99,7 +99,7 @@ TensorStore::EntryRef SerializedTensorStore::setTensor(const vespalib::eval::Value &tensor) { vespalib::nbostream stream; - vespalib::eval::EngineOrFactory::get().encode(tensor, stream); + encode_value(tensor, stream); auto raw = allocRawBuffer(stream.size()); memcpy(raw.data, stream.peek(), stream.size()); return raw.ref; diff --git a/searchlib/src/vespa/searchlib/tensor/streamed_value_saver.cpp b/searchlib/src/vespa/searchlib/tensor/streamed_value_saver.cpp new file mode 100644 index 00000000000..d4fd681f2cb --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/streamed_value_saver.cpp @@ -0,0 +1,48 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "streamed_value_saver.h" +#include "streamed_value_store.h" + +#include <vespa/searchlib/attribute/iattributesavetarget.h> +#include <vespa/searchlib/util/bufferwriter.h> +#include <vespa/vespalib/objects/nbostream.h> + +using vespalib::GenerationHandler; + +namespace search::tensor { + +StreamedValueSaver:: +StreamedValueSaver(GenerationHandler::Guard &&guard, + const attribute::AttributeHeader &header, + RefCopyVector &&refs, + const StreamedValueStore &tensorStore) + : AttributeSaver(std::move(guard), header), + _refs(std::move(refs)), + _tensorStore(tensorStore) +{ +} + +StreamedValueSaver::~StreamedValueSaver() = default; + +bool +StreamedValueSaver::onSave(IAttributeSaveTarget &saveTarget) +{ + auto datWriter = saveTarget.datWriter().allocBufferWriter(); + const uint32_t docIdLimit(_refs.size()); + vespalib::nbostream stream; + for (uint32_t lid = 0; lid < docIdLimit; ++lid) { + if (_tensorStore.encode_tensor(_refs[lid], stream)) { + uint32_t sz = stream.size(); + datWriter->write(&sz, sizeof(sz)); + datWriter->write(stream.peek(), stream.size()); + stream.clear(); + } else { + uint32_t sz = 0; + datWriter->write(&sz, sizeof(sz)); + } + } + datWriter->flush(); + return true; +} + +} // namespace search::tensor diff --git a/searchlib/src/vespa/searchlib/tensor/streamed_value_saver.h b/searchlib/src/vespa/searchlib/tensor/streamed_value_saver.h new file mode 100644 index 00000000000..71d56539679 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/streamed_value_saver.h @@ -0,0 +1,35 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/attribute/attributesaver.h> +#include "tensor_attribute.h" + +namespace search::tensor { + +class StreamedValueStore; + +/* + * Class for saving a tensor attribute. + */ +class StreamedValueSaver : public AttributeSaver +{ +public: + using RefCopyVector = TensorAttribute::RefCopyVector; +private: + using GenerationHandler = vespalib::GenerationHandler; + + RefCopyVector _refs; + const StreamedValueStore &_tensorStore; + + bool onSave(IAttributeSaveTarget &saveTarget) override; +public: + StreamedValueSaver(GenerationHandler::Guard &&guard, + const attribute::AttributeHeader &header, + RefCopyVector &&refs, + const StreamedValueStore &tensorStore); + + virtual ~StreamedValueSaver(); +}; + +} // namespace search::tensor diff --git a/searchlib/src/vespa/searchlib/tensor/streamed_value_store.cpp b/searchlib/src/vespa/searchlib/tensor/streamed_value_store.cpp new file mode 100644 index 00000000000..c4579880409 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/streamed_value_store.cpp @@ -0,0 +1,220 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "streamed_value_store.h" +#include "tensor_deserialize.h" +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/eval/streamed/streamed_value_builder_factory.h> +#include <vespa/eval/streamed/streamed_value_view.h> +#include <vespa/vespalib/datastore/datastore.hpp> +#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/log/log.h> + +LOG_SETUP(".searchlib.tensor.streamed_value_store"); + +using vespalib::datastore::Handle; +using namespace vespalib::eval; + +namespace search::tensor { + +namespace { + +constexpr size_t MIN_BUFFER_ARRAYS = 1024; + +struct CellsMemBlock { + uint32_t num; + uint32_t total_sz; + const char *ptr; + CellsMemBlock(TypedCells cells) + : num(cells.size), + total_sz(CellTypeUtils::mem_size(cells.type, num)), + ptr((const char *)cells.data) + {} +}; + +template<typename T> +void check_alignment(T *ptr, size_t align) +{ + static_assert(sizeof(T) == 1); + size_t ptr_val = (size_t)ptr; + size_t unalign = ptr_val & (align - 1); + assert(unalign == 0); +} + +} // namespace <unnamed> + +StreamedValueStore::StreamedValueStore(const ValueType &tensor_type) + : TensorStore(_concreteStore), + _concreteStore(), + _bufferType(RefType::align(1), + MIN_BUFFER_ARRAYS, + RefType::offsetSize() / RefType::align(1)), + _tensor_type(tensor_type), + _data_from_type(_tensor_type) +{ + _store.addType(&_bufferType); + _store.initActiveBuffers(); + size_t align = CellTypeUtils::alignment(_data_from_type.cell_type); + // max alignment we can handle is 8: + assert(align <= 8); + // alignment must be a power of two: + assert((align & (align-1)) == 0); +} + +StreamedValueStore::~StreamedValueStore() +{ + _store.dropBuffers(); +} + +std::pair<const char *, uint32_t> +StreamedValueStore::getRawBuffer(RefType ref) const +{ + if (!ref.valid()) { + return std::make_pair(nullptr, 0u); + } + const char *buf = _store.getEntry<char>(ref); + uint32_t len = *reinterpret_cast<const uint32_t *>(buf); + return std::make_pair(buf + sizeof(uint32_t), len); +} + +Handle<char> +StreamedValueStore::allocRawBuffer(uint32_t size) +{ + if (size == 0) { + return Handle<char>(); + } + size_t extSize = size + sizeof(uint32_t); + size_t bufSize = RefType::align(extSize); + auto result = _concreteStore.rawAllocator<char>(_typeId).alloc(bufSize); + *reinterpret_cast<uint32_t *>(result.data) = size; + char *padWritePtr = result.data + extSize; + for (size_t i = extSize; i < bufSize; ++i) { + *padWritePtr++ = 0; + } + // Hide length of buffer (first 4 bytes) from users of the buffer. + return Handle<char>(result.ref, result.data + sizeof(uint32_t)); +} + +void +StreamedValueStore::holdTensor(EntryRef ref) +{ + if (!ref.valid()) { + return; + } + RefType iRef(ref); + const char *buf = _store.getEntry<char>(iRef); + uint32_t len = *reinterpret_cast<const uint32_t *>(buf); + _concreteStore.holdElem(ref, len + sizeof(uint32_t)); +} + +TensorStore::EntryRef +StreamedValueStore::move(EntryRef ref) +{ + if (!ref.valid()) { + return RefType(); + } + auto oldraw = getRawBuffer(ref); + auto newraw = allocRawBuffer(oldraw.second); + memcpy(newraw.data, oldraw.first, oldraw.second); + _concreteStore.holdElem(ref, oldraw.second + sizeof(uint32_t)); + return newraw.ref; +} + +StreamedValueStore::StreamedValueData +StreamedValueStore::get_tensor_data(EntryRef ref) const +{ + StreamedValueData retval; + retval.valid = false; + auto raw = getRawBuffer(ref); + if (raw.second == 0u) { + return retval; + } + vespalib::nbostream source(raw.first, raw.second); + uint32_t num_cells = source.readValue<uint32_t>(); + check_alignment(source.peek(), CellTypeUtils::alignment(_data_from_type.cell_type)); + retval.cells_ref = TypedCells(source.peek(), _data_from_type.cell_type, num_cells); + source.adjustReadPos(CellTypeUtils::mem_size(_data_from_type.cell_type, num_cells)); + assert((num_cells % _data_from_type.dense_subspace_size) == 0); + retval.num_subspaces = num_cells / _data_from_type.dense_subspace_size; + retval.labels_buffer = vespalib::ConstArrayRef<char>(source.peek(), source.size()); + retval.valid = true; + return retval; +} + +bool +StreamedValueStore::encode_tensor(EntryRef ref, vespalib::nbostream &target) const +{ + if (auto data = get_tensor_data(ref)) { + StreamedValueView value( + _tensor_type, _data_from_type.num_mapped_dimensions, + data.cells_ref, data.num_subspaces, data.labels_buffer); + vespalib::eval::encode_value(value, target); + return true; + } else { + return false; + } +} + +void +StreamedValueStore::serialize_labels(const Value::Index &index, + vespalib::nbostream &target) const +{ + uint32_t num_subspaces = index.size(); + uint32_t num_mapped_dims = _data_from_type.num_mapped_dimensions; + std::vector<vespalib::stringref> labels(num_mapped_dims * num_subspaces); + auto view = index.create_view({}); + view->lookup({}); + std::vector<vespalib::stringref> addr(num_mapped_dims); + std::vector<vespalib::stringref *> addr_refs; + for (auto & label : addr) { + addr_refs.push_back(&label); + } + size_t subspace; + for (size_t ss = 0; ss < num_subspaces; ++ss) { + bool ok = view->next_result(addr_refs, subspace); + assert(ok); + size_t idx = subspace * num_mapped_dims; + for (auto label : addr) { + labels[idx++] = label; + } + } + bool ok = view->next_result(addr_refs, subspace); + assert(!ok); + for (auto label : labels) { + target.writeSmallString(label); + } +} + +TensorStore::EntryRef +StreamedValueStore::store_tensor(const Value &tensor) +{ + assert(tensor.type() == _tensor_type); + CellsMemBlock cells_mem(tensor.cells()); + vespalib::nbostream stream; + stream << uint32_t(cells_mem.num); + serialize_labels(tensor.index(), stream); + size_t mem_size = stream.size() + cells_mem.total_sz; + auto raw = allocRawBuffer(mem_size); + char *target = raw.data; + memcpy(target, stream.peek(), sizeof(uint32_t)); + stream.adjustReadPos(sizeof(uint32_t)); + target += sizeof(uint32_t); + check_alignment(target, CellTypeUtils::alignment(_data_from_type.cell_type)); + memcpy(target, cells_mem.ptr, cells_mem.total_sz); + target += cells_mem.total_sz; + memcpy(target, stream.peek(), stream.size()); + target += stream.size(); + assert(target <= raw.data + mem_size); + return raw.ref; +} + +TensorStore::EntryRef +StreamedValueStore::store_encoded_tensor(vespalib::nbostream &encoded) +{ + const auto &factory = StreamedValueBuilderFactory::get(); + auto val = vespalib::eval::decode_value(encoded, factory); + return store_tensor(*val); +} + +} diff --git a/searchlib/src/vespa/searchlib/tensor/streamed_value_store.h b/searchlib/src/vespa/searchlib/tensor/streamed_value_store.h new file mode 100644 index 00000000000..de94dc043d3 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/streamed_value_store.h @@ -0,0 +1,96 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "tensor_store.h" +#include <vespa/eval/eval/value_type.h> +#include <vespa/eval/eval/value.h> +#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/typify.h> + +namespace search::tensor { + +/** + * Class for storing tensors in memory, with a special serialization + * format that can be used directly to make a StreamedValueView. + * + * The tensor type is owned by the store itself and will not be + * serialized at all. + * + * The parameters for serialization (see DataFromType) are: + * - number of mapped dimensions [MD] + * - dense subspace size [DS] + * - size of each cell [CS] - currently 4 (float) or 8 (double) + * - alignment for cells - currently 4 (float) or 8 (double) + * While the tensor value to be serialized has: + * - number of dense subspaces [ND] + * - labels for dense subspaces, ND * MD strings + * - cell values, ND * DS cells (each either float or double) + * The serialization format looks like: + * + * [bytes] : [format] : [description] + * 4 : n.b.o. uint32_ t : num cells = ND * DS + * CS * ND * DS : native float or double : cells + * (depends) : n.b.o. strings : ND * MD label strings + * + * Here, n.b.o. means network byte order, or more precisely + * it's the format vespalib::nbostream uses for the given data type, + * including strings (where exact format depends on the string length). + * Note that the only unpredictably-sized data (the labels) are kept + * last. + * If we ever make a "hbostream" which uses host byte order, we + * could switch to that instead since these data are only kept in + * memory. + */ +class StreamedValueStore : public TensorStore { +public: + using RefType = vespalib::datastore::AlignedEntryRefT<22, 3>; + using DataStoreType = vespalib::datastore::DataStoreT<RefType>; + + struct StreamedValueData { + bool valid; + vespalib::eval::TypedCells cells_ref; + size_t num_subspaces; + vespalib::ConstArrayRef<char> labels_buffer; + operator bool() const { return valid; } + }; + + struct DataFromType { + uint32_t num_mapped_dimensions; + uint32_t dense_subspace_size; + vespalib::eval::CellType cell_type; + + DataFromType(const vespalib::eval::ValueType& type) + : num_mapped_dimensions(type.count_mapped_dimensions()), + dense_subspace_size(type.dense_subspace_size()), + cell_type(type.cell_type()) + {} + }; + +private: + DataStoreType _concreteStore; + vespalib::datastore::BufferType<char> _bufferType; + vespalib::eval::ValueType _tensor_type; + DataFromType _data_from_type; + + void serialize_labels(const vespalib::eval::Value::Index &index, + vespalib::nbostream &target) const; + + std::pair<const char *, uint32_t> getRawBuffer(RefType ref) const; + vespalib::datastore::Handle<char> allocRawBuffer(uint32_t size); +public: + StreamedValueStore(const vespalib::eval::ValueType &tensor_type); + virtual ~StreamedValueStore(); + + virtual void holdTensor(EntryRef ref) override; + virtual EntryRef move(EntryRef ref) override; + + StreamedValueData get_tensor_data(EntryRef ref) const; + bool encode_tensor(EntryRef ref, vespalib::nbostream &target) const; + + EntryRef store_tensor(const vespalib::eval::Value &tensor); + EntryRef store_encoded_tensor(vespalib::nbostream &encoded); +}; + + +} diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp index 0748329694c..e0b21290284 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp @@ -7,13 +7,14 @@ #include <vespa/vespalib/data/slime/cursor.h> #include <vespa/vespalib/data/slime/inserter.h> #include <vespa/vespalib/util/rcuvector.hpp> -#include <vespa/eval/eval/engine_or_factory.h> +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/value_codec.h> #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value.h> using document::TensorDataType; using document::WrongTensorTypeException; -using vespalib::eval::EngineOrFactory; +using vespalib::eval::FastValueBuilderFactory; using vespalib::eval::TensorSpec; using vespalib::eval::Value; using vespalib::eval::ValueType; @@ -31,9 +32,9 @@ constexpr size_t DEAD_SLACK = 0x10000u; Value::UP createEmptyTensor(const ValueType &type) { - auto engine = EngineOrFactory::get(); + const auto &factory = FastValueBuilderFactory::get(); TensorSpec empty_spec(type.to_spec()); - return engine.from_spec(empty_spec); + return vespalib::eval::value_from_spec(empty_spec, factory); } vespalib::string makeWrongTensorTypeMsg(const ValueType &fieldTensorType, const ValueType &tensorType) @@ -183,23 +184,19 @@ TensorAttribute::populate_state(vespalib::slime::Cursor& object) const vespalib::eval::Value::UP TensorAttribute::getEmptyTensor() const { - return EngineOrFactory::get().copy(*_emptyTensor); + return FastValueBuilderFactory::get().copy(*_emptyTensor); } -void -TensorAttribute::extract_dense_view(uint32_t docid, vespalib::tensor::MutableDenseTensorView& tensor) const +vespalib::eval::TypedCells +TensorAttribute::extract_cells_ref(uint32_t /*docid*/) const { - (void) docid; - (void) tensor; notImplemented(); } const vespalib::eval::Value& -TensorAttribute::get_tensor_ref(uint32_t docid) const +TensorAttribute::get_tensor_ref(uint32_t /*docid*/) const { - (void) docid; notImplemented(); - abort(); // Needed to avoid compile error } const vespalib::eval::ValueType & diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h index b88ffcf0f2c..7abfe66a2e4 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h @@ -47,9 +47,9 @@ public: void onGenerationChange(generation_t generation) override; bool addDoc(DocId &docId) override; std::unique_ptr<vespalib::eval::Value> getEmptyTensor() const override; - void extract_dense_view(uint32_t docid, vespalib::tensor::MutableDenseTensorView& tensor) const override; + vespalib::eval::TypedCells extract_cells_ref(uint32_t docid) const override; const vespalib::eval::Value& get_tensor_ref(uint32_t docid) const override; - bool supports_extract_dense_view() const override { return false; } + bool supports_extract_cells_ref() const override { return false; } bool supports_get_tensor_ref() const override { return false; } const vespalib::eval::ValueType & getTensorType() const override; void get_state(const vespalib::slime::Inserter& inserter) const override; diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_deserialize.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_deserialize.cpp index 83988a3af11..4fddd092451 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_deserialize.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_deserialize.cpp @@ -1,24 +1,30 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "tensor_deserialize.h" #include <vespa/document/util/serializableexceptions.h> -#include <vespa/eval/eval/engine_or_factory.h> +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/value_codec.h> #include <vespa/eval/eval/value.h> -#include <vespa/vespalib/objects/nbostream.h> using document::DeserializeException; -using vespalib::eval::EngineOrFactory; +using vespalib::eval::FastValueBuilderFactory; using vespalib::eval::Value; namespace search::tensor { -std::unique_ptr<Value> deserialize_tensor(const void *data, size_t size) +std::unique_ptr<Value> deserialize_tensor(vespalib::nbostream &buffer) { - vespalib::nbostream wrapStream(data, size); - auto tensor = EngineOrFactory::get().decode(wrapStream); - if (wrapStream.size() != 0) { + auto tensor = vespalib::eval::decode_value(buffer, FastValueBuilderFactory::get()); + if (buffer.size() != 0) { throw DeserializeException("Leftover bytes deserializing tensor attribute value.", VESPA_STRLOC); } return tensor; } +std::unique_ptr<Value> deserialize_tensor(const void *data, size_t size) +{ + vespalib::nbostream wrapStream(data, size); + return deserialize_tensor(wrapStream); +} + } // namespace diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_deserialize.h b/searchlib/src/vespa/searchlib/tensor/tensor_deserialize.h index 18e166543d6..6f9521c1355 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_deserialize.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_deserialize.h @@ -1,10 +1,14 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/eval/eval/value.h> +#include <vespa/vespalib/objects/nbostream.h> namespace search::tensor { extern std::unique_ptr<vespalib::eval::Value> deserialize_tensor(const void *data, size_t size); +extern std::unique_ptr<vespalib::eval::Value> +deserialize_tensor(vespalib::nbostream &stream); + } // namespace diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp index 830384ee538..9f7f39b7d5c 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp @@ -236,11 +236,8 @@ DomainPart::buildPacketMapping(bool allowTruncate) _range.from(firstSerial); } _range.to(packet.range().to()); - _packets.insert(std::make_pair(firstSerial, std::move(packet))); - { - std::lock_guard guard(_lock); - _skipList.push_back(SkipInfo(firstSerial, firstPos)); - } + // Called only from constructor so no need to hold lock + _skipList.emplace_back(firstSerial, firstPos); } else { fSize = transLog.GetSize(); } @@ -259,7 +256,6 @@ DomainPart::DomainPart(const string & name, const string & baseDir, SerialNum s, _range(s), _sz(0), _byteSize(0), - _packets(), _fileName(fmt("%s/%s-%016" PRIu64, baseDir.c_str(), name.c_str(), s)), _transLog(std::make_unique<FastOS_File>(_fileName.c_str())), _skipList(), @@ -345,10 +341,6 @@ DomainPart::close() throw runtime_error(fmt("Failed closing file '%s' of size %" PRId64 ".", _transLog->GetFileName(), _transLog->GetSize())); } - { - std::lock_guard guard(_lock); - _packets.clear(); - } return retval; } @@ -364,11 +356,9 @@ DomainPart::openAndFind(FastOS_FileInterface &file, const SerialNum &from) if (retval) { int64_t pos(_headerLen); std::lock_guard guard(_lock); - for(SkipList::const_iterator it(_skipList.begin()), mt(_skipList.end()); - (it < mt) && (it->id() <= from); - it++) - { - pos = it->filePos(); + for (const auto & skipInfo : _skipList) { + if (skipInfo.id() > from) break; + pos = skipInfo.filePos(); } retval = file.SetPosition(pos); } @@ -419,20 +409,8 @@ DomainPart::commit(SerialNum firstSerial, const Packet &packet) if ( ! chunk->getEntries().empty()) { write(*_transLog, *chunk); } - - bool merged(false); std::lock_guard guard(_lock); - if ( ! _packets.empty() ) { - Packet & lastPacket = _packets.rbegin()->second; - if (lastPacket.sizeBytes() < 0xf000) { - lastPacket.merge(packet); - merged = true; - } - } - if (! merged ) { - _packets.insert(std::make_pair(firstSerial, std::move(packet))); - _skipList.push_back(SkipInfo(firstSerial, firstPos)); - } + _skipList.emplace_back(firstSerial, firstPos); } void @@ -452,76 +430,6 @@ DomainPart::sync() } bool -DomainPart::visit(SerialNumRange &r, Packet &packet) -{ - bool retval(false); - std::lock_guard guard(_lock); - LOG(spam, "Visit r(%" PRIu64 ", %" PRIu64 "] Checking %" PRIu64 " packets", - r.from(), r.to(), uint64_t(_packets.size())); - if ( ! isClosed() ) { - PacketList::const_iterator start(_packets.lower_bound(r.from() + 1)); - PacketList::const_iterator end(_packets.upper_bound(r.to())); - if (start != _packets.end()) { - if ( ! start->second.range().contains(r.from() + 1) && - (start != _packets.begin())) { - PacketList::const_iterator prev(start); - prev--; - if (prev->second.range().contains(r.from() + 1)) { - start--; - } - } - } else { - if (!_packets.empty()) - start--; - } - if ( start != _packets.end() && start->first <= r.to()) { - PacketList::const_iterator next(start); - next++; - if ((r.from() < start->first) && - ((next != end) || ((next != _packets.end()) && ((r.to() + 1) == next->first)))) - { - packet = start->second; - LOG(spam, "Visit whole packet[%" PRIu64 ", %" PRIu64 "]", packet.range().from(), packet.range().to()); - if (next != _packets.end()) { - r.from(next->first - 1); - retval = true; - } else { - /// This is the very last package. Can safely finish. - } - } else { - const nbostream & tmp = start->second.getHandle(); - nbostream_longlivedbuf h(tmp.data(), tmp.size()); - LOG(spam, "Visit partial[%" PRIu64 ", %" PRIu64 "] (%zd, %zd, %zd)", - start->second.range().from(), start->second.range().to(), h.rp(), h.size(), h.capacity()); - Packet newPacket(h.size()); - for (; (h.size() > 0) && (r.from() < r.to()); ) { - Packet::Entry e; - e.deserialize(h); - if (r.from() < e.serial()) { - if (e.serial() <= r.to()) { - LOG(spam, "Adding serial #%" PRIu64 ", of type %d and size %zd into packet of size %zu and %zu bytes", - e.serial(), e.type(), e.data().size(), newPacket.size(), newPacket.sizeBytes()); - newPacket.add(e); - r.from(e.serial()); - } else { - // Force breakout on visiting empty interval. - r.from(r.to()); - } - } - } - packet = std::move(newPacket); - retval = next != _packets.end(); - } - } - } else { - /// File has been closed must continue from file. - retval = true; - } - return retval; -} - - -bool DomainPart::visit(FastOS_FileInterface &file, SerialNumRange &r, Packet &packet) { if ( ! file.IsOpened() && ! openAndFind(file, r.from() + 1)) { diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.h b/searchlib/src/vespa/searchlib/transactionlog/domainpart.h index a956932be19..f5a10cafe3e 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.h @@ -27,7 +27,6 @@ public: const vespalib::string &fileName() const { return _fileName; } void commit(SerialNum firstSerial, const Packet &packet); bool erase(SerialNum to); - bool visit(SerialNumRange &r, Packet &packet); bool visit(FastOS_FileInterface &file, SerialNumRange &r, Packet &packet); bool close(); void sync(); @@ -56,22 +55,20 @@ private: class SkipInfo { public: - SkipInfo(SerialNum s, uint64_t p) : _id(s), _pos(p) {} + SkipInfo(SerialNum s, uint64_t p) noexcept : _id(s), _pos(p) {} - bool operator ==(const SkipInfo &b) const { return cmp(b) == 0; } - bool operator <(const SkipInfo &b) const { return cmp(b) < 0; } - bool operator >(const SkipInfo &b) const { return cmp(b) > 0; } - bool operator <=(const SkipInfo &b) const { return cmp(b) <= 0; } - bool operator >=(const SkipInfo &b) const { return cmp(b) >= 0; } - int64_t filePos() const { return _pos; } - SerialNum id() const { return _id; } + bool operator ==(const SkipInfo &b) const noexcept { return cmp(b) == 0; } + bool operator <(const SkipInfo &b) const noexcept { return cmp(b) < 0; } + bool operator >(const SkipInfo &b) const noexcept { return cmp(b) > 0; } + bool operator <=(const SkipInfo &b) const noexcept { return cmp(b) <= 0; } + bool operator >=(const SkipInfo &b) const noexcept { return cmp(b) >= 0; } + int64_t filePos() const noexcept { return _pos; } + SerialNum id() const noexcept { return _id; } private: - int64_t cmp(const SkipInfo & b) const { return _id - b._id; } + int64_t cmp(const SkipInfo & b) const noexcept { return _id - b._id; } SerialNum _id; uint64_t _pos; }; - typedef std::vector<SkipInfo> SkipList; - typedef std::map<SerialNum, Packet> PacketList; const Encoding _encoding; const uint8_t _compressionLevel; std::mutex _lock; @@ -79,10 +76,9 @@ private: SerialNumRange _range; size_t _sz; std::atomic<uint64_t> _byteSize; - PacketList _packets; vespalib::string _fileName; std::unique_ptr<FastOS_FileInterface> _transLog; - SkipList _skipList; + std::vector<SkipInfo> _skipList; uint32_t _headerLen; mutable std::mutex _writeLock; // Protected by _writeLock diff --git a/searchlib/src/vespa/searchlib/transactionlog/session.cpp b/searchlib/src/vespa/searchlib/transactionlog/session.cpp index edfd7367a1f..022a891b3fc 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/session.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/session.cpp @@ -33,12 +33,8 @@ Session::VisitTask::run() bool Session::visit(FastOS_FileInterface & file, DomainPart & dp) { Packet packet(size_t(-1)); - bool more(false); - if (dp.isClosed()) { - more = dp.visit(file, _range, packet); - } else { - more = dp.visit(_range, packet); - } + bool more = dp.visit(file, _range, packet); + if ( ! packet.getHandle().empty()) { send(packet); } diff --git a/searchlib/src/vespa/searchlib/util/rawbuf.cpp b/searchlib/src/vespa/searchlib/util/rawbuf.cpp index c4fb3dd72cc..0fcef353f33 100644 --- a/searchlib/src/vespa/searchlib/util/rawbuf.cpp +++ b/searchlib/src/vespa/searchlib/util/rawbuf.cpp @@ -220,7 +220,7 @@ RawBuf::operator+=(const RawBuf& buffer) bool -RawBuf::operator==(const RawBuf &buffer) +RawBuf::operator==(const RawBuf &buffer) const { size_t nbytes = buffer.GetUsedLen(); if (nbytes != GetUsedLen()) diff --git a/searchlib/src/vespa/searchlib/util/rawbuf.h b/searchlib/src/vespa/searchlib/util/rawbuf.h index ffebd035950..fd4ce8976a9 100644 --- a/searchlib/src/vespa/searchlib/util/rawbuf.h +++ b/searchlib/src/vespa/searchlib/util/rawbuf.h @@ -38,7 +38,7 @@ public: void operator+=(const char *src); void operator+=(const RawBuf& buffer); - bool operator==(const RawBuf &buffer); + bool operator==(const RawBuf &buffer) const; void addNum(size_t num, size_t fieldw, char fill); void addNum32(int32_t num, size_t fieldw, char fill); void addNum64(int64_t num, size_t fieldw, char fill); |