test tensor feature extraction for proton

also make sure we avoid auto-unboxing for both indexed and streaming
search

2 files changed, 41 insertions, 13 deletions

diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp
index b34a8c84237..967d8bfd0aa 100644
--- a/searchcore/src/tests/proton/matching/matching_test.cpp
+++ b/searchcore/src/tests/proton/matching/matching_test.cpp
@@ -33,6 +33,9 @@
 #include <vespa/searchcore/proton/matching/match_params.h>
 #include <vespa/searchcore/proton/matching/match_tools.h>
 #include <vespa/searchcore/proton/matching/match_context.h>
+#include <vespa/eval/eval/tensor_spec.h>
+#include <vespa/eval/tensor/default_tensor_engine.h>
+#include <vespa/vespalib/objects/nbostream.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP("matching_test");
@@ -55,6 +58,10 @@ using search::index::schema::DataType;
 using storage::spi::Timestamp;
 using search::fef::indexproperties::hitcollector::HeapSize;
 
+using vespalib::nbostream;
+using vespalib::eval::TensorSpec;
+using vespalib::tensor::DefaultTensorEngine;
+
 void inject_match_phase_limiting(Properties &setup, const vespalib::string &attribute, size_t max_hits, bool descending)
 {
     Properties cfg;
@@ -134,7 +141,10 @@ struct MyWorld {
         config.add(indexproperties::hitcollector::HeapSize::NAME, (vespalib::asciistream() << heapSize).str());
         config.add(indexproperties::hitcollector::ArraySize::NAME, (vespalib::asciistream() << arraySize).str());
         config.add(indexproperties::summary::Feature::NAME, "attribute(a1)");
+        config.add(indexproperties::summary::Feature::NAME, "rankingExpression(\"reduce(tensor(x[3])(x),sum)\")");
+        config.add(indexproperties::summary::Feature::NAME, "rankingExpression(\"tensor(x[3])(x)\")");
         config.add(indexproperties::summary::Feature::NAME, "value(100)");
+
         config.add(indexproperties::dump::IgnoreDefaultFeatures::NAME, "true");
         config.add(indexproperties::dump::Feature::NAME, "attribute(a2)");
 
@@ -631,20 +641,34 @@ TEST("require that summary features are filled") {
     DocsumRequest::SP req = world.createSimpleDocsumRequest("f1", "foo");
     FeatureSet::SP fs = world.getSummaryFeatures(req);
     const FeatureSet::Value * f = NULL;
-    EXPECT_EQUAL(2u, fs->numFeatures());
+    EXPECT_EQUAL(4u, fs->numFeatures());
     EXPECT_EQUAL("attribute(a1)", fs->getNames()[0]);
-    EXPECT_EQUAL("value(100)", fs->getNames()[1]);
+    EXPECT_EQUAL("rankingExpression(\"reduce(tensor(x[3])(x),sum)\")", fs->getNames()[1]);
+    EXPECT_EQUAL("rankingExpression(\"tensor(x[3])(x)\")", fs->getNames()[2]);
+    EXPECT_EQUAL("value(100)", fs->getNames()[3]);
     EXPECT_EQUAL(2u, fs->numDocs());
     f = fs->getFeaturesByDocId(10);
     EXPECT_TRUE(f != NULL);
     EXPECT_EQUAL(10, f[0].as_double());
-    EXPECT_EQUAL(100, f[1].as_double());
+    EXPECT_EQUAL(100, f[3].as_double());
     f = fs->getFeaturesByDocId(15);
     EXPECT_TRUE(f == NULL);
     f = fs->getFeaturesByDocId(30);
     EXPECT_TRUE(f != NULL);
     EXPECT_EQUAL(30, f[0].as_double());
-    EXPECT_EQUAL(100, f[1].as_double());
+    EXPECT_EQUAL(100, f[3].as_double());
+    EXPECT_TRUE(f[1].is_double());
+    EXPECT_TRUE(!f[1].is_data());
+    EXPECT_EQUAL(f[1].as_double(), 3.0); // 0 + 1 + 2
+    EXPECT_TRUE(!f[2].is_double());
+    EXPECT_TRUE(f[2].is_data());
+    {
+        auto &engine = DefaultTensorEngine::ref();
+        nbostream buf(f[2].as_data().data, f[2].as_data().size);
+        auto actual = engine.to_spec(*engine.decode(buf));
+        auto expect = TensorSpec("tensor(x[3])").add({{"x", 0}}, 0).add({{"x", 1}}, 1).add({{"x", 2}}, 2);
+        EXPECT_EQUAL(actual, expect);
+    }
 }
 
 TEST("require that rank features are filled") {
@@ -699,25 +723,29 @@ TEST("require that getSummaryFeatures can use cached query setup") {
     docsum_request->hits.back().docid = 30;
 
     FeatureSet::SP fs = world.getSummaryFeatures(docsum_request);
-    ASSERT_EQUAL(2u, fs->numFeatures());
+    ASSERT_EQUAL(4u, fs->numFeatures());
     EXPECT_EQUAL("attribute(a1)", fs->getNames()[0]);
-    EXPECT_EQUAL("value(100)", fs->getNames()[1]);
+    EXPECT_EQUAL("rankingExpression(\"reduce(tensor(x[3])(x),sum)\")", fs->getNames()[1]);
+    EXPECT_EQUAL("rankingExpression(\"tensor(x[3])(x)\")", fs->getNames()[2]);
+    EXPECT_EQUAL("value(100)", fs->getNames()[3]);
     ASSERT_EQUAL(1u, fs->numDocs());
     const auto *f = fs->getFeaturesByDocId(30);
     ASSERT_TRUE(f);
     EXPECT_EQUAL(30, f[0].as_double());
-    EXPECT_EQUAL(100, f[1].as_double());
+    EXPECT_EQUAL(100, f[3].as_double());
 
     // getSummaryFeatures can be called multiple times.
     fs = world.getSummaryFeatures(docsum_request);
-    ASSERT_EQUAL(2u, fs->numFeatures());
+    ASSERT_EQUAL(4u, fs->numFeatures());
     EXPECT_EQUAL("attribute(a1)", fs->getNames()[0]);
-    EXPECT_EQUAL("value(100)", fs->getNames()[1]);
+    EXPECT_EQUAL("rankingExpression(\"reduce(tensor(x[3])(x),sum)\")", fs->getNames()[1]);
+    EXPECT_EQUAL("rankingExpression(\"tensor(x[3])(x)\")", fs->getNames()[2]);
+    EXPECT_EQUAL("value(100)", fs->getNames()[3]);
     ASSERT_EQUAL(1u, fs->numDocs());
     f = fs->getFeaturesByDocId(30);
     ASSERT_TRUE(f);
     EXPECT_EQUAL(30, f[0].as_double());
-    EXPECT_EQUAL(100, f[1].as_double());
+    EXPECT_EQUAL(100, f[3].as_double());
 }
 
 TEST("require that getSummaryFeatures prefers cached query setup") {
@@ -733,7 +761,7 @@ TEST("require that getSummaryFeatures prefers cached query setup") {
     req->sessionId = request->sessionId;
     req->propertiesMap.lookupCreate(search::MapNames::CACHES).add("query", "true");
     FeatureSet::SP fs = world.getSummaryFeatures(req);
-    EXPECT_EQUAL(2u, fs->numFeatures());
+    EXPECT_EQUAL(4u, fs->numFeatures());
     ASSERT_EQUAL(0u, fs->numDocs());  // "spread" has no hits
 
     // Empty cache
@@ -742,7 +770,7 @@ TEST("require that getSummaryFeatures prefers cached query setup") {
     world.sessionManager->pruneTimedOutSessions(pruneTime);
 
     fs = world.getSummaryFeatures(req);
-    EXPECT_EQUAL(2u, fs->numFeatures());
+    EXPECT_EQUAL(4u, fs->numFeatures());
     ASSERT_EQUAL(2u, fs->numDocs());  // "foo" has two hits
 }
 
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp
index b86ccc22bcf..c2262cc51e5 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp
@@ -130,7 +130,7 @@ MatchMaster::getFeatureSet(const MatchToolsFactory &mtf,
     RankProgram &rankProgram = matchTools->rank_program();
 
     std::vector<vespalib::string> featureNames;
-    FeatureResolver resolver(rankProgram.get_seeds());
+    FeatureResolver resolver(rankProgram.get_seeds(false));
     featureNames.reserve(resolver.num_features());
     for (size_t i = 0; i < resolver.num_features(); ++i) {
         featureNames.emplace_back(resolver.name_of(i));