Merge pull request #9792 from vespa-engine/havardpe/tensors-in-summary-set

Havardpe/tensors in summary set

3 files changed, 78 insertions, 50 deletions

diff --git a/searchlib/src/tests/common/summaryfeatures/summaryfeatures.cpp b/searchlib/src/tests/common/summaryfeatures/summaryfeatures.cpp
index 0dc43898441..646b2b818b3 100644
--- a/searchlib/src/tests/common/summaryfeatures/summaryfeatures.cpp
+++ b/searchlib/src/tests/common/summaryfeatures/summaryfeatures.cpp
@@ -5,6 +5,7 @@ LOG_SETUP("summaryfeatures_test");
 #include <vespa/searchlib/common/featureset.h>
 
 using namespace search;
+using vespalib::Memory;
 
 TEST_SETUP(Test);
 
@@ -43,34 +44,34 @@ Test::Main()
         EXPECT_EQUAL(sf.addDocId(40), 3u);
         EXPECT_EQUAL(sf.addDocId(50), 4u);
         EXPECT_EQUAL(sf.numDocs(), 5u);
-        feature_t *f;
-        const feature_t *cf;
+        FeatureSet::Value *f;
+        const FeatureSet::Value *cf;
         f = sf.getFeaturesByIndex(0);
         ASSERT_TRUE(f != 0);
-        f[0] = 11.0;
-        f[1] = 12.0;
-        f[2] = 13.0;
+        f[0].set_double(11.0);
+        f[1].set_double(12.0);
+        f[2].set_double(13.0);
         f = sf.getFeaturesByIndex(1);
         ASSERT_TRUE(f != 0);
-        f[0] = 21.0;
-        f[1] = 22.0;
-        f[2] = 23.0;
+        f[0].set_double(21.0);
+        f[1].set_double(22.0);
+        f[2].set_double(23.0);
         f = sf.getFeaturesByIndex(2);
         ASSERT_TRUE(f != 0);
-        f[0] = 31.0;
-        f[1] = 32.0;
-        f[2] = 33.0;
+        f[0].set_double(31.0);
+        f[1].set_double(32.0);
+        f[2].set_double(33.0);
         f = sf.getFeaturesByIndex(3);
         ASSERT_TRUE(f != 0);
-        f[0] = 41.0;
-        f[1] = 42.0;
-        f[2] = 43.0;
+        f[0].set_double(41.0);
+        f[1].set_data(Memory("test", 4));
+        f[2].set_double(43.0);
         f = sf.getFeaturesByIndex(4);
         ASSERT_TRUE(f != 0);
-        f[0] = 51.0;
-        f[1] = 52.0;
-        f[2] = 53.0;
-        EXPECT_TRUE(sf.getFeaturesByIndex(5) == 0);
+        f[0].set_double(51.0);
+        f[1].set_double(52.0);
+        f[2].set_double(53.0);
+        EXPECT_TRUE(sf.getFeaturesByIndex(5) == nullptr);
         {
             std::vector<uint32_t> docs;
             EXPECT_TRUE(sf.contains(docs));
@@ -107,45 +108,49 @@ Test::Main()
         }
         {
             cf = sf.getFeaturesByDocId(10);
-            ASSERT_TRUE(cf != 0);
-            EXPECT_APPROX(cf[0], 11.0, 10e-6);
-            EXPECT_APPROX(cf[1], 12.0, 10e-6);
-            EXPECT_APPROX(cf[2], 13.0, 10e-6);
+            ASSERT_TRUE(cf != nullptr);
+            EXPECT_APPROX(cf[0].as_double(), 11.0, 10e-6);
+            EXPECT_APPROX(cf[1].as_double(), 12.0, 10e-6);
+            EXPECT_APPROX(cf[2].as_double(), 13.0, 10e-6);
         }
         {
             cf = sf.getFeaturesByDocId(20);
-            ASSERT_TRUE(cf != 0);
-            EXPECT_APPROX(cf[0], 21.0, 10e-6);
-            EXPECT_APPROX(cf[1], 22.0, 10e-6);
-            EXPECT_APPROX(cf[2], 23.0, 10e-6);
+            ASSERT_TRUE(cf != nullptr);
+            EXPECT_APPROX(cf[0].as_double(), 21.0, 10e-6);
+            EXPECT_APPROX(cf[1].as_double(), 22.0, 10e-6);
+            EXPECT_APPROX(cf[2].as_double(), 23.0, 10e-6);
         }
         {
             cf = sf.getFeaturesByDocId(30);
-            ASSERT_TRUE(cf != 0);
-            EXPECT_APPROX(cf[0], 31.0, 10e-6);
-            EXPECT_APPROX(cf[1], 32.0, 10e-6);
-            EXPECT_APPROX(cf[2], 33.0, 10e-6);
+            ASSERT_TRUE(cf != nullptr);
+            EXPECT_APPROX(cf[0].as_double(), 31.0, 10e-6);
+            EXPECT_APPROX(cf[1].as_double(), 32.0, 10e-6);
+            EXPECT_APPROX(cf[2].as_double(), 33.0, 10e-6);
         }
         {
             cf = sf.getFeaturesByDocId(40);
-            ASSERT_TRUE(cf != 0);
-            EXPECT_APPROX(cf[0], 41.0, 10e-6);
-            EXPECT_APPROX(cf[1], 42.0, 10e-6);
-            EXPECT_APPROX(cf[2], 43.0, 10e-6);
+            ASSERT_TRUE(cf != nullptr);
+            EXPECT_TRUE(cf[0].is_double());
+            EXPECT_TRUE(!cf[0].is_data());
+            EXPECT_EQUAL(cf[0].as_double(), 41.0);
+            EXPECT_TRUE(!cf[1].is_double());
+            EXPECT_TRUE(cf[1].is_data());
+            EXPECT_EQUAL(cf[1].as_data(), Memory("test", 4));
+            EXPECT_EQUAL(cf[2].as_double(), 43.0);
         }
         {
             cf = sf.getFeaturesByDocId(50);
-            ASSERT_TRUE(cf != 0);
-            EXPECT_APPROX(cf[0], 51.0, 10e-6);
-            EXPECT_APPROX(cf[1], 52.0, 10e-6);
-            EXPECT_APPROX(cf[2], 53.0, 10e-6);
+            ASSERT_TRUE(cf != nullptr);
+            EXPECT_APPROX(cf[0].as_double(), 51.0, 10e-6);
+            EXPECT_APPROX(cf[1].as_double(), 52.0, 10e-6);
+            EXPECT_APPROX(cf[2].as_double(), 53.0, 10e-6);
         }
-        EXPECT_TRUE(sf.getFeaturesByDocId(5) == 0);
-        EXPECT_TRUE(sf.getFeaturesByDocId(15) == 0);
-        EXPECT_TRUE(sf.getFeaturesByDocId(25) == 0);
-        EXPECT_TRUE(sf.getFeaturesByDocId(35) == 0);
-        EXPECT_TRUE(sf.getFeaturesByDocId(45) == 0);
-        EXPECT_TRUE(sf.getFeaturesByDocId(55) == 0);
+        EXPECT_TRUE(sf.getFeaturesByDocId(5) == nullptr);
+        EXPECT_TRUE(sf.getFeaturesByDocId(15) == nullptr);
+        EXPECT_TRUE(sf.getFeaturesByDocId(25) == nullptr);
+        EXPECT_TRUE(sf.getFeaturesByDocId(35) == nullptr);
+        EXPECT_TRUE(sf.getFeaturesByDocId(45) == nullptr);
+        EXPECT_TRUE(sf.getFeaturesByDocId(55) == nullptr);
     }
     TEST_DONE();
 }
diff --git a/searchlib/src/vespa/searchlib/common/featureset.cpp b/searchlib/src/vespa/searchlib/common/featureset.cpp
index 07b04f2f675..adf24196200 100644
--- a/searchlib/src/vespa/searchlib/common/featureset.cpp
+++ b/searchlib/src/vespa/searchlib/common/featureset.cpp
@@ -59,7 +59,7 @@ FeatureSet::contains(const std::vector<uint32_t> &docIds) const
     return true;
 }
 
-feature_t *
+FeatureSet::Value *
 FeatureSet::getFeaturesByIndex(uint32_t idx)
 {
     if (idx >= _docIds.size()) {
@@ -68,7 +68,7 @@ FeatureSet::getFeaturesByIndex(uint32_t idx)
     return &(_values[idx * _names.size()]);
 }
 
-const feature_t *
+const FeatureSet::Value *
 FeatureSet::getFeaturesByDocId(uint32_t docId) const
 {
     uint32_t low = 0;
diff --git a/searchlib/src/vespa/searchlib/common/featureset.h b/searchlib/src/vespa/searchlib/common/featureset.h
index f57cf918ba5..1ec662685a7 100644
--- a/searchlib/src/vespa/searchlib/common/featureset.h
+++ b/searchlib/src/vespa/searchlib/common/featureset.h
@@ -4,6 +4,7 @@
 
 #include "feature.h"
 #include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/data/memory.h>
 #include <map>
 #include <vector>
 
@@ -16,12 +17,34 @@ namespace search {
 class FeatureSet
 {
 public:
+    class Value {
+    private:
+        std::vector<char> _data;
+        double _value;
+    public:
+        bool operator==(const Value &rhs) const {
+            return ((_data == rhs._data) && (_value == rhs._value));
+        }
+        bool is_double() const { return _data.empty(); }
+        bool is_data() const { return !_data.empty(); }
+        double as_double() const { return _value; }
+        vespalib::Memory as_data() const { return vespalib::Memory(&_data[0], _data.size()); }
+        void set_double(double value) {
+            _data.clear();
+            _value = value;
+        }
+        void set_data(vespalib::Memory data) {
+            _data.assign(data.data, data.data + data.size);
+            _value = 0.0;
+        }
+    };
+
     typedef vespalib::string string;
     typedef std::vector<string> StringVector;
 private:
     StringVector _names;
-    std::vector<uint32_t>    _docIds;
-    std::vector<feature_t>   _values;
+    std::vector<uint32_t> _docIds;
+    std::vector<Value> _values;
 
     FeatureSet(const FeatureSet &);
     FeatureSet & operator=(const FeatureSet &);
@@ -112,7 +135,7 @@ public:
      * @return pointer to features
      * @param idx index into docid array
      **/
-    feature_t *getFeaturesByIndex(uint32_t idx);
+    Value *getFeaturesByIndex(uint32_t idx);
 
     /**
      * Obtain the feature values belonging to a document based on the
@@ -122,7 +145,7 @@ public:
      * @return pointer to features
      * @param docId docid value
      **/
-    const feature_t *getFeaturesByDocId(uint32_t docId) const;
+    const Value *getFeaturesByDocId(uint32_t docId) const;
 };
 
 } // namespace search