Revert "Add virtual fields to index environment for streaming mode."

13 files changed, 17 insertions, 114 deletions

diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp
index e78bcdc8757..759792d205d 100644
--- a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp
+++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp
@@ -220,12 +220,10 @@ VerifyRankSetup::verifyConfig(const VerifyRanksetupConfig &myCfg,
     if (_searchMode == SearchMode::STREAMING) {
         streamingProto.set_ranking_assets_repo(repo);
         streamingProto.detectFields(vsmFieldsCfg);
-        streamingProto.add_virtual_fields();
         factory = [&](const search::fef::Properties &properties)
                   {
                       auto indexEnv = streamingProto.clone();
                       indexEnv->getProperties().import(properties);
-                      indexEnv->fixup_fields();
                       return indexEnv;
                   };
     } else {
diff --git a/searchlib/src/vespa/searchlib/query/streaming/queryterm.cpp b/searchlib/src/vespa/searchlib/query/streaming/queryterm.cpp
index 90bf276af77..07fc60d2243 100644
--- a/searchlib/src/vespa/searchlib/query/streaming/queryterm.cpp
+++ b/searchlib/src/vespa/searchlib/query/streaming/queryterm.cpp
@@ -159,10 +159,4 @@ QueryTerm::as_equiv_query_node() const noexcept
     return nullptr;
 }
 
-bool
-QueryTerm::is_same_element_query_node() const noexcept
-{
-    return false;
-}
-
 }
diff --git a/searchlib/src/vespa/searchlib/query/streaming/queryterm.h b/searchlib/src/vespa/searchlib/query/streaming/queryterm.h
index 05b12804d52..78b0a1fea7d 100644
--- a/searchlib/src/vespa/searchlib/query/streaming/queryterm.h
+++ b/searchlib/src/vespa/searchlib/query/streaming/queryterm.h
@@ -108,7 +108,6 @@ public:
     virtual RegexpTerm* as_regexp_term() noexcept;
     virtual FuzzyTerm* as_fuzzy_term() noexcept;
     virtual const EquivQueryNode* as_equiv_query_node() const noexcept;
-    virtual bool is_same_element_query_node() const noexcept;
     virtual void unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data, const fef::IIndexEnvironment& index_env);
 protected:
     template <typename HitListType>
diff --git a/searchlib/src/vespa/searchlib/query/streaming/same_element_query_node.cpp b/searchlib/src/vespa/searchlib/query/streaming/same_element_query_node.cpp
index fee1feb511d..cd9c693ca1c 100644
--- a/searchlib/src/vespa/searchlib/query/streaming/same_element_query_node.cpp
+++ b/searchlib/src/vespa/searchlib/query/streaming/same_element_query_node.cpp
@@ -95,10 +95,4 @@ SameElementQueryNode::multi_index_terms() const noexcept
     return true;
 }
 
-bool
-SameElementQueryNode::is_same_element_query_node() const noexcept
-{
-    return true;
-}
-
 }
diff --git a/searchlib/src/vespa/searchlib/query/streaming/same_element_query_node.h b/searchlib/src/vespa/searchlib/query/streaming/same_element_query_node.h
index 87f5d06d35b..37fb3dbba52 100644
--- a/searchlib/src/vespa/searchlib/query/streaming/same_element_query_node.h
+++ b/searchlib/src/vespa/searchlib/query/streaming/same_element_query_node.h
@@ -18,7 +18,6 @@ public:
     const HitList & evaluateHits(HitList & hl) const override;
     void unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data, const fef::IIndexEnvironment& index_env) override;
     bool multi_index_terms() const noexcept override;
-    bool is_same_element_query_node() const noexcept override;
 };
 
 }
diff --git a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.cpp b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.cpp
index 104309f50fa..726afcc959b 100644
--- a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.cpp
+++ b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.cpp
@@ -3,7 +3,6 @@
 #include "indexenvironment.h"
 #include <vespa/searchlib/fef/i_ranking_assets_repo.h>
 #include <vespa/searchlib/fef/indexproperties.h>
-#include <vespa/vespalib/stllike/hash_set.h>
 
 using namespace search::fef;
 
@@ -40,37 +39,6 @@ IndexEnvironment::addField(const vespalib::string& name,
     return true;
 }
 
-/*
- * Ensure that array and map fields are known by the index
- * environment, allowing the matches features to be used with the
- * sameElement query operator. FieldSearchSpecMap::buildFromConfig()
- * propagates the name to field id mapping for the added virtual
- * fields.
- */
-void
-IndexEnvironment::add_virtual_fields()
-{
-    vespalib::hash_set<vespalib::string> vfields;
-    for (auto& field : _fields) {
-        vespalib::stringref name(field.name());
-        auto pos = name.rfind('.');
-        while (pos != vespalib::string::npos) {
-            name = name.substr(0, pos);
-            if (_fieldNames.contains(name)) {
-                break;
-            }
-            vfields.insert(name);
-            pos = name.rfind('.');
-        }
-    }
-    for (auto& vfield : vfields) {
-        FieldInfo info(FieldType::VIRTUAL, FieldInfo::CollectionType::ARRAY, vfield, _fields.size());
-        info.set_data_type(FieldInfo::DataType::COMBINED);
-        _fields.push_back(info);
-        _fieldNames[vfield] = info.id();
-    }
-}
-
 void
 IndexEnvironment::fixup_fields()
 {
diff --git a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h
index fdf5d7d870e..50e6898262d 100644
--- a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h
+++ b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h
@@ -78,8 +78,6 @@ public:
                   bool isAttribute,
                   search::fef::FieldInfo::DataType data_type);
 
-    void add_virtual_fields();
-
     void fixup_fields();
 
     search::fef::Properties & getProperties() { return _properties; }
diff --git a/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp b/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp
index c1ca5daf1cb..3efeb8ef168 100644
--- a/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp
+++ b/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp
@@ -78,37 +78,20 @@ IndexEnvPrototype::detectFields(const vespa::config::search::vsm::VsmfieldsConfi
     }
 }
 
-void
-IndexEnvPrototype::add_virtual_fields()
-{
-    _prototype.add_virtual_fields();
-}
-
 namespace {
 
 FieldIdTList
 buildFieldSet(const VsmfieldsConfig::Documenttype::Index & ci, const search::fef::IIndexEnvironment & indexEnv,
-              const VsmfieldsConfig::Documenttype::IndexVector & indexes, bool prefer_virtual_fields)
+              const VsmfieldsConfig::Documenttype::IndexVector & indexes)
 {
     LOG(spam, "Index %s with %zd fields", ci.name.c_str(), ci.field.size());
     FieldIdTList ifm;
-    if (prefer_virtual_fields) {
-        /*
-         * Stop at an existing virtual field when setting up views
-         * used by the same element query operator.
-         */
-        auto info = indexEnv.getFieldByName(ci.name);
-        if (info != nullptr && info->type() == search::fef::FieldType::VIRTUAL) {
-            ifm.push_back(info->id());
-            return ifm;
-        }
-    }
     for (const VsmfieldsConfig::Documenttype::Index::Field & cf : ci.field) {
         LOG(spam, "Parsing field %s", cf.name.c_str());
         auto foundIndex = std::find_if(indexes.begin(), indexes.end(),
                                        [&cf](const auto & v) { return v.name == cf.name;});
         if ((foundIndex != indexes.end()) && (cf.name != ci.name)) {
-            FieldIdTList sub = buildFieldSet(*foundIndex, indexEnv, indexes, prefer_virtual_fields);
+            FieldIdTList sub = buildFieldSet(*foundIndex, indexEnv, indexes);
             ifm.insert(ifm.end(), sub.begin(), sub.end());
         } else {
             const FieldInfo * info = indexEnv.getFieldByName(cf.name);
@@ -128,15 +111,15 @@ buildFieldSet(const VsmfieldsConfig::Documenttype::Index & ci, const search::fef
 }
 
 void
-RankManager::Snapshot::build_field_mappings(const VsmfieldsHandle& fields, ViewMap& views, bool prefer_virtual_fields)
+RankManager::Snapshot::buildFieldMappings(const VsmfieldsHandle & fields)
 {
     for(const VsmfieldsConfig::Documenttype & di : fields->documenttype) {
         LOG(debug, "Looking through indexes for documenttype '%s'", di.name.c_str());
         for(const VsmfieldsConfig::Documenttype::Index & ci : di.index) {
-            FieldIdTList view = buildFieldSet(ci, _protoEnv.current(), di.index, prefer_virtual_fields);
-            if (views.find(ci.name) == views.end()) {
+            FieldIdTList view = buildFieldSet(ci, _protoEnv.current(), di.index);
+            if (_views.find(ci.name) == _views.end()) {
                 std::sort(view.begin(), view.end()); // lowest field id first
-                views[ci.name] = view;
+                _views[ci.name] = view;
             } else {
                 LOG(warning, "We already have a view for index '%s'. Drop the new view.", ci.name.c_str());
             }
@@ -144,13 +127,6 @@ RankManager::Snapshot::build_field_mappings(const VsmfieldsHandle& fields, ViewM
     }
 }
 
-void
-RankManager::Snapshot::build_field_mappings(const VsmfieldsHandle& fields)
-{
-    build_field_mappings(fields, _views, false);
-    build_field_mappings(fields, _same_element_views, true);
-}
-
 bool
 RankManager::Snapshot::initRankSetup(const BlueprintFactory & factory)
 {
@@ -194,8 +170,7 @@ RankManager::Snapshot::Snapshot() :
     _indexEnv(),
     _rankSetup(),
     _rpmap(),
-    _views(),
-    _same_element_views()
+    _views()
 {
 }
 
@@ -206,8 +181,7 @@ RankManager::Snapshot::setup(const RankManager & rm)
 {
     VsmfieldsHandle fields = rm._vsmAdapter->getFieldsConfig();
     _protoEnv.detectFields(*fields);
-    _protoEnv.add_virtual_fields();
-    build_field_mappings(fields);
+    buildFieldMappings(fields);
     if (!initRankSetup(rm._blueprintFactory)) {
         return false;
     }
diff --git a/streamingvisitors/src/vespa/searchvisitor/rankmanager.h b/streamingvisitors/src/vespa/searchvisitor/rankmanager.h
index 52d44420ebc..12785daeb89 100644
--- a/streamingvisitors/src/vespa/searchvisitor/rankmanager.h
+++ b/streamingvisitors/src/vespa/searchvisitor/rankmanager.h
@@ -19,7 +19,6 @@ private:
 public:
     IndexEnvPrototype();
     void detectFields(const vespa::config::search::vsm::VsmfieldsConfig &fields);
-    void add_virtual_fields();
     void set_ranking_assets_repo(std::shared_ptr<const search::fef::IRankingAssetsRepo> repo) {
         _prototype.set_ranking_assets_repo(std::move(repo));
     }
@@ -54,11 +53,9 @@ public:
         std::vector<std::shared_ptr<const search::fef::RankSetup>> _rankSetup;  // rank setup per rank profile
         Map                                       _rpmap;
         ViewMap                                   _views;
-        ViewMap                                   _same_element_views;
 
         void addProperties(const vespa::config::search::RankProfilesConfig & cfg);
-        void build_field_mappings(const vsm::VsmfieldsHandle& fields, ViewMap& views, bool prefer_virtual_fields);
-        void build_field_mappings(const vsm::VsmfieldsHandle& fields);
+        void buildFieldMappings(const vsm::VsmfieldsHandle & fields);
         bool initRankSetup(const search::fef::BlueprintFactory & factory);
         bool setup(const RankManager & manager);
         int getIndex(const vespalib::string & key) const {
@@ -77,13 +74,9 @@ public:
         const IndexEnvironment & getIndexEnvironment(const vespalib::string &rankProfile) const {
             return _indexEnv[getIndex(rankProfile)];
         }
-        const IndexEnvironment& get_proto_index_environment() const {
-            return _protoEnv.current();
-        }
-        const View *getView(const vespalib::string & index, bool is_same_element) const {
-            auto&  views = is_same_element ? _same_element_views : _views;
-            auto itr = views.find(index);
-            if (itr != views.end()) {
+        const View *getView(const vespalib::string & index) const {
+            auto itr = _views.find(index);
+            if (itr != _views.end()) {
                 return &itr->second;
             }
             return nullptr;
diff --git a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp
index 72c1ca60814..cfaaac8b197 100644
--- a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp
+++ b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp
@@ -7,7 +7,6 @@
 #include <vespa/searchlib/query/streaming/equiv_query_node.h>
 #include <vespa/searchlib/query/streaming/nearest_neighbor_query_node.h>
 #include <vespa/vsm/vsm/fieldsearchspec.h>
-#include <vespa/vespalib/stllike/hash_set.h>
 #include <algorithm>
 #include <cmath>
 #include <vespa/log/log.h>
@@ -62,7 +61,7 @@ RankProcessor::resolve_fields_from_children(QueryTermData& qtd, const MultiTerm&
     vespalib::hash_set<uint32_t> field_ids;
     for (auto& subterm : mt.get_terms()) {
         vespalib::string expandedIndexName = vsm::FieldSearchSpecMap::stripNonFields(subterm->index());
-        const RankManager::View *view = _rankManagerSnapshot->getView(expandedIndexName, false);
+        const RankManager::View *view = _rankManagerSnapshot->getView(expandedIndexName);
         if (view != nullptr) {
             for (auto field_id : *view) {
                 field_ids.insert(field_id);
@@ -87,7 +86,7 @@ void
 RankProcessor::resolve_fields_from_term(QueryTermData& qtd, const search::streaming::QueryTerm& term)
 {
     vespalib::string expandedIndexName = vsm::FieldSearchSpecMap::stripNonFields(term.index());
-    const RankManager::View *view = _rankManagerSnapshot->getView(expandedIndexName, term.is_same_element_query_node());
+    const RankManager::View *view = _rankManagerSnapshot->getView(expandedIndexName);
     if (view != nullptr) {
         for (auto field_id : *view) {
             qtd.getTermData().addField(field_id).setHandle(_mdLayout.allocTermField(field_id));
diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp
index 95d7f01b15d..1b8e215ac0b 100644
--- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp
+++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp
@@ -499,7 +499,7 @@ SearchVisitor::init(const Parameters & params)
             VISITOR_TRACE(9, vespalib::make_string("Setting up for query blob of %zu bytes", queryBlob.size()));
             // Create mapping from field name to field id, from field id to search spec,
             // and from index name to list of field ids
-            _fieldSearchSpecMap.buildFromConfig(_env->get_vsm_fields_config(), _env->get_rank_manager_snapshot()->get_proto_index_environment());
+            _fieldSearchSpecMap.buildFromConfig(_env->get_vsm_fields_config());
             auto additionalFields = registerAdditionalFields(_env->get_docsum_tools()->getFieldSpecs());
             // Add extra elements to mapping from field name to field id
             _fieldSearchSpecMap.buildFromConfig(additionalFields);
diff --git a/streamingvisitors/src/vespa/vsm/vsm/fieldsearchspec.cpp b/streamingvisitors/src/vespa/vsm/vsm/fieldsearchspec.cpp
index 1ab1b16cb86..1dbac859262 100644
--- a/streamingvisitors/src/vespa/vsm/vsm/fieldsearchspec.cpp
+++ b/streamingvisitors/src/vespa/vsm/vsm/fieldsearchspec.cpp
@@ -1,8 +1,6 @@
 // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "fieldsearchspec.h"
-#include <vespa/searchlib/fef/fieldinfo.h>
-#include <vespa/searchlib/fef/iindexenvironment.h>
 #include <vespa/searchlib/query/streaming/equiv_query_node.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <vespa/vsm/searcher/boolfieldsearcher.h>
@@ -286,7 +284,7 @@ normalize_mode(VsmfieldsConfig::Fieldspec::Normalize normalize_mode) {
 }
 
 void
-FieldSearchSpecMap::buildFromConfig(const VsmfieldsHandle & conf, const search::fef::IIndexEnvironment& index_env)
+FieldSearchSpecMap::buildFromConfig(const VsmfieldsHandle & conf)
 {
     LOG(spam, "Parsing %zd fields", conf->fieldspec.size());
     for(const VsmfieldsConfig::Fieldspec & cfs : conf->fieldspec) {
@@ -297,15 +295,6 @@ FieldSearchSpecMap::buildFromConfig(const VsmfieldsHandle & conf, const search::
         _nameIdMap.add(cfs.name, fieldId);
         LOG(spam, "M in %d = %s", fieldId, cfs.name.c_str());
     }
-    /*
-     * Index env is based on same vsm fields config but has additional
-     * virtual fields, cf. IndexEnvironment::add_virtual_fields().
-     */
-    for (uint32_t field_id = specMap().size(); field_id < index_env.getNumFields(); ++field_id) {
-        auto& field = *index_env.getField(field_id);
-        assert(field.type() == search::fef::FieldType::VIRTUAL);
-        _nameIdMap.add(field.name(), field_id);
-    }
 
     LOG(spam, "Parsing %zd document types", conf->documenttype.size());
     for(const VsmfieldsConfig::Documenttype & di : conf->documenttype) {
diff --git a/streamingvisitors/src/vespa/vsm/vsm/fieldsearchspec.h b/streamingvisitors/src/vespa/vsm/vsm/fieldsearchspec.h
index 5b5a6b9a783..e90aea1371b 100644
--- a/streamingvisitors/src/vespa/vsm/vsm/fieldsearchspec.h
+++ b/streamingvisitors/src/vespa/vsm/vsm/fieldsearchspec.h
@@ -5,8 +5,6 @@
 #include <vespa/vsm/searcher/fieldsearcher.h>
 #include <vespa/vsm/config/vsm-cfif.h>
 
-namespace search::fef { class IIndexEnvironment; }
-
 namespace vsm {
 
 class FieldSearchSpec
@@ -69,7 +67,7 @@ public:
      * and a mapping from field name to field id. It then iterates over all document types and index names
      * and creates a mapping from index name to list of field ids for each document type.
      **/
-    void buildFromConfig(const VsmfieldsHandle & conf, const search::fef::IIndexEnvironment& index_env);
+    void buildFromConfig(const VsmfieldsHandle & conf);
 
     /**
      * Iterates over the given field name vector adding extra elements to the mapping from field name to field id.