diff options
17 files changed, 403 insertions, 245 deletions
diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index 7f73eaaf617..98c50fb0be8 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -4761,9 +4761,9 @@ prettier-linter-helpers@^1.0.0: fast-diff "^1.1.2" prettier@3: - version "3.1.0" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.1.0.tgz#c6d16474a5f764ea1a4a373c593b779697744d5e" - integrity sha512-TQLvXjq5IAibjh8EpBIkNKxO749UEWABoiIZehEPiY4GNpVdhaFKqSTu+QrlU6D2dPAfubRmtJTi4K4YkQ5eXw== + version "3.1.1" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.1.1.tgz#6ba9f23165d690b6cbdaa88cb0807278f7019848" + integrity sha512-22UbSzg8luF4UuZtzgiUOfcGM8s4tjBv6dJRT7j275NXsy2jb4aJa4NNveul5x4eqlF1wuhuR2RElK71RvmVaw== pretty-format@^29.7.0: version "29.7.0" diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java index 60f325cbe43..e9038ff2b0f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java @@ -15,6 +15,9 @@ import java.util.logging.Level; */ public class JvmHeapSizeValidator extends Validator { + public static final int percentLimit = 15; + public static final double gbLimit = 0.6; + @Override public void validate(VespaModel model, DeployState ds) { if (!ds.featureFlags().dynamicHeapSize()) return; @@ -29,8 +32,6 @@ public class JvmHeapSizeValidator extends Validator { } long jvmModelCost = appCluster.onnxModelCostCalculator().aggregatedModelCostInBytes(); if (jvmModelCost > 0) { - int percentLimit = 15; - double gbLimit = 0.6; double availableMemoryGb = mp.availableMemoryGb().getAsDouble(); double modelCostGb = jvmModelCost / (1024D * 1024 * 1024); ds.getDeployLogger().log(Level.FINE, () -> Text.format("JVM: %d%% (limit: %d%%), %.2fGB (limit: %.2fGB), ONNX: %.2fGB", diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java index 355bce24c0f..15e9e526db2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.model.deploy.DeployState; @@ -15,8 +16,11 @@ import java.util.Optional; import java.util.Set; import java.util.logging.Logger; +import static com.yahoo.vespa.model.application.validation.JvmHeapSizeValidator.gbLimit; +import static com.yahoo.vespa.model.application.validation.JvmHeapSizeValidator.percentLimit; import static java.util.logging.Level.FINE; import static com.yahoo.config.model.api.OnnxModelCost.ModelInfo; +import static java.util.logging.Level.INFO; /** * If Onnx models change in a way that requires restart of containers in @@ -36,7 +40,6 @@ public class RestartOnDeployForOnnxModelChangesValidator implements ChangeValida // Compare onnx models used by each cluster and set restart on deploy for cluster if estimated cost, // model hash or model options have changed - // TODO: Skip if container has enough memory to handle reload of onnx model (2 models in memory at the same time) for (var cluster : nextModel.getContainerClusters().values()) { var clusterInCurrentModel = currentModel.getContainerClusters().get(cluster.getName()); if (clusterInCurrentModel == null) continue; @@ -44,6 +47,9 @@ public class RestartOnDeployForOnnxModelChangesValidator implements ChangeValida var currentModels = clusterInCurrentModel.onnxModelCostCalculator().models(); var nextModels = cluster.onnxModelCostCalculator().models(); + if (enoughMemoryToAvoidRestart(clusterInCurrentModel, cluster, deployState.getDeployLogger())) + continue; + log.log(FINE, "Validating " + cluster + ", current models=" + currentModels + ", next models=" + nextModels); actions.addAll(validateModelChanges(cluster, currentModels, nextModels)); actions.addAll(validateSetOfModels(cluster, currentModels, nextModels)); @@ -58,7 +64,6 @@ public class RestartOnDeployForOnnxModelChangesValidator implements ChangeValida for (var nextModelInfo : nextModels.values()) { if (! currentModels.containsKey(nextModelInfo.modelId())) continue; - log.log(FINE, "Checking if " + nextModelInfo + " has changed"); modelChanged(nextModelInfo, currentModels.get(nextModelInfo.modelId())).ifPresent(change -> { String message = "Onnx model '%s' has changed (%s), need to restart services in %s" .formatted(nextModelInfo.modelId(), change, cluster); @@ -84,6 +89,7 @@ public class RestartOnDeployForOnnxModelChangesValidator implements ChangeValida } private Optional<String> modelChanged(OnnxModelCost.ModelInfo a, OnnxModelCost.ModelInfo b) { + log.log(FINE, "Checking if model has changed (%s) -> (%s)".formatted(a, b)); if (a.estimatedCost() != b.estimatedCost()) return Optional.of("estimated cost"); if (a.hash() != b.hash()) return Optional.of("model hash"); if (a.onnxModelOptions().isPresent() && b.onnxModelOptions().isEmpty()) return Optional.of("model option(s)"); @@ -94,16 +100,38 @@ public class RestartOnDeployForOnnxModelChangesValidator implements ChangeValida } private static void setRestartOnDeployAndAddRestartAction(List<ConfigChangeAction> actions, ApplicationContainerCluster cluster, String message) { + log.log(INFO, message); cluster.onnxModelCostCalculator().setRestartOnDeploy(); actions.add(new VespaRestartAction(cluster.id(), message)); } - private static boolean enoughMemoryToAvoidRestart(ApplicationContainerCluster cluster) { - // Node memory is known so convert available memory percentage to node memory percentage + private static boolean enoughMemoryToAvoidRestart(ApplicationContainerCluster clusterInCurrentModel, + ApplicationContainerCluster cluster, + DeployLogger deployLogger) { + double currentModelCostInGb = onnxModelCostInGb(clusterInCurrentModel); + double nextModelCostInGb = onnxModelCostInGb(cluster); + double totalMemory = cluster.getContainers().get(0).getHostResource().realResources().memoryGb(); - double availableMemory = Math.max(0, totalMemory - Host.memoryOverheadGb); - double costInGb = (double) cluster.onnxModelCostCalculator().aggregatedModelCostInBytes() / 1024 / 1024 / 1024; - return ( 2 * costInGb < availableMemory); + double availableMemory = Math.max(0, totalMemory - Host.memoryOverheadGb - currentModelCostInGb - currentModelCostInGb); + if (availableMemory <= 0.0) + return false; + + var availableMemoryPercentage = cluster.availableMemoryPercentage(); + int memoryPercentage = (int) (availableMemory / totalMemory * availableMemoryPercentage); + + if (memoryPercentage < percentLimit || availableMemory < gbLimit) { + deployLogger.log(INFO, "Validating %s, not enough memory (%s) to avoid restart (models require %s), consider a flavor with more memory to avoid this" + .formatted(cluster, availableMemory, currentModelCostInGb + nextModelCostInGb)); + return false; + } + + log.log(FINE, "Validating " + cluster + ", enough memory (%s) to avoid restart (models require %s)" + .formatted(availableMemory, currentModelCostInGb + nextModelCostInGb)); + return true; + } + + private static double onnxModelCostInGb(ApplicationContainerCluster clusterInCurrentModel) { + return (double) clusterInCurrentModel.onnxModelCostCalculator().aggregatedModelCostInBytes() / 1024 / 1024 / 1024; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 20b5c687257..8c4adfb96cb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -210,9 +210,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat if (memoryPercentage != null) return Optional.of(JvmMemoryPercentage.of(memoryPercentage)); if (isHostedVespa()) { - int availableMemoryPercentage = getHostClusterId().isPresent() ? - heapSizePercentageOfTotalAvailableMemoryWhenCombinedCluster : - heapSizePercentageOfAvailableMemory; + int availableMemoryPercentage = availableMemoryPercentage(); if (getContainers().isEmpty()) return Optional.of(JvmMemoryPercentage.of(availableMemoryPercentage)); // Node memory is not known // Node memory is known so convert available memory percentage to node memory percentage @@ -229,6 +227,12 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat return Optional.empty(); } + public int availableMemoryPercentage() { + return getHostClusterId().isPresent() ? + heapSizePercentageOfTotalAvailableMemoryWhenCombinedCluster : + heapSizePercentageOfAvailableMemory; + } + /** Create list of endpoints, these will be consumed later by LbServicesProducer */ private void createEndpoints(DeployState deployState) { if (!configureEndpoints(deployState)) return; diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index 484f6d8b3b3..b088baeefe6 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -65,7 +65,7 @@ <assertj.vespa.version>3.24.2</assertj.vespa.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> - <athenz.vespa.version>1.11.47</athenz.vespa.version> + <athenz.vespa.version>1.11.48</athenz.vespa.version> <aws-sdk.vespa.version>1.12.595</aws-sdk.vespa.version> <!-- Athenz END --> @@ -108,17 +108,17 @@ <jetty.vespa.version>11.0.18</jetty.vespa.version> <jetty-servlet-api.vespa.version>5.0.2</jetty-servlet-api.vespa.version> <jimfs.vespa.version>1.3.0</jimfs.vespa.version> - <jna.vespa.version>5.13.0</jna.vespa.version> + <jna.vespa.version>5.14.0</jna.vespa.version> <joda-time.vespa.version>2.12.5</joda-time.vespa.version> <junit.vespa.version>5.10.0</junit.vespa.version> <junit.platform.vespa.version>1.10.0</junit.platform.vespa.version> <junit4.vespa.version>4.13.2</junit4.vespa.version> <luben.zstd.vespa.version>1.5.5-11</luben.zstd.vespa.version> - <lucene.vespa.version>9.8.0</lucene.vespa.version> + <lucene.vespa.version>9.9.0</lucene.vespa.version> <maven-archiver.vespa.version>3.6.1</maven-archiver.vespa.version> <maven-wagon.vespa.version>3.5.3</maven-wagon.vespa.version> <mimepull.vespa.version>1.10.0</mimepull.vespa.version> - <mockito.vespa.version>5.7.0</mockito.vespa.version> + <mockito.vespa.version>5.8.0</mockito.vespa.version> <mojo-executor.vespa.version>2.4.0</mojo-executor.vespa.version> <netty.vespa.version>4.1.101.Final</netty.vespa.version> <netty-tcnative.vespa.version>2.0.62.Final</netty-tcnative.vespa.version> @@ -129,9 +129,9 @@ <org.lz4.vespa.version>1.8.0</org.lz4.vespa.version> <prometheus.client.vespa.version>0.16.0</prometheus.client.vespa.version> <protobuf.vespa.version>3.25.1</protobuf.vespa.version> - <questdb.vespa.version>7.3.5</questdb.vespa.version> + <questdb.vespa.version>7.3.7</questdb.vespa.version> <spifly.vespa.version>1.3.7</spifly.vespa.version> - <spotbugs.vespa.version>4.8.1</spotbugs.vespa.version> <!-- Must match major version in https://github.com/apache/zookeeper/blob/master/pom.xml --> + <spotbugs.vespa.version>4.8.2</spotbugs.vespa.version> <!-- Must match major version in https://github.com/apache/zookeeper/blob/master/pom.xml --> <snappy.vespa.version>1.1.10.5</snappy.vespa.version> <surefire.vespa.version>3.2.2</surefire.vespa.version> <velocity.vespa.version>2.3</velocity.vespa.version> diff --git a/searchlib/src/tests/attribute/direct_multi_term_blueprint/direct_multi_term_blueprint_test.cpp b/searchlib/src/tests/attribute/direct_multi_term_blueprint/direct_multi_term_blueprint_test.cpp index 400bc39c88b..cfdeb35e0fc 100644 --- a/searchlib/src/tests/attribute/direct_multi_term_blueprint/direct_multi_term_blueprint_test.cpp +++ b/searchlib/src/tests/attribute/direct_multi_term_blueprint/direct_multi_term_blueprint_test.cpp @@ -49,9 +49,9 @@ concat(const Docids& a, const Docids& b) } std::shared_ptr<AttributeVector> -make_attribute(bool field_is_filter) +make_attribute(bool field_is_filter, CollectionType col_type) { - Config cfg(BasicType::INT64, CollectionType::WSET); + Config cfg(BasicType::INT64, col_type); cfg.setFastSearch(true); if (field_is_filter) { cfg.setIsFilter(field_is_filter); @@ -78,7 +78,7 @@ make_attribute(bool field_is_filter) } void -expect_has_weight_iterator(const IDocidWithWeightPostingStore& store, int64_t term_value) +expect_has_weight_iterator(const IDirectPostingStore& store, int64_t term_value) { auto snapshot = store.get_dictionary_snapshot(); auto res = store.lookup(IntegerKey(term_value), snapshot); @@ -86,7 +86,7 @@ expect_has_weight_iterator(const IDocidWithWeightPostingStore& store, int64_t te } void -expect_has_bitvector_iterator(const IDocidWithWeightPostingStore& store, int64_t term_value) +expect_has_bitvector_iterator(const IDirectPostingStore& store, int64_t term_value) { auto snapshot = store.get_dictionary_snapshot(); auto res = store.lookup(IntegerKey(term_value), snapshot); @@ -106,18 +106,28 @@ validate_posting_lists(const IDocidWithWeightPostingStore& store) expect_has_bitvector_iterator(store, 300); } -class DirectMultiTermBlueprintTest : public ::testing::Test { +struct TestParam { + CollectionType col_type; + TestParam(CollectionType col_type_in) : col_type(col_type_in) {} + ~TestParam() = default; +}; + +std::ostream& operator<<(std::ostream& os, const TestParam& param) +{ + os << param.col_type.asString(); + return os; +} + +class DirectMultiTermBlueprintTest : public ::testing::TestWithParam<TestParam> { public: using BlueprintType = DirectMultiTermBlueprint<IDocidWithWeightPostingStore, WeightedSetTermSearch>; std::shared_ptr<AttributeVector> attr; - const IDocidWithWeightPostingStore* store; std::shared_ptr<BlueprintType> blueprint; Blueprint::HitEstimate estimate; fef::TermFieldMatchData tfmd; fef::TermFieldMatchDataArray tfmda; DirectMultiTermBlueprintTest() : attr(), - store(), blueprint(), tfmd(), tfmda() @@ -125,8 +135,8 @@ public: tfmda.add(&tfmd); } void setup(bool field_is_filter, bool need_term_field_match_data) { - attr = make_attribute(field_is_filter); - store = attr->as_docid_with_weight_posting_store(); + attr = make_attribute(field_is_filter, GetParam().col_type); + const auto* store = attr->as_docid_with_weight_posting_store(); ASSERT_TRUE(store); validate_posting_lists(*store); blueprint = std::make_shared<BlueprintType>(FieldSpec(field_name, field_id, fef::TermFieldHandle(), field_is_filter), *attr, *store, 2); @@ -168,7 +178,12 @@ expect_or_child(SearchIterator& itr, size_t child, const vespalib::string& exp_c EXPECT_THAT(real.getChildren()[child]->asString(), StartsWith(exp_child_itr)); } -TEST_F(DirectMultiTermBlueprintTest, weight_iterators_used_for_none_filter_field) +INSTANTIATE_TEST_SUITE_P(DefaultInstantiation, + DirectMultiTermBlueprintTest, + testing::Values(CollectionType::WSET), + testing::PrintToStringParamName()); + +TEST_P(DirectMultiTermBlueprintTest, weight_iterators_used_for_none_filter_field) { setup(false, true); add_term(1); @@ -178,7 +193,7 @@ TEST_F(DirectMultiTermBlueprintTest, weight_iterators_used_for_none_filter_field expect_hits({10, 30, 31}, *itr); } -TEST_F(DirectMultiTermBlueprintTest, weight_iterators_used_instead_of_bitvectors_for_none_filter_field) +TEST_P(DirectMultiTermBlueprintTest, weight_iterators_used_instead_of_bitvectors_for_none_filter_field) { setup(false, true); add_term(1); @@ -188,7 +203,7 @@ TEST_F(DirectMultiTermBlueprintTest, weight_iterators_used_instead_of_bitvectors expect_hits(concat({10}, range(100, 128)), *itr); } -TEST_F(DirectMultiTermBlueprintTest, bitvectors_and_weight_iterators_used_for_filter_field) +TEST_P(DirectMultiTermBlueprintTest, bitvectors_and_weight_iterators_used_for_filter_field) { setup(true, true); add_term(1); @@ -203,7 +218,7 @@ TEST_F(DirectMultiTermBlueprintTest, bitvectors_and_weight_iterators_used_for_fi expect_hits(concat({10, 30, 31}, concat(range(100, 128), range(300, 128))), *itr); } -TEST_F(DirectMultiTermBlueprintTest, only_bitvectors_used_for_filter_field) +TEST_P(DirectMultiTermBlueprintTest, only_bitvectors_used_for_filter_field) { setup(true, true); add_term(100); @@ -215,7 +230,7 @@ TEST_F(DirectMultiTermBlueprintTest, only_bitvectors_used_for_filter_field) expect_hits(concat(range(100, 128), range(300, 128)), *itr); } -TEST_F(DirectMultiTermBlueprintTest, filter_iterator_used_for_filter_field_and_ranking_not_needed) +TEST_P(DirectMultiTermBlueprintTest, filter_iterator_used_for_filter_field_and_ranking_not_needed) { setup(true, false); add_term(1); @@ -225,7 +240,7 @@ TEST_F(DirectMultiTermBlueprintTest, filter_iterator_used_for_filter_field_and_r expect_hits({10, 30, 31}, *itr); } -TEST_F(DirectMultiTermBlueprintTest, bitvectors_and_filter_iterator_used_for_filter_field_and_ranking_not_needed) +TEST_P(DirectMultiTermBlueprintTest, bitvectors_and_filter_iterator_used_for_filter_field_and_ranking_not_needed) { setup(true, false); add_term(1); @@ -240,7 +255,7 @@ TEST_F(DirectMultiTermBlueprintTest, bitvectors_and_filter_iterator_used_for_fil expect_hits(concat({10, 30, 31}, concat(range(100, 128), range(300, 128))), *itr); } -TEST_F(DirectMultiTermBlueprintTest, only_bitvectors_used_for_filter_field_and_ranking_not_needed) +TEST_P(DirectMultiTermBlueprintTest, only_bitvectors_used_for_filter_field_and_ranking_not_needed) { setup(true, false); add_term(100); diff --git a/searchlib/src/vespa/searchlib/attribute/direct_posting_store_adapter.h b/searchlib/src/vespa/searchlib/attribute/direct_posting_store_adapter.h new file mode 100644 index 00000000000..125c265afcf --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/direct_posting_store_adapter.h @@ -0,0 +1,44 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_direct_posting_store.h" +#include <vespa/vespalib/datastore/entryref.h> +#include <vector> + +namespace search { class IEnumStoreDictionary; } + +namespace search::attribute { + +/** + * Base adapter class used to implement a specific IDirectPostingStore interface for + * an attribute vector with underlying posting lists (fast-search). + */ +template <typename ParentType, typename PostingStoreType, typename EnumStoreType> +class DirectPostingStoreAdapter : public ParentType { +protected: + const PostingStoreType& _posting_store; + const EnumStoreType& _enum_store; + const IEnumStoreDictionary& _dict; + bool _attr_is_filter; + +public: + using IteratorType = typename ParentType::IteratorType; + + DirectPostingStoreAdapter(const PostingStoreType& posting_store, + const EnumStoreType& enum_store, + bool attr_is_filter); + + vespalib::datastore::EntryRef get_dictionary_snapshot() const override; + bool has_weight_iterator(vespalib::datastore::EntryRef posting_idx) const noexcept override; + std::unique_ptr<queryeval::SearchIterator> make_bitvector_iterator(vespalib::datastore::EntryRef posting_idx, uint32_t doc_id_limit, + fef::TermFieldMatchData& match_data, bool strict) const override; + bool has_bitvector(vespalib::datastore::EntryRef posting_idx) const noexcept override; + int64_t get_integer_value(vespalib::datastore::EntryRef enum_idx) const noexcept override; + + void create(vespalib::datastore::EntryRef idx, std::vector<IteratorType>& dst) const override; + IteratorType create(vespalib::datastore::EntryRef idx) const override; + bool has_always_weight_iterator() const noexcept override { return !_attr_is_filter; } +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/direct_posting_store_adapter.hpp b/searchlib/src/vespa/searchlib/attribute/direct_posting_store_adapter.hpp new file mode 100644 index 00000000000..02fc1a84ec6 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/direct_posting_store_adapter.hpp @@ -0,0 +1,74 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_enum_store_dictionary.h" +#include "direct_posting_store_adapter.h" +#include <cassert> + +namespace search::attribute { + +template <typename ParentType, typename PostingStoreType, typename EnumStoreType> +DirectPostingStoreAdapter<ParentType, PostingStoreType, EnumStoreType>:: +DirectPostingStoreAdapter(const PostingStoreType& posting_store, + const EnumStoreType& enum_store, + bool attr_is_filter) + : _posting_store(posting_store), + _enum_store(enum_store), + _dict(enum_store.get_dictionary()), + _attr_is_filter(attr_is_filter) +{ +} + +template <typename ParentType, typename PostingStoreType, typename EnumStoreType> +vespalib::datastore::EntryRef +DirectPostingStoreAdapter<ParentType, PostingStoreType, EnumStoreType>:: +get_dictionary_snapshot() const +{ + return _dict.get_frozen_root(); +} + +template <typename ParentType, typename PostingStoreType, typename EnumStoreType> +std::unique_ptr<queryeval::SearchIterator> +DirectPostingStoreAdapter<ParentType, PostingStoreType, EnumStoreType>:: +make_bitvector_iterator(vespalib::datastore::EntryRef posting_idx, uint32_t doc_id_limit, + fef::TermFieldMatchData& match_data, bool strict) const +{ + return _posting_store.make_bitvector_iterator(posting_idx, doc_id_limit, match_data, strict); +} + +template <typename ParentType, typename PostingStoreType, typename EnumStoreType> +bool +DirectPostingStoreAdapter<ParentType, PostingStoreType, EnumStoreType>:: +has_weight_iterator(vespalib::datastore::EntryRef posting_idx) const noexcept +{ + return _posting_store.has_btree(posting_idx); +} + +template <typename ParentType, typename PostingStoreType, typename EnumStoreType> +bool +DirectPostingStoreAdapter<ParentType, PostingStoreType, EnumStoreType>:: +has_bitvector(vespalib::datastore::EntryRef posting_idx) const noexcept +{ + return _posting_store.has_bitvector(posting_idx); +} + +template <typename ParentType, typename PostingStoreType, typename EnumStoreType> +void +DirectPostingStoreAdapter<ParentType, PostingStoreType, EnumStoreType>:: +create(vespalib::datastore::EntryRef posting_idx, std::vector<IteratorType>& dst) const +{ + assert(posting_idx.valid()); + _posting_store.beginFrozen(posting_idx, dst); +} + +template <typename ParentType, typename PostingStoreType, typename EnumStoreType> +DirectPostingStoreAdapter<ParentType, PostingStoreType, EnumStoreType>::IteratorType +DirectPostingStoreAdapter<ParentType, PostingStoreType, EnumStoreType>:: +create(vespalib::datastore::EntryRef posting_idx) const +{ + assert(posting_idx.valid()); + return _posting_store.beginFrozen(posting_idx); +} + +} diff --git a/searchlib/src/vespa/searchlib/attribute/i_docid_with_weight_posting_store.h b/searchlib/src/vespa/searchlib/attribute/i_docid_with_weight_posting_store.h index 1907279b39d..bdb4054b2d7 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_docid_with_weight_posting_store.h +++ b/searchlib/src/vespa/searchlib/attribute/i_docid_with_weight_posting_store.h @@ -13,6 +13,8 @@ namespace search { */ class IDocidWithWeightPostingStore : public IDirectPostingStore { public: + using IteratorType = DocidWithWeightIterator; + virtual void create(vespalib::datastore::EntryRef idx, std::vector<DocidWithWeightIterator> &dst) const = 0; virtual DocidWithWeightIterator create(vespalib::datastore::EntryRef idx) const = 0; @@ -24,5 +26,7 @@ public: virtual bool has_always_weight_iterator() const noexcept = 0; }; + + } diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h index 38cf12ca909..3e7ff01d484 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h @@ -2,6 +2,7 @@ #pragma once +#include "numeric_direct_posting_store_adapter.h" #include "multinumericenumattribute.h" #include "postinglistattribute.h" #include "i_docid_with_weight_posting_store.h" @@ -32,25 +33,6 @@ public: using EnumStoreBatchUpdater = typename EnumStore::BatchUpdater; private: - class DocidWithWeightPostingStoreAdapter final : public IDocidWithWeightPostingStore { - public: - const MultiValueNumericPostingAttribute &self; - bool _is_filter; - DocidWithWeightPostingStoreAdapter(const MultiValueNumericPostingAttribute &self_in) - : self(self_in), _is_filter(self_in.getIsFilter()) {} - vespalib::datastore::EntryRef get_dictionary_snapshot() const override; - LookupResult lookup(const LookupKey & key, vespalib::datastore::EntryRef dictionary_snapshot) const override; - void collect_folded(vespalib::datastore::EntryRef enum_idx, vespalib::datastore::EntryRef dictionary_snapshot, const std::function<void(vespalib::datastore::EntryRef)>& callback) const override; - void create(vespalib::datastore::EntryRef posting_idx, std::vector<DocidWithWeightIterator> &dst) const override; - DocidWithWeightIterator create(vespalib::datastore::EntryRef posting_idx) const override; - std::unique_ptr<queryeval::SearchIterator> make_bitvector_iterator(vespalib::datastore::EntryRef posting_idx, uint32_t doc_id_limit, fef::TermFieldMatchData &match_data, bool strict) const override; - bool has_weight_iterator(vespalib::datastore::EntryRef posting_idx) const noexcept override; - bool has_bitvector(vespalib::datastore::EntryRef posting_idx) const noexcept override; - int64_t get_integer_value(vespalib::datastore::EntryRef enum_idx) const noexcept override; - bool has_always_weight_iterator() const noexcept override { return !_is_filter; } - }; - DocidWithWeightPostingStoreAdapter _posting_store_adapter; - friend class PostingListAttributeTest; template <typename, typename, typename> friend class attribute::PostingSearchContext; // getEnumStore() @@ -73,6 +55,10 @@ private: using WeightedIndex = typename MultiValueNumericEnumAttribute<B, M>::WeightedIndex; using generation_t = typename MultiValueNumericEnumAttribute<B, M>::generation_t; + using DirectPostingStoreAdapterType = attribute::NumericDirectPostingStoreAdapter<IDocidWithWeightPostingStore, + PostingStore, EnumStore>; + DirectPostingStoreAdapterType _posting_store_adapter; + using PostingParent::_posting_store; using PostingParent::clearAllPostings; using PostingParent::handle_load_posting_lists; diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp index e90940f6ca0..ea1058d88fb 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp @@ -4,6 +4,7 @@ #include "multinumericpostattribute.h" #include "multi_numeric_enum_search_context.h" +#include "numeric_direct_posting_store_adapter.hpp" #include <vespa/searchcommon/attribute/config.h> #include <charconv> @@ -43,7 +44,7 @@ MultiValueNumericPostingAttribute<B, M>::MultiValueNumericPostingAttribute(const const AttributeVector::Config & cfg) : MultiValueNumericEnumAttribute<B, M>(name, cfg), PostingParent(*this, this->getEnumStore()), - _posting_store_adapter(*this) + _posting_store_adapter(this->get_posting_store(), this->_enumStore, this->getIsFilter()) { } @@ -85,88 +86,6 @@ MultiValueNumericPostingAttribute<B, M>::getSearch(QueryTermSimpleUP qTerm, } template <typename B, typename M> -vespalib::datastore::EntryRef -MultiValueNumericPostingAttribute<B, M>::DocidWithWeightPostingStoreAdapter::get_dictionary_snapshot() const -{ - const IEnumStoreDictionary& dictionary = self._enumStore.get_dictionary(); - return dictionary.get_frozen_root(); -} - -template <typename B, typename M> -IDirectPostingStore::LookupResult -MultiValueNumericPostingAttribute<B, M>::DocidWithWeightPostingStoreAdapter::lookup(const LookupKey & key, vespalib::datastore::EntryRef dictionary_snapshot) const -{ - const IEnumStoreDictionary& dictionary = self._enumStore.get_dictionary(); - int64_t int_term; - if ( !key.asInteger(int_term)) { - return LookupResult(); - } - auto comp = self._enumStore.make_comparator(int_term); - auto find_result = dictionary.find_posting_list(comp, dictionary_snapshot); - if (find_result.first.valid()) { - auto pidx = find_result.second; - if (pidx.valid()) { - const auto& store = self.get_posting_store(); - auto minmax = store.getAggregated(pidx); - return LookupResult(pidx, store.frozenSize(pidx), minmax.getMin(), minmax.getMax(), find_result.first); - } - } - return LookupResult(); -} - -template <typename B, typename M> -void -MultiValueNumericPostingAttribute<B, M>::DocidWithWeightPostingStoreAdapter::collect_folded(vespalib::datastore::EntryRef enum_idx, vespalib::datastore::EntryRef dictionary_snapshot, const std::function<void(vespalib::datastore::EntryRef)>& callback)const -{ - (void) dictionary_snapshot; - callback(enum_idx); -} - -template <typename B, typename M> -void -MultiValueNumericPostingAttribute<B, M>::DocidWithWeightPostingStoreAdapter::create(vespalib::datastore::EntryRef posting_idx, std::vector<DocidWithWeightIterator> &dst) const -{ - assert(posting_idx.valid()); - self.get_posting_store().beginFrozen(posting_idx, dst); -} - -template <typename B, typename M> -DocidWithWeightIterator -MultiValueNumericPostingAttribute<B, M>::DocidWithWeightPostingStoreAdapter::create(vespalib::datastore::EntryRef posting_idx) const -{ - assert(posting_idx.valid()); - return self.get_posting_store().beginFrozen(posting_idx); -} - -template <typename B, typename M> -std::unique_ptr<queryeval::SearchIterator> -MultiValueNumericPostingAttribute<B, M>::DocidWithWeightPostingStoreAdapter::make_bitvector_iterator(vespalib::datastore::EntryRef posting_idx, uint32_t doc_id_limit, fef::TermFieldMatchData &match_data, bool strict) const -{ - return self.get_posting_store().make_bitvector_iterator(posting_idx, doc_id_limit, match_data, strict); -} - -template <typename B, typename M> -bool -MultiValueNumericPostingAttribute<B, M>::DocidWithWeightPostingStoreAdapter::has_weight_iterator(vespalib::datastore::EntryRef posting_idx) const noexcept -{ - return self.get_posting_store().has_btree(posting_idx); -} - -template <typename B, typename M> -bool -MultiValueNumericPostingAttribute<B, M>::DocidWithWeightPostingStoreAdapter::has_bitvector(vespalib::datastore::EntryRef posting_idx) const noexcept -{ - return self.get_posting_store().has_bitvector(posting_idx); -} - -template <typename B, typename M> -int64_t -MultiValueNumericPostingAttribute<B, M>::DocidWithWeightPostingStoreAdapter::get_integer_value(vespalib::datastore::EntryRef enum_idx) const noexcept -{ - return self._enumStore.get_value(enum_idx); -} - -template <typename B, typename M> const IDocidWithWeightPostingStore* MultiValueNumericPostingAttribute<B, M>::as_docid_with_weight_posting_store() const { diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h index a8b93a93a34..63a445f0476 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h @@ -2,9 +2,10 @@ #pragma once +#include "i_docid_with_weight_posting_store.h" #include "multistringattribute.h" #include "postinglistattribute.h" -#include "i_docid_with_weight_posting_store.h" +#include "string_direct_posting_store_adapter.h" namespace search { @@ -30,25 +31,6 @@ public: using EnumStoreBatchUpdater = typename EnumStore::BatchUpdater; private: - class DocidWithWeightPostingStoreAdapter final : public IDocidWithWeightPostingStore { - public: - const MultiValueStringPostingAttributeT &self; - bool _is_filter; - DocidWithWeightPostingStoreAdapter(const MultiValueStringPostingAttributeT &self_in) - : self(self_in), _is_filter(self_in.getIsFilter()) {} - vespalib::datastore::EntryRef get_dictionary_snapshot() const override; - LookupResult lookup(const LookupKey & key, vespalib::datastore::EntryRef dictionary_snapshot) const override; - void collect_folded(vespalib::datastore::EntryRef enum_idx, vespalib::datastore::EntryRef dictionary_snapshot, const std::function<void(vespalib::datastore::EntryRef)>& callback) const override; - void create(vespalib::datastore::EntryRef posting_idx, std::vector<DocidWithWeightIterator> &dst) const override; - DocidWithWeightIterator create(vespalib::datastore::EntryRef posting_idx) const override; - std::unique_ptr<queryeval::SearchIterator> make_bitvector_iterator(vespalib::datastore::EntryRef posting_idx, uint32_t doc_id_limit, fef::TermFieldMatchData &match_data, bool strict) const override; - bool has_weight_iterator(vespalib::datastore::EntryRef posting_idx) const noexcept override; - bool has_bitvector(vespalib::datastore::EntryRef posting_idx) const noexcept override; - int64_t get_integer_value(vespalib::datastore::EntryRef enum_idx) const noexcept override; - bool has_always_weight_iterator() const noexcept override { return !_is_filter; } - }; - DocidWithWeightPostingStoreAdapter _posting_store_adapter; - using LoadedVector = typename B::LoadedVector; using PostingParent = PostingListAttributeSubBase<AttributeWeightPosting, LoadedVector, @@ -60,11 +42,18 @@ private: using DocIndices = typename MultiValueStringAttributeT<B, T>::DocIndices; using Posting = typename PostingParent::Posting; using PostingMap = typename PostingParent::PostingMap; +public: + using PostingStore = typename PostingParent::PostingStore; +private: using QueryTermSimpleUP = AttributeVector::QueryTermSimpleUP; using SelfType = MultiValueStringPostingAttributeT<B, T>; using WeightedIndex = typename MultiValueStringAttributeT<B, T>::WeightedIndex; using generation_t = typename MultiValueStringAttributeT<B, T>::generation_t; + using DirectPostingStoreAdapterType = attribute::StringDirectPostingStoreAdapter<IDocidWithWeightPostingStore, + PostingStore, EnumStore>; + DirectPostingStoreAdapterType _posting_store_adapter; + using PostingParent::_posting_store; using PostingParent::clearAllPostings; using PostingParent::handle_load_posting_lists; @@ -78,7 +67,6 @@ private: public: using PostingParent::get_posting_store; using Dictionary = EnumPostingTree; - using PostingStore = typename PostingParent::PostingStore; MultiValueStringPostingAttributeT(const vespalib::string & name, const AttributeVector::Config & c); MultiValueStringPostingAttributeT(const vespalib::string & name); diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp index 2909a6e0ea7..b6e9b69a81d 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp @@ -4,6 +4,7 @@ #include "multistringpostattribute.h" #include "multi_string_enum_search_context.h" +#include "string_direct_posting_store_adapter.hpp" #include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/query/query_term_simple.h> @@ -13,7 +14,7 @@ template <typename B, typename T> MultiValueStringPostingAttributeT<B, T>::MultiValueStringPostingAttributeT(const vespalib::string & name, const AttributeVector::Config & c) : MultiValueStringAttributeT<B, T>(name, c), PostingParent(*this, this->getEnumStore()), - _posting_store_adapter(*this) + _posting_store_adapter(this->get_posting_store(), this->_enumStore, this->getIsFilter()) { } @@ -103,90 +104,6 @@ MultiValueStringPostingAttributeT<B, T>::getSearch(QueryTermSimpleUP qTerm, return std::make_unique<SC>(std::move(base_sc), params.useBitVector(), *this); } - -template <typename B, typename T> -vespalib::datastore::EntryRef -MultiValueStringPostingAttributeT<B, T>::DocidWithWeightPostingStoreAdapter::get_dictionary_snapshot() const -{ - const IEnumStoreDictionary& dictionary = self._enumStore.get_dictionary(); - return dictionary.get_frozen_root(); -} - -template <typename B, typename T> -IDirectPostingStore::LookupResult -MultiValueStringPostingAttributeT<B, T>::DocidWithWeightPostingStoreAdapter::lookup(const LookupKey & key, vespalib::datastore::EntryRef dictionary_snapshot) const -{ - const IEnumStoreDictionary& dictionary = self._enumStore.get_dictionary(); - vespalib::stringref keyAsString = key.asString(); - // Assert the unfortunate assumption of the comparators. - // Should be lifted once they take the length too. - assert(keyAsString.data()[keyAsString.size()] == '\0'); - auto comp = self._enumStore.make_folded_comparator(keyAsString.data()); - auto find_result = dictionary.find_posting_list(comp, dictionary_snapshot); - if (find_result.first.valid()) { - auto pidx = find_result.second; - if (pidx.valid()) { - const auto& store = self.get_posting_store(); - auto minmax = store.getAggregated(pidx); - return LookupResult(pidx, store.frozenSize(pidx), minmax.getMin(), minmax.getMax(), find_result.first); - } - } - return LookupResult(); -} - -template <typename B, typename T> -void -MultiValueStringPostingAttributeT<B, T>::DocidWithWeightPostingStoreAdapter::collect_folded(vespalib::datastore::EntryRef enum_idx, vespalib::datastore::EntryRef dictionary_snapshot, const std::function<void(vespalib::datastore::EntryRef)>& callback) const -{ - const IEnumStoreDictionary &dictionary = self._enumStore.get_dictionary(); - dictionary.collect_folded(enum_idx, dictionary_snapshot, callback); -} - -template <typename B, typename T> -void -MultiValueStringPostingAttributeT<B, T>::DocidWithWeightPostingStoreAdapter::create(vespalib::datastore::EntryRef posting_idx, std::vector<DocidWithWeightIterator> &dst) const -{ - assert(posting_idx.valid()); - self.get_posting_store().beginFrozen(posting_idx, dst); -} - -template <typename B, typename M> -DocidWithWeightIterator -MultiValueStringPostingAttributeT<B, M>::DocidWithWeightPostingStoreAdapter::create(vespalib::datastore::EntryRef posting_idx) const -{ - assert(posting_idx.valid()); - return self.get_posting_store().beginFrozen(posting_idx); -} - -template <typename B, typename M> -bool -MultiValueStringPostingAttributeT<B, M>::DocidWithWeightPostingStoreAdapter::has_weight_iterator(vespalib::datastore::EntryRef posting_idx) const noexcept -{ - return self.get_posting_store().has_btree(posting_idx); -} - -template <typename B, typename M> -bool -MultiValueStringPostingAttributeT<B, M>::DocidWithWeightPostingStoreAdapter::has_bitvector(vespalib::datastore::EntryRef posting_idx) const noexcept -{ - return self.get_posting_store().has_bitvector(posting_idx); -} - -template <typename B, typename M> -int64_t -MultiValueStringPostingAttributeT<B, M>::DocidWithWeightPostingStoreAdapter::get_integer_value(vespalib::datastore::EntryRef) const noexcept -{ - // This is not supported for string attributes and is never called. - abort(); -} - -template <typename B, typename M> -std::unique_ptr<queryeval::SearchIterator> -MultiValueStringPostingAttributeT<B, M>::DocidWithWeightPostingStoreAdapter::make_bitvector_iterator(vespalib::datastore::EntryRef posting_idx, uint32_t doc_id_limit, fef::TermFieldMatchData &match_data, bool strict) const -{ - return self.get_posting_store().make_bitvector_iterator(posting_idx, doc_id_limit, match_data, strict); -} - template <typename B, typename T> const IDocidWithWeightPostingStore* MultiValueStringPostingAttributeT<B, T>::as_docid_with_weight_posting_store() const diff --git a/searchlib/src/vespa/searchlib/attribute/numeric_direct_posting_store_adapter.h b/searchlib/src/vespa/searchlib/attribute/numeric_direct_posting_store_adapter.h new file mode 100644 index 00000000000..16416df61e9 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/numeric_direct_posting_store_adapter.h @@ -0,0 +1,31 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "direct_posting_store_adapter.h" +#include <vespa/vespalib/datastore/entryref.h> + +namespace search::attribute { + +/** + * Adapter used to implement a specific IDirectPostingStore interface for + * a numeric attribute vector with underlying posting lists (fast-search). + */ +template <typename ParentType, typename PostingStoreType, typename EnumStoreType> +class NumericDirectPostingStoreAdapter : public DirectPostingStoreAdapter<ParentType, PostingStoreType, EnumStoreType> { +public: + using LookupKey = typename ParentType::LookupKey; + using LookupResult = typename ParentType::LookupResult; + + NumericDirectPostingStoreAdapter(const PostingStoreType& posting_store, + const EnumStoreType& enum_store, + bool attr_is_filter); + + LookupResult lookup(const LookupKey& key, + vespalib::datastore::EntryRef dictionary_snapshot) const override; + void collect_folded(vespalib::datastore::EntryRef enum_idx, vespalib::datastore::EntryRef dictionary_snapshot, + const std::function<void(vespalib::datastore::EntryRef)>& callback) const override; + int64_t get_integer_value(vespalib::datastore::EntryRef enum_idx) const noexcept override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/numeric_direct_posting_store_adapter.hpp b/searchlib/src/vespa/searchlib/attribute/numeric_direct_posting_store_adapter.hpp new file mode 100644 index 00000000000..b5a1282d09c --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/numeric_direct_posting_store_adapter.hpp @@ -0,0 +1,58 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "numeric_direct_posting_store_adapter.h" +#include "direct_posting_store_adapter.hpp" + +namespace search::attribute { + +template <typename ParentType, typename PostingStoreType, typename EnumStoreType> +NumericDirectPostingStoreAdapter<ParentType, PostingStoreType, EnumStoreType>:: +NumericDirectPostingStoreAdapter(const PostingStoreType& posting_store, + const EnumStoreType& enum_store, + bool attr_is_filter) + : DirectPostingStoreAdapter<ParentType, PostingStoreType, EnumStoreType>(posting_store, enum_store, attr_is_filter) +{ +} + +template <typename ParentType, typename PostingStoreType, typename EnumStoreType> +NumericDirectPostingStoreAdapter<ParentType, PostingStoreType, EnumStoreType>::LookupResult +NumericDirectPostingStoreAdapter<ParentType, PostingStoreType, EnumStoreType>:: +lookup(const LookupKey& key, vespalib::datastore::EntryRef dictionary_snapshot) const +{ + int64_t int_term; + if (!key.asInteger(int_term)) { + return LookupResult(); + } + auto comp = this->_enum_store.make_comparator(int_term); + auto find_result = this->_dict.find_posting_list(comp, dictionary_snapshot); + if (find_result.first.valid()) { + auto pidx = find_result.second; + if (pidx.valid()) { + auto minmax = this->_posting_store.getAggregated(pidx); + return LookupResult(pidx, this->_posting_store.frozenSize(pidx), minmax.getMin(), minmax.getMax(), find_result.first); + } + } + return LookupResult(); +} + +template <typename ParentType, typename PostingStoreType, typename EnumStoreType> +void +NumericDirectPostingStoreAdapter<ParentType, PostingStoreType, EnumStoreType>:: +collect_folded(vespalib::datastore::EntryRef enum_idx, vespalib::datastore::EntryRef dictionary_snapshot, + const std::function<void(vespalib::datastore::EntryRef)>& callback) const +{ + (void) dictionary_snapshot; + callback(enum_idx); +} + +template <typename ParentType, typename PostingStoreType, typename EnumStoreType> +int64_t +NumericDirectPostingStoreAdapter<ParentType, PostingStoreType, EnumStoreType>:: +get_integer_value(vespalib::datastore::EntryRef enum_idx) const noexcept +{ + return this->_enum_store.get_value(enum_idx); +} + +} diff --git a/searchlib/src/vespa/searchlib/attribute/string_direct_posting_store_adapter.h b/searchlib/src/vespa/searchlib/attribute/string_direct_posting_store_adapter.h new file mode 100644 index 00000000000..ca345c60d64 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/string_direct_posting_store_adapter.h @@ -0,0 +1,31 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "direct_posting_store_adapter.h" +#include <vespa/vespalib/datastore/entryref.h> + +namespace search::attribute { + +/** + * Adapter used to implement a specific IDirectPostingStore interface for + * a string attribute vector with underlying posting lists (fast-search). + */ +template <typename ParentType, typename PostingStoreType, typename EnumStoreType> +class StringDirectPostingStoreAdapter : public DirectPostingStoreAdapter<ParentType, PostingStoreType, EnumStoreType> { +public: + using LookupKey = typename ParentType::LookupKey; + using LookupResult = typename ParentType::LookupResult; + + StringDirectPostingStoreAdapter(const PostingStoreType& posting_store, + const EnumStoreType& enum_store, + bool attr_is_filter); + + LookupResult lookup(const LookupKey& key, + vespalib::datastore::EntryRef dictionary_snapshot) const override; + void collect_folded(vespalib::datastore::EntryRef enum_idx, vespalib::datastore::EntryRef dictionary_snapshot, + const std::function<void(vespalib::datastore::EntryRef)>& callback) const override; + int64_t get_integer_value(vespalib::datastore::EntryRef enum_idx) const noexcept override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/string_direct_posting_store_adapter.hpp b/searchlib/src/vespa/searchlib/attribute/string_direct_posting_store_adapter.hpp new file mode 100644 index 00000000000..9f29fe0ef46 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/string_direct_posting_store_adapter.hpp @@ -0,0 +1,58 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "string_direct_posting_store_adapter.h" +#include "direct_posting_store_adapter.hpp" + +namespace search::attribute { + +template <typename ParentType, typename PostingStoreType, typename EnumStoreType> +StringDirectPostingStoreAdapter<ParentType, PostingStoreType, EnumStoreType>:: +StringDirectPostingStoreAdapter(const PostingStoreType& posting_store, + const EnumStoreType& enum_store, + bool attr_is_filter) + : DirectPostingStoreAdapter<ParentType, PostingStoreType, EnumStoreType>(posting_store, enum_store, attr_is_filter) +{ +} + +template <typename ParentType, typename PostingStoreType, typename EnumStoreType> +StringDirectPostingStoreAdapter<ParentType, PostingStoreType, EnumStoreType>::LookupResult +StringDirectPostingStoreAdapter<ParentType, PostingStoreType, EnumStoreType>:: +lookup(const LookupKey& key, vespalib::datastore::EntryRef dictionary_snapshot) const +{ + vespalib::stringref keyAsString = key.asString(); + // Assert the unfortunate assumption of the comparators. + // Should be lifted once they take the length too. + assert(keyAsString.data()[keyAsString.size()] == '\0'); + auto comp = this->_enum_store.make_folded_comparator(keyAsString.data()); + auto find_result = this->_dict.find_posting_list(comp, dictionary_snapshot); + if (find_result.first.valid()) { + auto pidx = find_result.second; + if (pidx.valid()) { + auto minmax = this->_posting_store.getAggregated(pidx); + return LookupResult(pidx, this->_posting_store.frozenSize(pidx), minmax.getMin(), minmax.getMax(), find_result.first); + } + } + return LookupResult(); +} + +template <typename ParentType, typename PostingStoreType, typename EnumStoreType> +void +StringDirectPostingStoreAdapter<ParentType, PostingStoreType, EnumStoreType>:: +collect_folded(vespalib::datastore::EntryRef enum_idx, vespalib::datastore::EntryRef dictionary_snapshot, + const std::function<void(vespalib::datastore::EntryRef)>& callback) const +{ + this->_dict.collect_folded(enum_idx, dictionary_snapshot, callback); +} + +template <typename ParentType, typename PostingStoreType, typename EnumStoreType> +int64_t +StringDirectPostingStoreAdapter<ParentType, PostingStoreType, EnumStoreType>:: +get_integer_value(vespalib::datastore::EntryRef) const noexcept +{ + // This is not supported for string attributes and is never called. + abort(); +} + +} |