diff options
18 files changed, 57 insertions, 59 deletions
diff --git a/Gemfile.lock b/Gemfile.lock index 0134d845e34..e7b22dd38be 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -25,7 +25,7 @@ GEM fiber-annotation (0.2.0) fiber-local (1.0.0) forwardable-extended (2.6.0) - google-protobuf (3.24.4) + google-protobuf (3.24.4-x86_64-linux) hashery (2.1.2) html-proofer (5.0.8) addressable (~> 2.3) @@ -77,7 +77,7 @@ GEM jekyll (>= 3.5, < 5.0) jekyll-feed (~> 0.9) jekyll-seo-tag (~> 2.1) - nokogiri (1.15.4-x86_64-linux) + nokogiri (1.16.2-x86_64-linux) racc (~> 1.4) pathutil (0.16.2) forwardable-extended (~> 2.6) @@ -88,7 +88,7 @@ GEM ruby-rc4 ttfunk public_suffix (5.0.4) - racc (1.7.1) + racc (1.7.3) rainbow (3.1.1) rake (13.1.0) rb-fsevent (0.11.2) diff --git a/client/go/go.mod b/client/go/go.mod index 8699f3e9245..48cceef3685 100644 --- a/client/go/go.mod +++ b/client/go/go.mod @@ -8,7 +8,7 @@ require ( github.com/fatih/color v1.16.0 // This is the most recent version compatible with Go 1.20. Upgrade when we upgrade our Go version github.com/go-json-experiment/json v0.0.0-20230324203220-04923b7a9528 - github.com/klauspost/compress v1.17.5 + github.com/klauspost/compress v1.17.6 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.20 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c diff --git a/client/go/go.sum b/client/go/go.sum index fc5730a071d..98637fe244a 100644 --- a/client/go/go.sum +++ b/client/go/go.sum @@ -22,6 +22,8 @@ github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/compress v1.17.5 h1:d4vBd+7CHydUqpFBgUEKkSdtSugf9YFmSkvUYPquI5E= github.com/klauspost/compress v1.17.5/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= +github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index a6910c059fc..d44babdd10b 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -115,7 +115,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"vekterli"}, removeAfter = "8.292.x") default long mergingMaxMemoryUsagePerNode() { return 0; } @ModelFeatureFlag(owners = {"vekterli"}, removeAfter = "8.292.x") default boolean usePerDocumentThrottledDeleteBucket() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean alwaysMarkPhraseExpensive() { return false; } - @ModelFeatureFlag(owners = {"hmusum"}) default boolean restartOnDeployWhenOnnxModelChanges() { return false; } + @ModelFeatureFlag(owners = {"hmusum"}, removeAfter = "8.300.x") default boolean restartOnDeployWhenOnnxModelChanges() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean sortBlueprintsByCost() { return false; } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index b088231e84a..a6429c43d0e 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -84,7 +84,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private List<DataplaneToken> dataplaneTokens; private int contentLayerMetadataFeatureLevel = 0; private boolean dynamicHeapSize = false; - private boolean restartOnDeployWhenOnnxModelChanges = false; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -142,7 +141,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public List<DataplaneToken> dataplaneTokens() { return dataplaneTokens; } @Override public int contentLayerMetadataFeatureLevel() { return contentLayerMetadataFeatureLevel; } @Override public boolean dynamicHeapSize() { return dynamicHeapSize; } - @Override public boolean restartOnDeployWhenOnnxModelChanges() { return restartOnDeployWhenOnnxModelChanges; } public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim; @@ -375,11 +373,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea public TestProperties setDynamicHeapSize(boolean b) { this.dynamicHeapSize = b; return this; } - public TestProperties setRestartOnDeployForOnnxModelChanges(boolean enable) { - this.restartOnDeployWhenOnnxModelChanges = enable; - return this; - } - public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java index b809f36a436..008a3fc5547 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java @@ -35,8 +35,7 @@ public class RestartOnDeployForOnnxModelChangesValidator implements ChangeValida @Override public void validate(ChangeContext context) { - if ( ! context.deployState().featureFlags().restartOnDeployWhenOnnxModelChanges() - || ! context.deployState().isHosted()) return; + if ( ! context.deployState().isHosted()) return; // Compare onnx models used by each cluster and set restart on deploy for cluster if estimated cost, // model hash or model options have changed diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java index 92c7fffd72f..4c0786ea879 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java @@ -218,9 +218,7 @@ public class RestartOnDeployForOnnxModelChangesValidatorTest { private static DeployState.Builder deployStateBuilder(boolean hosted) { var builder = new DeployState.Builder() - .properties((new TestProperties()) - .setRestartOnDeployForOnnxModelChanges(true) - .setHostedVespa(hosted)); + .properties((new TestProperties()).setHostedVespa(hosted)); if (hosted) builder.endpoints(Set.of(new ContainerEndpoint("cluster1", ApplicationClusterEndpoint.Scope.zone, List.of("tc.example.com")))) .modelHostProvisioner(new InMemoryProvisioner(5, new NodeResources(1, 2, 25, 0.3), true)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index d500e56d079..26732d2e20f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -206,7 +206,6 @@ public class ModelContextImpl implements ModelContext { private final int contentLayerMetadataFeatureLevel; private final String unknownConfigDefinition; private final int searchHandlerThreadpool; - private final boolean restartOnDeployWhenOnnxModelChanges; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -248,7 +247,6 @@ public class ModelContextImpl implements ModelContext { this.unknownConfigDefinition = flagValue(source, appId, version, Flags.UNKNOWN_CONFIG_DEFINITION); this.searchHandlerThreadpool = flagValue(source, appId, version, Flags.SEARCH_HANDLER_THREADPOOL); this.alwaysMarkPhraseExpensive = flagValue(source, appId, version, Flags.ALWAYS_MARK_PHRASE_EXPENSIVE); - this.restartOnDeployWhenOnnxModelChanges = flagValue(source, appId, version, Flags.RESTART_ON_DEPLOY_WHEN_ONNX_MODEL_CHANGES); this.sortBlueprintsByCost = flagValue(source, appId, version, Flags.SORT_BLUEPRINTS_BY_COST); } @@ -299,7 +297,6 @@ public class ModelContextImpl implements ModelContext { @Override public int contentLayerMetadataFeatureLevel() { return contentLayerMetadataFeatureLevel; } @Override public String unknownConfigDefinition() { return unknownConfigDefinition; } @Override public int searchHandlerThreadpool() { return searchHandlerThreadpool; } - @Override public boolean restartOnDeployWhenOnnxModelChanges() { return restartOnDeployWhenOnnxModelChanges; } @Override public boolean sortBlueprintsByCost() { return sortBlueprintsByCost; } private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { diff --git a/container-search/src/main/java/com/yahoo/search/query/Ranking.java b/container-search/src/main/java/com/yahoo/search/query/Ranking.java index 3c2a8a83c40..b1dd5624d18 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Ranking.java +++ b/container-search/src/main/java/com/yahoo/search/query/Ranking.java @@ -233,7 +233,13 @@ public class Ranking implements Cloneable { public Sorting getSorting() { return sorting; } /** Sets how this query should be sorted. Set to null to turn off explicit sorting. */ - public void setSorting(Sorting sorting) { this.sorting = sorting; } + public void setSorting(Sorting sorting) { + if (sorting == null || sorting.fieldOrders().isEmpty()) { + this.sorting = null; + } else { + this.sorting = sorting; + } + } /** Sets sorting from a string. See {@link Sorting} on syntax */ public void setSorting(String sortingString) { diff --git a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java index 91acc883803..1003cd942a8 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java @@ -37,7 +37,10 @@ public class GlobalPhaseRanker { var setup = globalPhaseSetupFor(query, schema).orElse(null); if (setup == null) return Optional.empty(); var sorting = query.getRanking().getSorting(); - if (sorting == null || sorting.fieldOrders() == null) return Optional.empty(); + if (sorting == null) return Optional.empty(); + if (sorting.fieldOrders() == null || sorting.fieldOrders().isEmpty()) { + return Optional.of(ErrorMessage.createIllegalQuery("Invalid sorting for global phase")); + } for (var fieldOrder : sorting.fieldOrders()) { if (!fieldOrder.getSorter().getName().equals("[rank]") || fieldOrder.getSortOrder() != Sorting.Order.DESCENDING) { diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index 79ead2bd075..fc928b918be 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -62,7 +62,7 @@ <apache.httpcore5.vespa.version>5.2.4</apache.httpcore5.vespa.version> <apiguardian.vespa.version>1.1.2</apiguardian.vespa.version> <asm.vespa.version>9.6</asm.vespa.version> - <assertj.vespa.version>3.25.2</assertj.vespa.version> + <assertj.vespa.version>3.25.3</assertj.vespa.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> <aws-sdk.vespa.version>1.12.651</aws-sdk.vespa.version> @@ -109,7 +109,7 @@ <jetty-servlet-api.vespa.version>5.0.2</jetty-servlet-api.vespa.version> <jimfs.vespa.version>1.3.0</jimfs.vespa.version> <jna.vespa.version>5.14.0</jna.vespa.version> - <joda-time.vespa.version>2.12.6</joda-time.vespa.version> + <joda-time.vespa.version>2.12.7</joda-time.vespa.version> <junit.vespa.version>5.10.2</junit.vespa.version> <junit.platform.vespa.version>1.10.2</junit.platform.vespa.version> <junit4.vespa.version>4.13.2</junit4.vespa.version> diff --git a/integration/intellij/build.gradle.kts b/integration/intellij/build.gradle.kts index b4f2c92ec44..39eb867e964 100644 --- a/integration/intellij/build.gradle.kts +++ b/integration/intellij/build.gradle.kts @@ -4,7 +4,7 @@ import org.jetbrains.grammarkit.tasks.GenerateParserTask plugins { id("java-library") - id("org.jetbrains.intellij") version "1.17.0" + id("org.jetbrains.intellij") version "1.17.1" id("org.jetbrains.grammarkit") version "2022.3.2.1" id("maven-publish") // to deploy the plugin into a Maven repo } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp index 5a58aa869e0..c434203898e 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp @@ -223,11 +223,6 @@ public: setEstimate(HitEstimate(_activeLids.size(), false)); } - FlowStats calculate_flow_stats(uint32_t docid_limit) const override { - auto est = abs_to_rel_est(getState().estimate().estHits, docid_limit); - return {est, 1.0, est}; - } - bool isWhiteList() const noexcept final { return true; } SearchIterator::UP createFilterSearch(bool strict, FilterConstraint) const override { diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index 8742f53bc8e..09da4697583 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -129,6 +129,7 @@ private: // as only a few ISearchContext implementations exposes the query term. vespalib::string _query_term; ISearchContext::UP _search_context; + attribute::HitEstimate _hit_estimate; enum Type {INT, FLOAT, OTHER}; Type _type; @@ -139,6 +140,18 @@ public: QueryTermSimple::UP term, const SearchContextParams ¶ms); ~AttributeFieldBlueprint() override; + search::queryeval::FlowStats calculate_flow_stats(uint32_t docid_limit) const override { + if (_hit_estimate.is_unknown()) { + // E.g. attributes without fast-search are not able to provide a hit estimate. + // In this case we just assume matching half of the document corpus. + // In addition, we are not able to skip documents efficiently when being strict. + return {0.5, 1.0, 1.0}; + } else { + double rel_est = abs_to_rel_est(_hit_estimate.est_hits(), docid_limit); + return {rel_est, 1.0, rel_est}; + } + } + SearchIteratorUP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool strict) const override { assert(tfmda.size() == 1); return _search_context->createIterator(tfmda[0], strict); @@ -182,9 +195,10 @@ AttributeFieldBlueprint::AttributeFieldBlueprint(FieldSpecBase field, const IAtt _attr(attribute), _query_term(term->getTermString()), _search_context(attribute.createSearchContext(std::move(term), params)), + _hit_estimate(_search_context->calc_hit_estimate()), _type(OTHER) { - uint32_t estHits = _search_context->calc_hit_estimate().est_hits(); + uint32_t estHits = _hit_estimate.est_hits(); HitEstimate estimate(estHits, estHits == 0); setEstimate(estimate); if (attribute.isFloatingPointType()) { @@ -194,8 +208,6 @@ AttributeFieldBlueprint::AttributeFieldBlueprint(FieldSpecBase field, const IAtt } } - - void AttributeFieldBlueprint::visitMembers(vespalib::ObjectVisitor &visitor) const { diff --git a/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.cpp b/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.cpp index 1317d1c0651..07d16c7bddc 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.cpp @@ -1,6 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "nearest_neighbor_query_node.h" +#include <vespa/searchlib/fef/itermdata.h> +#include <vespa/searchlib/fef/matchdata.h> #include <cassert> namespace search::streaming { @@ -49,4 +51,18 @@ NearestNeighborQueryNode::get_raw_score() const return std::nullopt; } +void +NearestNeighborQueryNode::unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data) +{ + auto raw_score = get_raw_score(); + if (raw_score.has_value()) { + if (td.numFields() == 1u) { + auto& tfd = td.field(0u); + auto tmd = match_data.resolveTermField(tfd.getHandle()); + assert(tmd != nullptr); + tmd->setRawScore(docid, raw_score.value()); + } + } +} + } diff --git a/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.h b/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.h index c3eaad45031..277956e27a4 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.h +++ b/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.h @@ -52,6 +52,7 @@ public: const std::optional<double>& get_distance() const { return _distance; } // This is used during unpacking, and also signals to the RawScoreCalculator that the entire document was a match. std::optional<double> get_raw_score() const; + void unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data) override; }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp index 7f34bc82c03..b05e5cbef3d 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp @@ -718,14 +718,7 @@ FlowStats LeafBlueprint::calculate_flow_stats(uint32_t docid_limit) const { double rel_est = abs_to_rel_est(_state.estimate().estHits, docid_limit); - if (rel_est > 0.9) { - // Assume we do not really know how much we are matching when - // we claim to match 'everything'. Also assume we are not able - // to skip documents efficiently when strict. - return {0.5, 1.0, 1.0}; - } else { - return {rel_est, 1.0, rel_est}; - } + return {rel_est, 1.0, rel_est}; } void diff --git a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp index 09699f79427..bc78c24ba1b 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp @@ -258,24 +258,7 @@ void RankProcessor::unpack_match_data(uint32_t docid, MatchData &matchData, QueryWrapper& query) { for (QueryWrapper::Term & term: query.getTermList()) { - auto nn_node = term.getTerm()->as_nearest_neighbor_query_node(); - if (nn_node != nullptr) { - auto raw_score = nn_node->get_raw_score(); - if (raw_score.has_value()) { - auto& qtd = static_cast<QueryTermData &>(term.getTerm()->getQueryItem()); - auto& td = qtd.getTermData(); - if (td.numFields() == 1u) { - auto tfd = td.field(0u); - auto tmd = matchData.resolveTermField(tfd.getHandle()); - assert(tmd != nullptr); - tmd->setRawScore(docid, raw_score.value()); - } - } - } else if (auto multi_term = term.getTerm()->as_multi_term()) { - auto& qtd = static_cast<QueryTermData &>(term.getTerm()->getQueryItem()); - auto& td = qtd.getTermData(); - multi_term->unpack_match_data(docid, td, matchData); - } else if (!term.isPhraseTerm() || term.isFirstPhraseTerm()) { // consider 1 term data per phrase + if (!term.isPhraseTerm() || term.isFirstPhraseTerm()) { // consider 1 term data per phrase bool isPhrase = term.isFirstPhraseTerm(); QueryTermData & qtd = static_cast<QueryTermData &>(term.getTerm()->getQueryItem()); const ITermData &td = qtd.getTermData(); |