diff options
30 files changed, 204 insertions, 44 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxMemoryStats.java b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxMemoryStats.java index 4e660c6fe73..c45d69f02cb 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxMemoryStats.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxMemoryStats.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.api; diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java index 300a55e521a..799324d26e9 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java @@ -156,7 +156,8 @@ public class SummaryClass extends Derived { summaryField.getTransform() == SummaryTransform.POSITIONS || summaryField.getTransform() == SummaryTransform.MATCHED_ELEMENTS_FILTER || summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER || - summaryField.getTransform() == SummaryTransform.TOKENS) + summaryField.getTransform() == SummaryTransform.TOKENS || + summaryField.getTransform() == SummaryTransform.ATTRIBUTE_TOKENS) { return summaryField.getSingleSource(); } else if (summaryField.getTransform().isDynamic()) { diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java index 2f60cd8eb06..349fd06ebdd 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java @@ -94,6 +94,8 @@ public class SummaryClassField { return Type.FEATUREDATA; } else if (transform != null && transform.equals(SummaryTransform.TOKENS)) { return Type.JSONSTRING; + } else if (transform != null && transform.equals(SummaryTransform.ATTRIBUTE_TOKENS)) { + return Type.JSONSTRING; } else { return Type.LONGSTRING; } diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/NormalizerFunctionExpander.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/NormalizerFunctionExpander.java index a8fee966656..c6d4eaba47d 100644 --- a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/NormalizerFunctionExpander.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/NormalizerFunctionExpander.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.schema.expressiontransforms; import com.yahoo.schema.FeatureNames; diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AdjustSummaryTransforms.java b/config-model/src/main/java/com/yahoo/schema/processing/AdjustSummaryTransforms.java index dd6f118d113..fe26ada5c8b 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/AdjustSummaryTransforms.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/AdjustSummaryTransforms.java @@ -36,6 +36,7 @@ public class AdjustSummaryTransforms extends Processor { makeDocumentIdTransformIfAppropriate(summaryField); makeAttributeTransformIfAppropriate(summaryField, schema); makeAttributeCombinerTransformIfAppropriate(summaryField, schema); + makeAttributeTokensTransformIfAppropriate(summaryField, summary.getName(), schema); makeCopyTransformIfAppropriate(summaryField, schema); } } @@ -67,6 +68,25 @@ public class AdjustSummaryTransforms extends Processor { } } + private void makeAttributeTokensTransformIfAppropriate(SummaryField summaryField, String docsumName, Schema schema) { + if (summaryField.getTransform() == SummaryTransform.TOKENS) { + String sourceFieldName = summaryField.getSingleSource(); + Attribute attribute = schema.getAttribute(sourceFieldName); + ImmutableSDField source = schema.getField(sourceFieldName); + if (!source.doesIndexing()) { + if (attribute != null) { + summaryField.setTransform(SummaryTransform.ATTRIBUTE_TOKENS); + } else { + throw new IllegalArgumentException("For schema '" + schema.getName() + + "', document-summary '" + docsumName + + "', summary field '" + summaryField.getName() + + "', source field '" + sourceFieldName + + "': tokens summary field setting requires index or attribute for source field"); + } + } + } + } + /* * This function must be called after makeAttributeCombinerTransformIfAppropriate(). */ diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java index e4116c3f9d5..3dd2ffdb02e 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java @@ -79,7 +79,8 @@ public class IndexingOutputs extends Processor { } dynamicSummary.add(summaryName); } else if (summaryTransform != SummaryTransform.ATTRIBUTE && - summaryTransform != SummaryTransform.TOKENS) { + summaryTransform != SummaryTransform.TOKENS && + summaryTransform != SummaryTransform.ATTRIBUTE_TOKENS) { staticSummary.add(summaryName); } } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java b/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java index 420df3ee575..6ee82b31d57 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java @@ -42,6 +42,7 @@ public class MakeDefaultSummaryTheSuperSet extends Processor { if (summaryField.getTransform() == SummaryTransform.ATTRIBUTECOMBINER) continue; if (summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER) continue; if (summaryField.getTransform() == SummaryTransform.TOKENS) continue; + if (summaryField.getTransform() == SummaryTransform.ATTRIBUTE_TOKENS) continue; defaultSummary.add(summaryField.clone()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java index 58f47680f9f..36d300a9b0b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java @@ -24,7 +24,8 @@ public enum SummaryTransform { MATCHED_ATTRIBUTE_ELEMENTS_FILTER("matchedattributeelementsfilter"), COPY("copy"), DOCUMENT_ID("documentid"), - TOKENS("tokens"); + TOKENS("tokens"), + ATTRIBUTE_TOKENS("attribute-tokens"); private final String name; @@ -69,7 +70,7 @@ public enum SummaryTransform { return this==DYNAMICBOLDED || this==DYNAMICTEASER; } - public boolean isTokens() { return this == TOKENS; } + public boolean isTokens() { return this == TOKENS || this == ATTRIBUTE_TOKENS; } /** Returns whether this transform always gets its value by accessing memory only */ public boolean isInMemory() { diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java index 5019ed0dd60..5e02f63ec39 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java @@ -25,6 +25,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** * Tests summary extraction @@ -226,9 +227,10 @@ public class SummaryTestCase extends AbstractSchemaTestCase { assertOverride(schema, "documentid", SummaryTransform.DOCUMENT_ID.getName(), "", "bar"); } - @Test - void tokens_override() throws ParseException { - var schema = buildSchema("field foo type string { indexing: summary }", + private void check_tokens_override(boolean index, boolean attribute, SummaryTransform exp) throws ParseException { + var schema = buildSchema("field foo type string { indexing: " + + (index ? "index | " : "") + + (attribute ? "attribute | " : "") + "summary }", joinLines("document-summary bar {", " summary baz {", " source: foo ", @@ -236,11 +238,26 @@ public class SummaryTestCase extends AbstractSchemaTestCase { " }", " from-disk", "}")); - assertOverride(schema, "baz", SummaryTransform.TOKENS.getName(), "foo", "bar"); + assertOverride(schema, "baz", exp.getName(), "foo", "bar"); assert(!schema.getSummary("default").getSummaryFields().containsKey("baz")); } @Test + void tokens_override() throws ParseException { + try { + check_tokens_override(false, false, SummaryTransform.TOKENS); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'test', document-summary 'bar'" + + ", summary field 'baz', source field 'foo'" + + ": tokens summary field setting requires index or attribute for source field", e.getMessage()); + } + check_tokens_override(false, true, SummaryTransform.ATTRIBUTE_TOKENS); + check_tokens_override(true, false, SummaryTransform.TOKENS); + check_tokens_override(true, true, SummaryTransform.TOKENS); + } + + @Test void documentid_summary_transform_requires_disk_access() { assertFalse(SummaryTransform.DOCUMENT_ID.isInMemory()); } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/TokensTransformValidatorTest.java b/config-model/src/test/java/com/yahoo/schema/processing/TokensTransformValidatorTest.java index 6ca62321617..6da536efe86 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/TokensTransformValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/TokensTransformValidatorTest.java @@ -17,7 +17,7 @@ public class TokensTransformValidatorTest { "search test {", " document test {", " field f type " + fieldType + " {", - " indexing: summary", + " indexing: index | summary", " summary: tokens", " }", " }", diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index c7877c23323..ae1215fd5aa 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -40,6 +40,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceListResponse; +import static com.yahoo.vespa.config.server.session.Session.Status.DELETE; /** * The process of deploying an application. @@ -131,15 +132,8 @@ public class Deployment implements com.yahoo.config.provision.Deployment { TimeoutBudget timeoutBudget = params.getTimeoutBudget(); timeoutBudget.assertNotTimedOut(() -> "Timeout exceeded when trying to activate '" + applicationId + "'"); - try { - Activation activation = applicationRepository.activate(session, applicationId, tenant, params.force()); - waitForActivation(applicationId, timeoutBudget, activation); - } catch (Exception e) { - log.log(Level.FINE, "Activating session " + session.getSessionId() + " failed, deleting it"); - deleteSession(); - throw e; - } - + Activation activation = applicationRepository.activate(session, applicationId, tenant, params.force()); + waitForActivation(applicationId, timeoutBudget, activation); restartServicesIfNeeded(applicationId); storeReindexing(applicationId, session.getMetaData().getGeneration()); @@ -162,6 +156,9 @@ public class Deployment implements com.yahoo.config.provision.Deployment { private void deleteSession() { sessionRepository().deleteLocalSession(session.getSessionId()); + try (var transaction = sessionRepository().createSetStatusTransaction(session, DELETE)) { + transaction.commit(); + } } private SessionRepository sessionRepository() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index da5169cf493..5105bcbdc28 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -55,7 +55,7 @@ public class RemoteSession extends Session { @Override public String toString() { - return super.toString() + ",application set=" + applicationVersions; + return super.toString() + ", application versions=" + applicationVersions; } } diff --git a/container-search/src/main/java/com/yahoo/search/ranking/DummyEvaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/DummyEvaluator.java index e83a308d99c..549c9e9a4a4 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/DummyEvaluator.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/DummyEvaluator.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.ranking; import ai.vespa.models.evaluation.FunctionEvaluator; diff --git a/container-search/src/main/java/com/yahoo/search/ranking/FunEvalSpec.java b/container-search/src/main/java/com/yahoo/search/ranking/FunEvalSpec.java index ac1b7c8e218..153bad7b44c 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/FunEvalSpec.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/FunEvalSpec.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.ranking; import java.util.List; diff --git a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseSetup.java b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseSetup.java index 7783eabcdcc..c7c516bf2d3 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseSetup.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseSetup.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.ranking; import ai.vespa.models.evaluation.FunctionEvaluator; diff --git a/container-search/src/main/java/com/yahoo/search/ranking/MatchFeatureInput.java b/container-search/src/main/java/com/yahoo/search/ranking/MatchFeatureInput.java index f80f29b3668..07978bcec82 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/MatchFeatureInput.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/MatchFeatureInput.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.ranking; record MatchFeatureInput(String inputName, String matchFeatureName) {} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/NormalizerContext.java b/container-search/src/main/java/com/yahoo/search/ranking/NormalizerContext.java index ceac202db47..662c08476a1 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/NormalizerContext.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/NormalizerContext.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.ranking; import java.util.List; diff --git a/container-search/src/main/java/com/yahoo/search/ranking/NormalizerSetup.java b/container-search/src/main/java/com/yahoo/search/ranking/NormalizerSetup.java index 32fbb3190fc..419eef62607 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/NormalizerSetup.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/NormalizerSetup.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.ranking; import java.util.function.Supplier; diff --git a/container-search/src/main/java/com/yahoo/search/ranking/PreparedInput.java b/container-search/src/main/java/com/yahoo/search/ranking/PreparedInput.java index 914635fef59..17353c0d902 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/PreparedInput.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/PreparedInput.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.ranking; import com.yahoo.component.annotation.Inject; diff --git a/container-search/src/main/java/com/yahoo/search/ranking/RangeAdjuster.java b/container-search/src/main/java/com/yahoo/search/ranking/RangeAdjuster.java index 6881eece620..3f784a34d81 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/RangeAdjuster.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/RangeAdjuster.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.ranking; // scale and adjust the score according to the range diff --git a/container-search/src/main/java/com/yahoo/search/ranking/WrappedHit.java b/container-search/src/main/java/com/yahoo/search/ranking/WrappedHit.java index 7c33b836e33..1b1313eae98 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/WrappedHit.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/WrappedHit.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.ranking; import com.yahoo.search.result.FeatureData; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/AcceptedCountries.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/AcceptedCountries.java index 082eaac7315..931251226f9 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/AcceptedCountries.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/AcceptedCountries.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.billing; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/MailTemplating.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/MailTemplating.java index 642840bf2b3..c54791f511e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/MailTemplating.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/MailTemplating.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.notification; import com.yahoo.config.provision.TenantName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TrialNotifications.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TrialNotifications.java index cf6923b1e2c..4ea4fe79e8f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TrialNotifications.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TrialNotifications.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/EndpointConfig.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/EndpointConfig.java index 555fd024e47..1d5bf5e6aa2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/EndpointConfig.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/EndpointConfig.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.routing; /** diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp index 9f8c38a720e..95ad4bd143b 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp @@ -6,6 +6,7 @@ #include <vespa/searchcore/proton/test/documentdb_config_builder.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/config-attributes.h> +#include <ostream> using namespace document; using namespace proton; @@ -17,6 +18,25 @@ using std::make_shared; using DDBCSP = shared_ptr<DocumentDBConfig>; +namespace vespa::config::search::internal { + +std::ostream& operator<<(std::ostream& os, const AttributesConfig::Attribute::Match match) { + os << AttributesConfig::Attribute::getMatchName(match); + return os; +} + +std::ostream& operator<<(std::ostream& os, const AttributesConfig::Attribute::Dictionary::Match match) { + os << AttributesConfig::Attribute::Dictionary::getMatchName(match); + return os; +} + +std::ostream& operator<<(std::ostream& os, const AttributesConfig::Attribute::Dictionary::Type type) { + os << AttributesConfig::Attribute::Dictionary::getTypeName(type); + return os; +} + +} + namespace { @@ -49,6 +69,44 @@ assertDefaultAttribute(const AttributesConfig::Attribute &attribute, return true; } +bool +assert_string_attribute(const AttributesConfig::Attribute& attribute, + const vespalib::string& name, + std::optional<bool> uncased, std::optional<AttributesConfig::Attribute::Dictionary::Type> dictionary_type) +{ + using Attribute = AttributesConfig::Attribute; + using Dictionary = Attribute::Dictionary; + using Match = Attribute::Match; + if (!assertDefaultAttribute(attribute, name)) { + return false; + } + if (!EXPECT_EQUAL(name, attribute.name)) { + return false; + } + if (uncased.has_value()) { + if (uncased.value()) { + if (!EXPECT_EQUAL(Match::UNCASED, attribute.match)) { + return false; + } + if (!EXPECT_EQUAL(Dictionary::Match::UNCASED, attribute.dictionary.match)) { + return false; + } + } else { + if (!EXPECT_EQUAL(Match::CASED, attribute.match)) { + return false; + } + if (!EXPECT_EQUAL(Dictionary::Match::CASED, attribute.dictionary.match)) { + return false; + } + } + } + if (dictionary_type.has_value()) { + if (!EXPECT_EQUAL(dictionary_type.value(), attribute.dictionary.type)) { + return false; + } + } + return true; +} bool assertFastSearchAttribute(const AttributesConfig::Attribute &attribute, @@ -114,7 +172,7 @@ assertTensorAttribute(const AttributesConfig::Attribute &attribute, bool assertAttributes(const AttributesConfig::AttributeVector &attributes) { - if (!EXPECT_EQUAL(6u, attributes.size())) { + if (!EXPECT_EQUAL(8u, attributes.size())) { return false; } if (!assertDefaultAttribute(attributes[0], "a1")) { @@ -134,7 +192,13 @@ assertAttributes(const AttributesConfig::AttributeVector &attributes) } if (!assertTensorAttribute(attributes[5], "tensor2", "tensor(x[100])", 16)) { return false; - } + } + if (!assert_string_attribute(attributes[6], "string1", std::nullopt, AttributesConfig::Attribute::Dictionary::Type::BTREE)) { + return false; + } + if (!assert_string_attribute(attributes[7], "string2", true, std::nullopt)) { + return false; + } return true; } @@ -142,7 +206,7 @@ assertAttributes(const AttributesConfig::AttributeVector &attributes) bool assertLiveAttributes(const AttributesConfig::AttributeVector &attributes) { - if (!EXPECT_EQUAL(7u, attributes.size())) { + if (!EXPECT_EQUAL(9u, attributes.size())) { return false; } if (!assertFastSearchAttribute(attributes[0], "a0")) { @@ -166,6 +230,12 @@ assertLiveAttributes(const AttributesConfig::AttributeVector &attributes) if (!assertTensorAttribute(attributes[6], "tensor2", "tensor(x[200])", 32)) { return false; } + if (!assert_string_attribute(attributes[7], "string1", std::nullopt, AttributesConfig::Attribute::Dictionary::Type::HASH)) { + return false; + } + if (!assert_string_attribute(attributes[8], "string2", false, std::nullopt)) { + return false; + } return true; } @@ -173,7 +243,7 @@ assertLiveAttributes(const AttributesConfig::AttributeVector &attributes) bool assertScoutedAttributes(const AttributesConfig::AttributeVector &attributes) { - if (!EXPECT_EQUAL(6u, attributes.size())) { + if (!EXPECT_EQUAL(8u, attributes.size())) { return false; } if (!assertFastSearchAndMoreAttribute(attributes[0], "a1")) { @@ -194,6 +264,12 @@ assertScoutedAttributes(const AttributesConfig::AttributeVector &attributes) if (!assertTensorAttribute(attributes[5], "tensor2", "tensor(x[100])", 16)) { return false; } + if (!assert_string_attribute(attributes[6], "string1", std::nullopt, AttributesConfig::Attribute::Dictionary::Type::HASH)) { + return false; + } + if (!assert_string_attribute(attributes[7], "string2", false, std::nullopt)) { + return false; + } return true; } @@ -206,6 +282,30 @@ setupDefaultAttribute(const vespalib::string & name) return attribute; } +AttributesConfig::Attribute +setup_string_attribute(const vespalib::string& name, std::optional<bool> uncased, std::optional<AttributesConfig::Attribute::Dictionary::Type> dictionary_type) +{ + using Attribute = AttributesConfig::Attribute; + using Datatype = Attribute::Datatype; + using Dictionary = Attribute::Dictionary; + using Match = Attribute::Match; + Attribute attribute; + attribute.name = name; + attribute.datatype = Datatype::STRING; + if (uncased.has_value()) { + if (uncased.value()) { + attribute.match = Match::UNCASED; + attribute.dictionary.match = Dictionary::Match::UNCASED; + } else { + attribute.match = Match::CASED; + attribute.dictionary.match = Dictionary::Match::CASED; + } + } + if (dictionary_type.has_value()) { + attribute.dictionary.type = dictionary_type.value(); + } + return attribute; +} AttributesConfig::Attribute setupFastSearchAttribute(const vespalib::string & name) @@ -249,6 +349,8 @@ setupDefaultAttributes(AttributesConfigBuilder::AttributeVector &attributes) attributes.push_back(setupDefaultAttribute("a4")); attributes.push_back(setupTensorAttribute("tensor1", "tensor(x[100])", 16)); attributes.push_back(setupTensorAttribute("tensor2", "tensor(x[100])", 16)); + attributes.push_back(setup_string_attribute("string1", std::nullopt, AttributesConfig::Attribute::Dictionary::Type::BTREE)); + attributes.push_back(setup_string_attribute("string2", true, std::nullopt)); } @@ -265,6 +367,8 @@ setupLiveAttributes(AttributesConfigBuilder::AttributeVector &attributes) attributes.back().createifnonexistent = true; attributes.push_back(setupTensorAttribute("tensor1", "tensor(x[100])", 32)); attributes.push_back(setupTensorAttribute("tensor2", "tensor(x[200])", 32)); + attributes.push_back(setup_string_attribute("string1", std::nullopt, AttributesConfig::Attribute::Dictionary::Type::HASH)); + attributes.push_back(setup_string_attribute("string2", false, std::nullopt)); } } diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp index e4335740343..984af9870a6 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp @@ -56,6 +56,22 @@ TEST_F(JobTest, job_returns_false_when_multiple_move_operations_or_compaction_ar assertJobContext(4, 7, 3, 7, 1); } +TEST_F(JobTest, job_document_is_not_moved_if_meta_has_changed) +{ + setupThreeDocumentsToCompact(); + EXPECT_FALSE(run()); + assertJobContext(2, 9, 1, 0, 0); + auto orig_ts = _handler->_docs[8].first.timestamp; + _handler->_docs[8].first.timestamp = 0; + EXPECT_FALSE(run()); + assertJobContext(2, 9, 1, 0, 0); + _handler->_docs[8].first.timestamp = orig_ts; + EXPECT_FALSE(run()); + assertJobContext(3, 7, 2, 0, 0); + endScan().compact(); + assertJobContext(3, 7, 2, 8, 1); +} + TEST_F(JobTest, job_can_restart_documents_scan_if_lid_bloat_is_still_to_large) { init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR); diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp index a3e1fad46aa..d4d2a6dc377 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp @@ -122,11 +122,8 @@ JobTestBase::notifyNodeRetired(bool nodeRetired) { } void -JobTestBase::assertJobContext(uint32_t moveToLid, - uint32_t moveFromLid, - uint32_t handleMoveCnt, - uint32_t wantedLidLimit, - uint32_t compactStoreCnt) const +JobTestBase::assertJobContext(uint32_t moveToLid, uint32_t moveFromLid, uint32_t handleMoveCnt, + uint32_t wantedLidLimit, uint32_t compactStoreCnt) const { sync(); EXPECT_EQ(moveToLid, _handler->_moveToLid); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp index 0c8262cd124..f44169e8921 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp @@ -32,6 +32,9 @@ AttributesConfigScout::adjust(AttributesConfig::Attribute &attr, attr.densepostinglistthreshold = liveAttr.densepostinglistthreshold; attr.distancemetric = liveAttr.distancemetric; attr.index = liveAttr.index; + attr.match = liveAttr.match; + attr.dictionary.match = liveAttr.dictionary.match; + attr.dictionary.type = liveAttr.dictionary.type; } diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp index 7345a07cd27..de583c0b36d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp @@ -233,7 +233,7 @@ CompactionJob::run() if (shouldRestartScanDocuments(stats)) { _scanItr = _handler->getIterator(); } else { - _scanItr = IDocumentScanIterator::UP(); + _scanItr.reset(); _shouldCompactLidSpace = true; return false; } |