diff options
Diffstat (limited to 'config-model/src')
11 files changed, 129 insertions, 18 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index c9ddcdf38eb..0cde8c361a7 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -72,6 +72,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private String mergeThrottlingPolicy = "STATIC"; private double persistenceThrottlingWsDecrementFactor = 1.2; private double persistenceThrottlingWsBackoff = 0.95; + private boolean inhibitDefaultMergesWhenGlobalMergesPending = false; private boolean useV8GeoPositions = false; private List<String> environmentVariables = List.of(); @@ -126,6 +127,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; } @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; } + @Override public boolean inhibitDefaultMergesWhenGlobalMergesPending() { return inhibitDefaultMergesWhenGlobalMergesPending; } @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } @Override public List<String> environmentVariables() { return environmentVariables; } @@ -330,6 +332,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties inhibitDefaultMergesWhenGlobalMergesPending(boolean value) { + this.inhibitDefaultMergesWhenGlobalMergesPending = value; + return this; + } + public TestProperties setUseV8GeoPositions(boolean value) { this.useV8GeoPositions = value; return this; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java index 9d933b8439d..51defffa00b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java @@ -26,11 +26,6 @@ public class AddExtraFieldsToDocument extends Processor { super(schema, deployLogger, rankProfileRegistry, queryProfiles); } - //TODO This is a tempoarry hack to avoid producing illegal code for fields not wanted anyway. - private boolean dirtyLegalFieldNameCheck(String fieldName) { - return ! fieldName.contains(".") && !"rankfeatures".equals(fieldName) && !"summaryfeatures".equals(fieldName); - } - @Override public void process(boolean validate, boolean documentsOnly) { SDDocumentType document = schema.getDocument(); @@ -38,10 +33,21 @@ public class AddExtraFieldsToDocument extends Processor { for (SDField field : schema.extraFieldList()) { addSdField(schema, document, field, validate); } - //TODO Vespa 8 or sooner we should avoid the dirty addition of fields from dirty 'default' summary to document at all - for (SummaryField field : schema.getSummary("default").getSummaryFields().values()) { - if (dirtyLegalFieldNameCheck(field.getName())) { - addSummaryField(schema, document, field, validate); + for (var docsum : schema.getSummaries().values()) { + for (var summaryField : docsum.getSummaryFields().values()) { + switch (summaryField.getTransform()) { + case NONE: + case BOLDED: + case DYNAMICBOLDED: + case DYNAMICTEASER: + case TEXTEXTRACTOR: + addSummaryField(schema, document, summaryField, validate); + break; + default: + // skip: generated from attribute or similar, + // so does not need to be included as an extra + // field in the document type + } } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index b8d2a4f91fe..3f01f5610f1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -43,6 +43,7 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl private final boolean useThreePhaseUpdates; private final int maxActivationInhibitedOutOfSyncGroups; private final boolean unorderedMergeChaining; + private final boolean inhibitDefaultMergesWhenGlobalMergesPending; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<DistributorCluster> { @@ -106,11 +107,12 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl boolean useThreePhaseUpdates = deployState.getProperties().featureFlags().useThreePhaseUpdates(); int maxInhibitedGroups = deployState.getProperties().featureFlags().maxActivationInhibitedOutOfSyncGroups(); boolean unorderedMergeChaining = deployState.getProperties().featureFlags().unorderedMergeChaining(); + boolean inhibitDefaultMerges = deployState.getProperties().featureFlags().inhibitDefaultMergesWhenGlobalMergesPending(); return new DistributorCluster(parent, new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc, hasIndexedDocumentType, useThreePhaseUpdates, - maxInhibitedGroups, unorderedMergeChaining); + maxInhibitedGroups, unorderedMergeChaining, inhibitDefaultMerges); } } @@ -118,7 +120,8 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl GcOptions gc, boolean hasIndexedDocumentType, boolean useThreePhaseUpdates, int maxActivationInhibitedOutOfSyncGroups, - boolean unorderedMergeChaining) + boolean unorderedMergeChaining, + boolean inhibitDefaultMergesWhenGlobalMergesPending) { super(parent, "distributor"); this.parent = parent; @@ -128,6 +131,7 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl this.useThreePhaseUpdates = useThreePhaseUpdates; this.maxActivationInhibitedOutOfSyncGroups = maxActivationInhibitedOutOfSyncGroups; this.unorderedMergeChaining = unorderedMergeChaining; + this.inhibitDefaultMergesWhenGlobalMergesPending = inhibitDefaultMergesWhenGlobalMergesPending; } @Override @@ -142,6 +146,7 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl builder.enable_metadata_only_fetch_phase_for_inconsistent_updates(useThreePhaseUpdates); builder.max_activation_inhibited_out_of_sync_groups(maxActivationInhibitedOutOfSyncGroups); builder.use_unordered_merge_chaining(unorderedMergeChaining); + builder.inhibit_default_merges_when_global_merges_pending(inhibitDefaultMergesWhenGlobalMergesPending); bucketSplitting.getConfig(builder); } diff --git a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg index 64d4bd3ba0a..5434b0770f7 100644 --- a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg +++ b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg @@ -12,14 +12,16 @@ ilscript[].docfield[7] "f" ilscript[].docfield[8] "g" ilscript[].docfield[9] "h" ilscript[].docfield[10] "loc" +ilscript[].docfield[11] "mytags" ilscript[].content[0] "clear_state | guard { input loc | to_pos | zcurve | attribute loc_pos_zcurve; }" ilscript[].content[1] "clear_state | guard { input a | tokenize normalize stem:\"BEST\" | summary abolded2 | summary aboldeddynamic | summary adynamic2 | attribute a; }" ilscript[].content[2] "clear_state | guard { input adynamic | tokenize normalize stem:\"BEST\" | summary adynamic | attribute adynamic; }" ilscript[].content[3] "clear_state | guard { input abolded | tokenize normalize stem:\"BEST\" | summary abolded | attribute abolded; }" -ilscript[].content[4] "clear_state | guard { input b | summary b; }" +ilscript[].content[4] "clear_state | guard { input b | summary anotherb | summary b; }" ilscript[].content[5] "clear_state | guard { input c | summary c | attribute c; }" ilscript[].content[6] "clear_state | guard { input d | tokenize normalize stem:\"BEST\" | summary d; }" ilscript[].content[7] "clear_state | guard { input e | tokenize normalize stem:\"BEST\" | summary dynamice | summary e; }" ilscript[].content[8] "clear_state | guard { input f | summary f; }" ilscript[].content[9] "clear_state | guard { input g | summary g; }" ilscript[].content[10] "clear_state | guard { input h | summary h; }" +ilscript[].content[11] "clear_state | guard { input mytags | for_each { tokenize normalize stem:\"BEST\" } | index mytags; }" diff --git a/config-model/src/test/derived/multiplesummaries/index-info.cfg b/config-model/src/test/derived/multiplesummaries/index-info.cfg index 9c53a66549c..d5002535761 100644 --- a/config-model/src/test/derived/multiplesummaries/index-info.cfg +++ b/config-model/src/test/derived/multiplesummaries/index-info.cfg @@ -71,6 +71,20 @@ indexinfo[].command[].indexname "loc" indexinfo[].command[].command "index" indexinfo[].command[].indexname "loc" indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "stem:BEST" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "normalize" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "plain-tokens" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "type Array<string>" indexinfo[].command[].indexname "abolded2" indexinfo[].command[].command "index" indexinfo[].command[].indexname "abolded2" @@ -83,6 +97,16 @@ indexinfo[].command[].indexname "adynamic2" indexinfo[].command[].command "index" indexinfo[].command[].indexname "adynamic2" indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "alltags" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "alltags" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "alltags" +indexinfo[].command[].command "type Array<string>" +indexinfo[].command[].indexname "anotherb" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "anotherb" +indexinfo[].command[].command "type string" indexinfo[].command[].indexname "dynamice" indexinfo[].command[].command "index" indexinfo[].command[].indexname "dynamice" diff --git a/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd b/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd index a1454a8d8a4..ae0e2fe92bc 100644 --- a/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd +++ b/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd @@ -67,6 +67,10 @@ search multiplesummaries { field loc type string { } + + field mytags type array<string> { + indexing: index + } } field loc_pos type position { @@ -120,7 +124,7 @@ search multiplesummaries { } # Since a here is a dynamic summary, it will be fetched from disk - document-summary notattributesonly2 { + document-summary anothernotattributesonly2 { summary adynamic2 type string { # Should still be dynamic here source: a @@ -130,6 +134,19 @@ search multiplesummaries { summary c type string { } + summary alltags type array<string> { + source: mytags + } + summary sometags type array<string> { + source: mytags + matched-elements-only + } + summary anothera type string { + source: a + } + summary anotherb type string { + source: b + } } # Not attributes only because d is bolded diff --git a/config-model/src/test/derived/multiplesummaries/summary.cfg b/config-model/src/test/derived/multiplesummaries/summary.cfg index ec5e0610385..1c8fc47878b 100644 --- a/config-model/src/test/derived/multiplesummaries/summary.cfg +++ b/config-model/src/test/derived/multiplesummaries/summary.cfg @@ -1,6 +1,6 @@ -defaultsummaryid 2038247029 +defaultsummaryid 456145241 usev8geopositions false -classes[].id 2038247029 +classes[].id 456145241 classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "loc_pos" @@ -37,6 +37,12 @@ classes[].fields[].name "e" classes[].fields[].type "longstring" classes[].fields[].name "adynamic2" classes[].fields[].type "longstring" +classes[].fields[].name "alltags" +classes[].fields[].type "jsonstring" +classes[].fields[].name "sometags" +classes[].fields[].type "jsonstring" +classes[].fields[].name "anotherb" +classes[].fields[].type "longstring" classes[].fields[].name "abolded2" classes[].fields[].type "longstring" classes[].fields[].name "aboldeddynamic" @@ -86,13 +92,21 @@ classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" -classes[].id 1527097108 -classes[].name "notattributesonly2" +classes[].id 1609068631 +classes[].name "anothernotattributesonly2" classes[].omitsummaryfeatures false classes[].fields[].name "adynamic2" classes[].fields[].type "longstring" classes[].fields[].name "c" classes[].fields[].type "longstring" +classes[].fields[].name "alltags" +classes[].fields[].type "jsonstring" +classes[].fields[].name "sometags" +classes[].fields[].type "jsonstring" +classes[].fields[].name "anothera" +classes[].fields[].type "longstring" +classes[].fields[].name "anotherb" +classes[].fields[].type "longstring" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" diff --git a/config-model/src/test/derived/multiplesummaries/summarymap.cfg b/config-model/src/test/derived/multiplesummaries/summarymap.cfg index adf0770a835..94adc250c54 100644 --- a/config-model/src/test/derived/multiplesummaries/summarymap.cfg +++ b/config-model/src/test/derived/multiplesummaries/summarymap.cfg @@ -20,6 +20,12 @@ override[].arguments "c" override[].field "adynamic2" override[].command "dynamicteaser" override[].arguments "adynamic2" +override[].field "sometags" +override[].command "matchedelementsfilter" +override[].arguments "mytags" +override[].field "anothera" +override[].command "attribute" +override[].arguments "a" override[].field "anotdynamic" override[].command "attribute" override[].arguments "adynamic" diff --git a/config-model/src/test/examples/nextgen/summaryfield.sd b/config-model/src/test/examples/nextgen/summaryfield.sd index 9b3cc6862b9..99c73d1be53 100644 --- a/config-model/src/test/examples/nextgen/summaryfield.sd +++ b/config-model/src/test/examples/nextgen/summaryfield.sd @@ -5,11 +5,21 @@ search summaryfield { indexing: index | summary summary bar: full } + field mytags type array<string> { + indexing: index + } } document-summary baz { summary cox type string { source: bar } + summary alltags type array<string> { + source: mytags + } + summary sometags type array<string> { + source: mytags + matched-elements-only + } } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java index b0b9ce81cc7..833a6effe4a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java @@ -38,7 +38,9 @@ public class ImplicitSchemaFieldsTestCase extends AbstractSchemaTestCase { assertNotNull(docType.getField("foo")); assertNotNull(docType.getField("bar")); assertNotNull(docType.getField("cox")); - assertEquals(3, docType.getFieldCount()); + assertNotNull(docType.getField("mytags")); + assertNotNull(docType.getField("alltags")); + assertEquals(5, docType.getFieldCount()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 88073d281c5..9f571167d8c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1159,6 +1159,24 @@ public class ContentClusterTest extends ContentBaseTest { } @Test + public void inhibit_default_merges_when_global_merges_pending_controlled_by_properties() throws Exception { + assertFalse(resolveInhibitDefaultMergesConfig(Optional.empty())); + assertFalse(resolveInhibitDefaultMergesConfig(Optional.of(false))); + assertTrue(resolveInhibitDefaultMergesConfig(Optional.of(true))); + } + + private boolean resolveInhibitDefaultMergesConfig(Optional<Boolean> inhibitDefaultMerges) throws Exception { + var props = new TestProperties(); + if (inhibitDefaultMerges.isPresent()) { + props.inhibitDefaultMergesWhenGlobalMergesPending(inhibitDefaultMerges.get()); + } + var cluster = createOneNodeCluster(props); + var builder = new StorDistributormanagerConfig.Builder(); + cluster.getDistributorNodes().getConfig(builder); + return (new StorDistributormanagerConfig(builder)).inhibit_default_merges_when_global_merges_pending(); + } + + @Test public void testDedicatedClusterControllers() { VespaModel noContentModel = createEnd2EndOneNode(new TestProperties().setHostedVespa(true) .setMultitenant(true), |