diff options
8 files changed, 61 insertions, 13 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 92975e47f75..9560065ee9b 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -118,6 +118,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"bjorncs", "baldersheim"}) default boolean enableJdiscPreshutdownCommand() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default boolean avoidRenamingSummaryFeatures() { return false; } @ModelFeatureFlag(owners = {"arnej"}) default boolean experimentalSdParsing() { return false; } + @ModelFeatureFlag(owners = {"balder"}) default boolean enableBitVectors() { return false; } @ModelFeatureFlag(owners = {"hmusum"}) default Architecture adminClusterArchitecture() { return Architecture.getDefault(); } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 4ef9c8b366c..9e94949c89b 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -77,6 +77,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private List<String> environmentVariables = List.of(); private boolean avoidRenamingSummaryFeatures = true; private boolean experimentalSdParsing = true; + private boolean enableBitVectors = true; private Architecture adminClusterNodeResourcesArchitecture = Architecture.getDefault(); @Override public ModelContext.FeatureFlags featureFlags() { return this; } @@ -133,6 +134,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public List<String> environmentVariables() { return environmentVariables; } @Override public boolean avoidRenamingSummaryFeatures() { return this.avoidRenamingSummaryFeatures; } @Override public boolean experimentalSdParsing() { return this.experimentalSdParsing; } + @Override public boolean enableBitVectors() { return this.enableBitVectors; } @Override public Architecture adminClusterArchitecture() { return adminClusterNodeResourcesArchitecture; } public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) { @@ -354,6 +356,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setEnableBitVectors(boolean value) { + this.enableBitVectors = value; + return this; + } + public TestProperties setAdminClusterNodeResourcesArchitecture(Architecture architecture) { this.adminClusterNodeResourcesArchitecture = architecture; return this; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index 30cb0d9c07d..cd10263b0ff 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -2,7 +2,6 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.subscription.ConfigInstanceUtil; -import com.yahoo.document.DataType; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.Case; @@ -196,7 +195,7 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce @Override public void getConfig(AttributesConfig.Builder builder) { //TODO This is just to get some exporting tests to work, Should be undone and removed - getConfig(builder, FieldSet.ALL, 77777); + getConfig(builder, FieldSet.ALL, 77777, false); } private boolean isAttributeInFieldSet(Attribute attribute, FieldSet fs) { @@ -302,11 +301,14 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce return AttributesConfig.Attribute.Match.UNCASED; } - public void getConfig(AttributesConfig.Builder builder, FieldSet fs, long maxUnCommittedMemory) { + public void getConfig(AttributesConfig.Builder builder, FieldSet fs, long maxUnCommittedMemory, boolean enableBitVectors) { for (Attribute attribute : attributes.values()) { if (isAttributeInFieldSet(attribute, fs)) { AttributesConfig.Attribute.Builder attrBuilder = getConfig(attribute.getName(), attribute, false); attrBuilder.maxuncommittedmemory(maxUnCommittedMemory); + if (enableBitVectors && attribute.isFastSearch()) { + attrBuilder.enablebitvectors(true); + } builder.attribute(attrBuilder); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java index 48bc972f7ef..77d742c63e9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java @@ -46,6 +46,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer { private ImportedFields importedFields; private final QueryProfileRegistry queryProfiles; private final long maxUncommittedMemory; + private final boolean enableBitVectors; /** * Creates a complete derived configuration from a search definition. @@ -86,6 +87,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer { this.schema = schema; this.queryProfiles = queryProfiles; this.maxUncommittedMemory = deployProperties.featureFlags().maxUnCommittedMemory(); + this.enableBitVectors = deployProperties.featureFlags().enableBitVectors(); if ( ! schema.isDocumentsOnly()) { streamingFields = new VsmFields(schema); streamingSummary = new VsmSummary(schema); @@ -145,10 +147,8 @@ public class DerivedConfiguration implements AttributesConfig.Producer { Writer writer = null; try { writer = IOUtils.createWriter(fileName, false); - if (writer != null) { - writer.write(instance.toString()); - writer.write("\n"); - } + writer.write(instance.toString()); + writer.write("\n"); } finally { if (writer != null) { IOUtils.closeWriter(writer); @@ -170,7 +170,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer { } public void getConfig(AttributesConfig.Builder builder, AttributeFields.FieldSet fs) { - attributeFields.getConfig(builder, fs, maxUncommittedMemory); + attributeFields.getConfig(builder, fs, maxUncommittedMemory, enableBitVectors); } public IndexingScript getIndexingScript() { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java index 2a58618e11f..92d72677086 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java @@ -220,10 +220,9 @@ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { @Test public void requireThatMutableConfigIsProperlyPropagated() throws ParseException { - AttributeFields attributes = new AttributeFields(getSearchWithMutables()); AttributesConfig.Builder builder = new AttributesConfig.Builder(); - attributes.getConfig(builder, AttributeFields.FieldSet.ALL, 13333); + attributes.getConfig(builder, AttributeFields.FieldSet.ALL, 13333, true); AttributesConfig cfg = builder.build(); assertEquals("a", cfg.attribute().get(0).name()); assertFalse(cfg.attribute().get(0).ismutable()); @@ -237,10 +236,9 @@ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { @Test public void requireMaxUnCommittedMemoryIsProperlyPropagated() throws ParseException { - AttributeFields attributes = new AttributeFields(getSearchWithMutables()); AttributesConfig.Builder builder = new AttributesConfig.Builder(); - attributes.getConfig(builder, AttributeFields.FieldSet.ALL, 13333); + attributes.getConfig(builder, AttributeFields.FieldSet.ALL, 13333, true); AttributesConfig cfg = builder.build(); assertEquals("a", cfg.attribute().get(0).name()); assertEquals(13333, cfg.attribute().get(0).maxuncommittedmemory()); @@ -252,6 +250,36 @@ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { assertEquals(13333, cfg.attribute().get(2).maxuncommittedmemory()); } + private void verifyEnableBitVectorDefault(Schema schema, boolean enableBitVectors) { + AttributeFields attributes = new AttributeFields(schema); + AttributesConfig.Builder builder = new AttributesConfig.Builder(); + attributes.getConfig(builder, AttributeFields.FieldSet.ALL, 13333, enableBitVectors); + AttributesConfig cfg = builder.build(); + assertEquals("a", cfg.attribute().get(0).name()); + assertEquals(enableBitVectors, cfg.attribute().get(0).enablebitvectors()); + + assertEquals("b", cfg.attribute().get(1).name()); + assertFalse(cfg.attribute().get(1).enablebitvectors()); + } + + @Test + public void requireEnableBitVectorsIsProperlyPropagated() throws ParseException { + Schema schema = getSchema( + "search test {\n" + + " document test { \n" + + " field a type int { \n" + + " indexing: attribute \n" + + " attribute: fast-search\n" + + " }\n" + + " field b type int { \n" + + " indexing: attribute \n" + + " }\n" + + " }\n" + + "}\n"); + verifyEnableBitVectorDefault(schema, false); + verifyEnableBitVectorDefault(schema, true); + } + @Test public void requireThatMutableIsAllowedThroughIndexing() throws ParseException { IndexingScript script = new IndexingScript(getSearchWithMutables()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java index a17992fad18..c0124ca0386 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java @@ -26,7 +26,7 @@ public class DictionaryTestCase { private static AttributesConfig getConfig(Schema schema) { AttributeFields attributes = new AttributeFields(schema); AttributesConfig.Builder builder = new AttributesConfig.Builder(); - attributes.getConfig(builder, AttributeFields.FieldSet.ALL, 130000); + attributes.getConfig(builder, AttributeFields.FieldSet.ALL, 130000, true); return builder.build(); } private Schema createSearch(String def) throws ParseException { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index b59dbda3174..588f8b98537 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -211,6 +211,7 @@ public class ModelContextImpl implements ModelContext { private final boolean useQrserverServiceName; private final boolean avoidRenamingSummaryFeatures; private final boolean experimentalSdParsing; + private final boolean enableBitVectors; private final Architecture adminClusterArchitecture; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { @@ -260,6 +261,7 @@ public class ModelContextImpl implements ModelContext { this.useQrserverServiceName = flagValue(source, appId, version, Flags.USE_QRSERVER_SERVICE_NAME); this.avoidRenamingSummaryFeatures = flagValue(source, appId, version, Flags.AVOID_RENAMING_SUMMARY_FEATURES); this.experimentalSdParsing = flagValue(source, appId, version, Flags.EXPERIMENTAL_SD_PARSING); + this.enableBitVectors = flagValue(source, appId, version, Flags.ENABLE_BIT_VECTORS); this.adminClusterArchitecture = Architecture.valueOf(flagValue(source, appId, version, PermanentFlags.ADMIN_CLUSTER_NODE_ARCHITECTURE)); } @@ -311,6 +313,7 @@ public class ModelContextImpl implements ModelContext { @Override public boolean useQrserverServiceName() { return useQrserverServiceName; } @Override public boolean avoidRenamingSummaryFeatures() { return avoidRenamingSummaryFeatures; } @Override public boolean experimentalSdParsing() { return experimentalSdParsing; } + @Override public boolean enableBitVectors() { return this.enableBitVectors; } @Override public Architecture adminClusterArchitecture() { return adminClusterArchitecture; } private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 30c065d2546..50b5d2e079b 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -383,6 +383,13 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag ENABLE_BIT_VECTORS = defineFeatureFlag( + "enable-bit-vectors", false, + List.of("arnej"), "2022-05-03", "2022-12-31", + "Enables bit vector by default for fast-search attributes", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag USE_ZSTD_IN_FILE_DISTRIBUTION = defineFeatureFlag( "use-zstd-in-file-distribution", false, List.of("hmusum"), "2022-04-07", "2022-06-01", |