Merge pull request #16188 from vespa-engine/balder/control-lidspace-job

Control which lidspace job implementation to use.

6 files changed, 50 insertions, 10 deletions

diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
index d020079bb17..e75038a863e 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
@@ -80,6 +80,7 @@ public interface ModelContext {
         @ModelFeatureFlag(owners = {"baldersheim"}) default int contentNodeBucketDBStripeBits() { throw new UnsupportedOperationException("TODO specify default value"); }
         @ModelFeatureFlag(owners = {"baldersheim"}) default int mergeChunkSize() { throw new UnsupportedOperationException("TODO specify default value"); }
         @ModelFeatureFlag(owners = {"baldersheim"}) default double feedConcurrency() { throw new UnsupportedOperationException("TODO specify default value"); }
+        @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useBucketExecutorForLidSpaceCompact() { throw new UnsupportedOperationException("TODO specify default value"); }
         @ModelFeatureFlag(owners = {"musum", "mpolden"}, comment = "Revisit in February 2021") default boolean reconfigurableZookeeperServer() { return false; }
         @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default boolean enableJdiscConnectionLog() { return false; }
         @ModelFeatureFlag(owners = {"bjorncs", "tokle", "baldersheim"}) default boolean enableZstdCompressionAccessLog() { return false; }
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index 9e8e9656e90..92d425a9d4e 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -26,7 +26,6 @@ import java.util.Set;
  *
  * @author hakonhall
  */
-@SuppressWarnings("deprecation")
 public class TestProperties implements ModelContext.Properties, ModelContext.FeatureFlags {
 
     private boolean multitenant = false;
@@ -53,6 +52,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     private double feedConcurrency = 0.5;
     private boolean enableAutomaticReindexing = false;
     private boolean reconfigurableZookeeperServer = false;
+    private boolean useBucketExecutorForLidSpaceCompact;
 
     @Override public ModelContext.FeatureFlags featureFlags() { return this; }
     @Override public boolean multitenant() { return multitenant; }
@@ -87,6 +87,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     @Override public double feedConcurrency() { return feedConcurrency; }
     @Override public boolean enableAutomaticReindexing() { return enableAutomaticReindexing; }
     @Override public boolean reconfigurableZookeeperServer() { return reconfigurableZookeeperServer; }
+    @Override public boolean useBucketExecutorForLidSpaceCompact() { return useBucketExecutorForLidSpaceCompact; }
 
     public TestProperties setFeedConcurrency(double feedConcurrency) {
         this.feedConcurrency = feedConcurrency;
@@ -195,6 +196,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
         return this;
     }
 
+    public TestProperties useBucketExecutorForLidSpaceCompact(boolean enabled) {
+        useBucketExecutorForLidSpaceCompact = enabled;
+        return this;
+    }
+
 
     public static class Spec implements ConfigServerSpec {
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
index 79b98837238..dd29df61f35 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
@@ -65,6 +65,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster>
     private Optional<ResourceLimits> resourceLimits = Optional.empty();
     private final ProtonConfig.Indexing.Optimize.Enum feedSequencerType;
     private final double defaultFeedConcurrency;
+    private final boolean useBucketExecutorForLidSpaceCompact;
 
     /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */
     private final boolean combined;
@@ -205,6 +206,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster>
         this.combined = combined;
         feedSequencerType = convertFeedSequencerType(featureFlags.feedSequencerType());
         defaultFeedConcurrency = featureFlags.feedConcurrency();
+        useBucketExecutorForLidSpaceCompact = featureFlags.useBucketExecutorForLidSpaceCompact();
     }
 
     public void setVisibilityDelay(double delay) {
@@ -425,6 +427,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster>
         } else {
             builder.indexing.optimize(feedSequencerType);
         }
+        builder.lidspacecompaction.usebucketexecutor(useBucketExecutorForLidSpaceCompact);
     }
 
     private boolean isGloballyDistributed(NewDocumentType docType) {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
index 00b3fc22956..55234a3a96e 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
@@ -719,6 +719,17 @@ public class ContentBuilderTest extends DomBuilderTest {
         assertThat(config.search().mmap().options(0), is(ProtonConfig.Search.Mmap.Options.POPULATE));
     }
 
+    private String singleNodeContentXml() {
+        return  "<services>" +
+                "<content version='1.0' id='search'>" +
+                "  <redundancy>1</redundancy>" +
+                "  <documents>" +
+                "    <document type='music' mode='index'/>" +
+                "  </documents>" +
+                "  <nodes count='1'/>" +
+                "</content>" +
+                "</services>";
+    }
     @Test
     public void ensurePruneRemovedDocumentsAgeForHostedVespa() {
         {
@@ -738,15 +749,7 @@ public class ContentBuilderTest extends DomBuilderTest {
         }
 
         {
-            String hostedXml = "<services>" +
-                    "<content version='1.0' id='search'>" +
-                    "  <redundancy>1</redundancy>" +
-                    "  <documents>" +
-                    "    <document type='music' mode='index'/>" +
-                    "  </documents>" +
-                    "  <nodes count='1'/>" +
-                    "</content>" +
-                    "</services>";
+            String hostedXml = singleNodeContentXml();
 
             DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().setHostedVespa(true));
             VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder()
@@ -821,6 +824,23 @@ public class ContentBuilderTest extends DomBuilderTest {
         verifyThatFeatureFlagControlsVisibilityDelayDefault(0.6, 0.6);
     }
 
+    private void verifyThatFeatureFlagControlsUseBucketExecutorForLidSpaceCompact(boolean flag) {
+        DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().useBucketExecutorForLidSpaceCompact(flag));
+        VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder()
+                .withServices(singleNodeContentXml())
+                .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION)
+                .build())
+                .create(deployStateBuilder);
+        ProtonConfig config = getProtonConfig(model.getContentClusters().values().iterator().next());
+        assertEquals(flag, config.lidspacecompaction().usebucketexecutor());
+    }
+
+    @Test
+    public void verifyUseBucketExecutorForLidSpaceCompact() {
+        verifyThatFeatureFlagControlsUseBucketExecutorForLidSpaceCompact(true);
+        verifyThatFeatureFlagControlsUseBucketExecutorForLidSpaceCompact(false);
+    }
+
     @Test
     public void failWhenNoDocumentsElementSpecified() {
         expectedException.expect(IllegalArgumentException.class);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index 596f9c0ce98..e1642efcd44 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -165,6 +165,7 @@ public class ModelContextImpl implements ModelContext {
         private final boolean reconfigurableZookeeperServer;
         private final boolean enableJdiscConnectionLog;
         private final boolean enableZstdCompressionAccessLog;
+        private final boolean useBucketExecutorForLidSpaceCompact;
 
         public FeatureFlags(FlagSource source, ApplicationId appId) {
             this.enableAutomaticReindexing = flagValue(source, appId, Flags.ENABLE_AUTOMATIC_REINDEXING);
@@ -185,6 +186,7 @@ public class ModelContextImpl implements ModelContext {
             this.reconfigurableZookeeperServer = flagValue(source, appId, Flags.RECONFIGURABLE_ZOOKEEPER_SERVER_FOR_CLUSTER_CONTROLLER);
             this.enableJdiscConnectionLog = flagValue(source, appId, Flags.ENABLE_JDISC_CONNECTION_LOG);
             this.enableZstdCompressionAccessLog = flagValue(source, appId, Flags.ENABLE_ZSTD_COMPRESSION_ACCESS_LOG);
+            this.useBucketExecutorForLidSpaceCompact = flagValue(source, appId, Flags.USE_BUCKET_EXECUTOR_FOR_LID_SPACE_COMPACT);
         }
 
         @Override public boolean enableAutomaticReindexing() { return enableAutomaticReindexing; }
@@ -205,6 +207,7 @@ public class ModelContextImpl implements ModelContext {
         @Override public boolean reconfigurableZookeeperServer() { return reconfigurableZookeeperServer; }
         @Override public boolean enableJdiscConnectionLog() { return enableJdiscConnectionLog; }
         @Override public boolean enableZstdCompressionAccessLog() { return enableZstdCompressionAccessLog; }
+        @Override public boolean useBucketExecutorForLidSpaceCompact() { return useBucketExecutorForLidSpaceCompact; }
 
         private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) {
             return flag.bindTo(source)
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index 6f7970e1357..7c2deeda69b 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -271,6 +271,13 @@ public class Flags {
             "Takes effect on (re)deployment",
             APPLICATION_ID);
 
+    public static final UnboundBooleanFlag USE_BUCKET_EXECUTOR_FOR_LID_SPACE_COMPACT = defineFeatureFlag(
+            "use-bucket-executor-for-lid-space-compact", false,
+            List.of("baldersheim"), "2021-01-24", "2021-03-01",
+            "Wheter to use content-level bucket executor or legacy frozen buckets",
+            "Takes effect on next internal redeployment",
+            APPLICATION_ID);
+
     public static final UnboundBooleanFlag USE_POWER_OF_TWO_CHOICES_LOAD_BALANCING = defineFeatureFlag(
             "use-power-of-two-choices-load-balancing", false,
             List.of("tokle"), "2020-12-02", "2021-02-01",