Add feature flag for new content layer metadata features

Exposed as an integer rather than a bool to account for future
additions without needing to add more feature flags to the mix.
In particular this is because those additions are expected to
require the other additions to be present, so it's not a mix
and match situation.

Only values 0 (legacy) and 1 (operation cancellation) map to
any underlying configs at this time, though any higher number
will transparently enable cancellation. Value 2 is documented
based on its intended future semantics.

6 files changed, 48 insertions, 3 deletions

diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
index bdad0dc4cee..e3d9a4f3335 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
@@ -116,6 +116,7 @@ public interface ModelContext {
         @ModelFeatureFlag(owners = {"mortent", "olaa"}) default boolean enableDataplaneProxy() { return false; }
         @ModelFeatureFlag(owners = {"baldersheim"}) default boolean enableNestedMultivalueGrouping() { return false; }
         @ModelFeatureFlag(owners = {"jonmv"}) default boolean useReconfigurableDispatcher() { return false; }
+        @ModelFeatureFlag(owners = {"vekterli"}) default int contentLayerMetadataFeatureLevel() { return 0; }
     }
 
     /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index 31e066cdd8c..c9a25e113a6 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -86,6 +86,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     private boolean allowUserFilters = true;
     private List<DataplaneToken> dataplaneTokens;
     private boolean enableDataplaneProxy;
+    private int contentLayerMetadataFeatureLevel = 0;
 
     @Override public ModelContext.FeatureFlags featureFlags() { return this; }
     @Override public boolean multitenant() { return multitenant; }
@@ -144,6 +145,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     @Override public boolean enableGlobalPhase() { return true; } // Enable global-phase by default for unit tests only
     @Override public List<DataplaneToken> dataplaneTokens() { return dataplaneTokens; }
     @Override public boolean enableDataplaneProxy() { return enableDataplaneProxy; }
+    @Override public int contentLayerMetadataFeatureLevel() { return contentLayerMetadataFeatureLevel; }
 
     public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) {
         this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim;
@@ -379,6 +381,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
         return this;
     }
 
+    public TestProperties setContentLayerMetadataFeatureLevel(int level) {
+        this.contentLayerMetadataFeatureLevel = level;
+        return this;
+    }
+
     public static class Spec implements ConfigServerSpec {
 
         private final String hostName;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java
index 8503eafa713..1d015e5e772 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java
@@ -34,6 +34,7 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen
     private final GcOptions gc;
     private final boolean hasIndexedDocumentType;
     private final int maxActivationInhibitedOutOfSyncGroups;
+    private final int contentLayerMetadataFeatureLevel;
     public static class Builder extends VespaDomBuilder.DomConfigProducerBuilderBase<DistributorCluster> {
 
         ContentCluster parent;
@@ -93,18 +94,22 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen
             final ModelElement documentsNode = clusterElement.child("documents");
             final GcOptions gc = parseGcOptions(documentsNode);
             final boolean hasIndexedDocumentType = clusterContainsIndexedDocumentType(documentsNode);
-            int maxInhibitedGroups = deployState.getProperties().featureFlags().maxActivationInhibitedOutOfSyncGroups();
+            var featureFlags = deployState.getProperties().featureFlags();
+            int maxInhibitedGroups = featureFlags.maxActivationInhibitedOutOfSyncGroups();
+            int contentLayerMetadataFeatureLevel = featureFlags.contentLayerMetadataFeatureLevel();
 
             return new DistributorCluster(parent,
                     new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc,
                     hasIndexedDocumentType,
-                    maxInhibitedGroups);
+                    maxInhibitedGroups,
+                    contentLayerMetadataFeatureLevel);
         }
     }
 
     private DistributorCluster(ContentCluster parent, BucketSplitting bucketSplitting,
                                GcOptions gc, boolean hasIndexedDocumentType,
-                               int maxActivationInhibitedOutOfSyncGroups)
+                               int maxActivationInhibitedOutOfSyncGroups,
+                               int contentLayerMetadataFeatureLevel)
     {
         super(parent, "distributor");
         this.parent = parent;
@@ -112,6 +117,7 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen
         this.gc = gc;
         this.hasIndexedDocumentType = hasIndexedDocumentType;
         this.maxActivationInhibitedOutOfSyncGroups = maxActivationInhibitedOutOfSyncGroups;
+        this.contentLayerMetadataFeatureLevel = contentLayerMetadataFeatureLevel;
     }
 
     @Override
@@ -125,6 +131,9 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen
         builder.disable_bucket_activation(!hasIndexedDocumentType);
         builder.max_activation_inhibited_out_of_sync_groups(maxActivationInhibitedOutOfSyncGroups);
         builder.enable_condition_probing(true);
+        if (contentLayerMetadataFeatureLevel > 0) {
+            builder.enable_operation_cancellation(true);
+        }
         bucketSplitting.getConfig(builder);
     }
 
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
index 2726d64eafc..c30b74ee533 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
@@ -1479,6 +1479,23 @@ public class ContentClusterTest extends ContentBaseTest {
         assertEquals(expectedGroupsAllowedDown, config.max_number_of_groups_allowed_to_be_down());
     }
 
+    private boolean resolveDistributorOperationCancellationConfig(Integer featureLevel) throws Exception {
+        var properties = new TestProperties();
+        if (featureLevel != null) {
+            properties.setContentLayerMetadataFeatureLevel(featureLevel);
+        }
+        var cfg = resolveStorDistributormanagerConfig(properties);
+        return cfg.enable_operation_cancellation();
+    }
+
+    @Test
+    void distributor_operation_cancelling_config_controlled_by_properties() throws Exception {
+        assertFalse(resolveDistributorOperationCancellationConfig(null)); // defaults to false
+        assertFalse(resolveDistributorOperationCancellationConfig(0));
+        assertTrue(resolveDistributorOperationCancellationConfig(1));
+        assertTrue(resolveDistributorOperationCancellationConfig(2));
+    }
+
     private String servicesWithGroups(int groupCount, double minGroupUpRatio) {
         String services = String.format("<?xml version='1.0' encoding='UTF-8' ?>" +
                 "<services version='1.0'>" +
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index ada82abf907..f08e5444b9e 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -201,6 +201,7 @@ public class ModelContextImpl implements ModelContext {
         private final boolean enableDataplaneProxy;
         private final boolean enableNestedMultivalueGrouping;
         private final boolean useReconfigurableDispatcher;
+        private final int contentLayerMetadataFeatureLevel;
 
         public FeatureFlags(FlagSource source, ApplicationId appId, Version version) {
             this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT);
@@ -243,6 +244,7 @@ public class ModelContextImpl implements ModelContext {
             this.enableDataplaneProxy = flagValue(source, appId, version, Flags.ENABLE_DATAPLANE_PROXY);
             this.enableNestedMultivalueGrouping = flagValue(source, appId, version, Flags.ENABLE_NESTED_MULTIVALUE_GROUPING);
             this.useReconfigurableDispatcher = flagValue(source, appId, version, Flags.USE_RECONFIGURABLE_DISPATCHER);
+            this.contentLayerMetadataFeatureLevel = flagValue(source, appId, version, Flags.CONTENT_LAYER_METADATA_FEATURE_LEVEL);
         }
 
         @Override public int heapSizePercentage() { return heapPercentage; }
@@ -293,6 +295,7 @@ public class ModelContextImpl implements ModelContext {
         @Override public boolean enableDataplaneProxy() { return enableDataplaneProxy; }
         @Override public boolean enableNestedMultivalueGrouping() { return enableNestedMultivalueGrouping; }
         @Override public boolean useReconfigurableDispatcher() { return useReconfigurableDispatcher; }
+        @Override public int contentLayerMetadataFeatureLevel() { return contentLayerMetadataFeatureLevel; }
 
         private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) {
             return flag.bindTo(source)
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index 8c3cf337ce3..947338c71e7 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -398,6 +398,14 @@ public class Flags {
             "Takes effect immediately",
             APPLICATION_ID);
 
+    public static final UnboundIntFlag CONTENT_LAYER_METADATA_FEATURE_LEVEL = defineIntFlag(
+            "content-layer-metadata-feature-level", 0,
+            List.of("vekterli"), "2022-09-12", "2024-02-01",
+            "Value semantics: 0) legacy behavior, 1) operation cancellation, 2) operation " +
+            "cancellation and ephemeral content node sequence numbers for bucket replicas",
+            "Takes effect at redeployment",
+            APPLICATION_ID);
+
     /** WARNING: public for testing: All flags should be defined in {@link Flags}. */
     public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners,
                                                        String createdAt, String expiresAt, String description,