Add large resource reduction validation check

5 files changed, 185 insertions, 1 deletions

diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json
index 25f35781a1f..6b466c65cdb 100644
--- a/config-model-api/abi-spec.json
+++ b/config-model-api/abi-spec.json
@@ -531,6 +531,7 @@
       "public static final enum com.yahoo.config.application.api.ValidationId indexModeChange",
       "public static final enum com.yahoo.config.application.api.ValidationId fieldTypeChange",
       "public static final enum com.yahoo.config.application.api.ValidationId clusterSizeReduction",
+      "public static final enum com.yahoo.config.application.api.ValidationId resourcesReduction",
       "public static final enum com.yahoo.config.application.api.ValidationId contentTypeRemoval",
       "public static final enum com.yahoo.config.application.api.ValidationId contentClusterRemoval",
       "public static final enum com.yahoo.config.application.api.ValidationId deploymentRemoval",
@@ -605,4 +606,4 @@
       "public static final com.yahoo.config.application.api.ValidationOverrides all"
     ]
   }
-}
+}
\ No newline at end of file
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationId.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationId.java
index 35ece71a72e..c0bae137b0d 100644
--- a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationId.java
+++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationId.java
@@ -14,6 +14,7 @@ public enum ValidationId {
     indexModeChange("indexing-mode-change"), // Changing the index mode (streaming, indexed, store-only) of documents 
     fieldTypeChange("field-type-change"), // Field type changes
     clusterSizeReduction("cluster-size-reduction"), // Large reductions in cluster size
+    resourcesReduction("resources-reduction"), // Large reductions in node resources
     contentTypeRemoval("content-type-removal"), // Removal of a data type (causes deletion of all data)
     contentClusterRemoval("content-cluster-removal"), // Removal (or id change) of content clusters
     deploymentRemoval("deployment-removal"), // Removal of production zones from deployment.xml
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
index 4959970d98e..8eabc61f71f 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
@@ -17,6 +17,7 @@ import com.yahoo.vespa.model.application.validation.change.ContentTypeRemovalVal
 import com.yahoo.vespa.model.application.validation.change.GlobalDocumentChangeValidator;
 import com.yahoo.vespa.model.application.validation.change.IndexedSearchClusterChangeValidator;
 import com.yahoo.vespa.model.application.validation.change.IndexingModeChangeValidator;
+import com.yahoo.vespa.model.application.validation.change.ResourcesReductionValidator;
 import com.yahoo.vespa.model.application.validation.change.StartupCommandChangeValidator;
 import com.yahoo.vespa.model.application.validation.change.StreamingSearchClusterChangeValidator;
 import com.yahoo.vespa.model.application.validation.first.AccessControlOnFirstDeploymentValidator;
@@ -84,6 +85,7 @@ public class Validation {
                 new ContentTypeRemovalValidator(),
                 new ContentClusterRemovalValidator(),
                 new ClusterSizeReductionValidator(),
+                new ResourcesReductionValidator(),
                 new ContainerRestartValidator(),
         };
         return Arrays.stream(validators)
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java
new file mode 100644
index 00000000000..f14ad9c51a5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java
@@ -0,0 +1,77 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation.change;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.config.application.api.ValidationId;
+import com.yahoo.config.application.api.ValidationOverrides;
+import com.yahoo.config.model.api.ConfigChangeAction;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.vespa.model.HostResource;
+import com.yahoo.vespa.model.VespaModel;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Checks that no nodes resources are reduced too much in one go.
+ *
+ * @author freva
+ */
+public class ResourcesReductionValidator implements ChangeValidator {
+
+    @Override
+    public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides, Instant now) {
+        var currentRequestedResourcesByClusterId = getRequestedResourcesByClusterId(current);
+        var nextRequestedResourcesByClusterId = getRequestedResourcesByClusterId(next);
+
+        for (var clusterTypeAndId : currentRequestedResourcesByClusterId.keySet()) {
+            if (!nextRequestedResourcesByClusterId.containsKey(clusterTypeAndId)) continue;
+            validate(currentRequestedResourcesByClusterId.get(clusterTypeAndId),
+                     nextRequestedResourcesByClusterId.get(clusterTypeAndId),
+                     clusterTypeAndId.getSecond(),
+                     overrides,
+                     now);
+        }
+
+        return List.of();
+    }
+
+    private void validate(NodeResources currentResources, NodeResources nextResources, ClusterSpec.Id clusterId,
+                          ValidationOverrides overrides, Instant now) {
+        List<String> illegalChanges = Stream.of(
+                validateResource("vCPU", currentResources.vcpu(), nextResources.vcpu()),
+                validateResource("memory GB", currentResources.memoryGb(), nextResources.memoryGb()),
+                validateResource("disk GB", currentResources.diskGb(), nextResources.diskGb()))
+                .flatMap(Optional::stream)
+                .collect(Collectors.toList());
+        if (illegalChanges.isEmpty()) return;
+
+        overrides.invalid(ValidationId.resourcesReduction,
+                "Resource reduction in '" + clusterId.value() + "' is too large. " +
+                String.join(" ", illegalChanges) + " New resources must be at least 50% of the current resources",
+                now);
+    }
+
+    private static Optional<String> validateResource(String resourceName, double currentValue, double nextValue) {
+        // don't allow more than 50% reduction, but always allow to reduce by 1
+        if (nextValue >= currentValue * 0.5 || nextValue >= currentValue - 1) return Optional.empty();
+        return Optional.of(String.format("Current %s: %.2f, new: %.2f.", resourceName, currentValue, nextValue));
+    }
+
+    private static Map<Pair<ClusterSpec.Type, ClusterSpec.Id>, NodeResources> getRequestedResourcesByClusterId(VespaModel vespaModel) {
+        return vespaModel.hostSystem().getHosts().stream()
+                .map(HostResource::spec)
+                .filter(spec -> spec.membership().isPresent() && spec.requestedResources().isPresent())
+                .filter(spec -> !spec.membership().get().retired())
+                .collect(Collectors.toMap(
+                        spec -> new Pair<>(spec.membership().get().cluster().type(), spec.membership().get().cluster().id()),
+                        spec -> spec.requestedResources().get(),
+                        (e1, e2) -> e1));
+    }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java
new file mode 100644
index 00000000000..1322a9061ed
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java
@@ -0,0 +1,103 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation.change;
+
+import com.yahoo.config.application.api.ValidationId;
+import com.yahoo.config.application.api.ValidationOverrides;
+import com.yahoo.config.model.provision.InMemoryProvisioner;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.application.validation.ValidationTester;
+import com.yahoo.yolean.Exceptions;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * @author freva
+ */
+public class ResourcesReductionValidatorTest {
+
+    private final InMemoryProvisioner provisioner = new InMemoryProvisioner(30, new NodeResources(64, 128, 1000, 10));
+    private final ValidationTester tester = new ValidationTester(provisioner);
+
+    @Test
+    public void fail_when_reduction_by_over_50_percent() {
+        VespaModel previous = tester.deploy(null, getServices(new NodeResources(8, 64, 800, 1)), Environment.prod, null).getFirst();
+        try {
+            tester.deploy(previous, getServices(new NodeResources(8, 16, 800, 1)), Environment.prod, null);
+            fail("Expected exception due to resources reduction");
+        } catch (IllegalArgumentException expected) {
+            assertEquals("resources-reduction: Resource reduction in 'default' is too large. " +
+                            "Current memory GB: 64.00, new: 16.00. New resources must be at least 50% of the current resources. " +
+                            ValidationOverrides.toAllowMessage(ValidationId.resourcesReduction),
+                    Exceptions.toMessageString(expected));
+        }
+    }
+
+    @Test
+    public void fail_when_reducing_multiple_resources_by_over_50_percent() {
+        VespaModel previous = tester.deploy(null, getServices(new NodeResources(8, 64, 800, 1)), Environment.prod, null).getFirst();
+        try {
+            tester.deploy(previous, getServices(new NodeResources(3, 16, 200, 1)), Environment.prod, null);
+            fail("Expected exception due to resources reduction");
+        } catch (IllegalArgumentException expected) {
+            assertEquals("resources-reduction: Resource reduction in 'default' is too large. " +
+                            "Current vCPU: 8.00, new: 3.00. Current memory GB: 64.00, new: 16.00. Current disk GB: 800.00, new: 200.00. " +
+                            "New resources must be at least 50% of the current resources. " +
+                            ValidationOverrides.toAllowMessage(ValidationId.resourcesReduction),
+                    Exceptions.toMessageString(expected));
+        }
+    }
+
+    @Test
+    public void small_resource_decrease_is_allowed() {
+        VespaModel previous = tester.deploy(null, getServices(new NodeResources(1.5, 64, 800, 1)), Environment.prod, null).getFirst();
+        tester.deploy(previous, getServices(new NodeResources(.5, 48, 600, 1)), Environment.prod, null);
+    }
+
+    @Test
+    public void overriding_resource_decrease() {
+        VespaModel previous = tester.deploy(null, getServices(new NodeResources(8, 64, 800, 1)), Environment.prod, null).getFirst();
+        tester.deploy(previous, getServices(new NodeResources(8, 16, 800, 1)), Environment.prod, resourcesReductionOverride); // Allowed due to override
+    }
+
+    @Test
+    public void allowed_to_go_to_not_specifying_resources() {
+        VespaModel previous = tester.deploy(null, getServices(new NodeResources(1.5, 64, 800, 1)), Environment.prod, null).getFirst();
+        tester.deploy(previous, getServices(null), Environment.prod, null);
+    }
+
+    @Test
+    public void allowed_to_go_from_not_specifying_resources() {
+        VespaModel previous = tester.deploy(null, getServices(null), Environment.prod, null).getFirst();
+        tester.deploy(previous, getServices(new NodeResources(1.5, 64, 800, 1)), Environment.prod, null);
+    }
+
+    private static String getServices(NodeResources resources) {
+        String resourcesStr = resources == null ?
+                "" :
+                String.format("        <resources vcpu='%.0f' memory='%.0fG' disk='%.0fG'/>",
+                        resources.vcpu(), resources.memoryGb(), resources.diskGb());
+        return "<services version='1.0'>" +
+                "  <content id='default' version='1.0'>" +
+                "    <redundancy>1</redundancy>" +
+                "    <engine>" +
+                "    <proton/>" +
+                "    </engine>" +
+                "    <documents>" +
+                "      <document type='music' mode='index'/>" +
+                "    </documents>" +
+                "    <nodes count='5'>" +
+                resourcesStr +
+                "    </nodes>" +
+                "   </content>" +
+                "</services>";
+    }
+
+    private static final String resourcesReductionOverride =
+            "<validation-overrides>\n" +
+            "    <allow until='2000-01-03'>resources-reduction</allow>\n" +
+            "</validation-overrides>\n";
+}
\ No newline at end of file