Add empty-host-ttl to deployment spec

3 files changed, 63 insertions, 11 deletions

diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java
index ac36e8e6c4d..fc170db5897 100644
--- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java
+++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java
@@ -1,6 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.config.application.api;
 
+import ai.vespa.validation.Validation;
 import com.yahoo.config.provision.AthenzService;
 import com.yahoo.config.provision.CloudAccount;
 import com.yahoo.config.provision.ClusterSpec;
@@ -31,6 +32,7 @@ import static ai.vespa.validation.Validation.requireAtLeast;
 import static ai.vespa.validation.Validation.requireInRange;
 import static com.yahoo.config.application.api.DeploymentSpec.RevisionChange.whenClear;
 import static com.yahoo.config.application.api.DeploymentSpec.RevisionTarget.next;
+import static com.yahoo.config.application.api.DeploymentSpec.illegal;
 import static com.yahoo.config.provision.Environment.prod;
 
 /**
@@ -58,6 +60,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps {
     private final Optional<String> globalServiceId;
     private final Optional<AthenzService> athenzService;
     private final Optional<CloudAccount> cloudAccount;
+    private final Optional<Duration> hostTTL;
     private final Notifications notifications;
     private final List<Endpoint> endpoints;
     private final Map<ClusterSpec.Id, Map<ZoneId, ZoneEndpoint>> zoneEndpoints;
@@ -75,6 +78,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps {
                                   Optional<String> globalServiceId,
                                   Optional<AthenzService> athenzService,
                                   Optional<CloudAccount> cloudAccount,
+                                  Optional<Duration> hostTTL,
                                   Notifications notifications,
                                   List<Endpoint> endpoints,
                                   Map<ClusterSpec.Id, Map<ZoneId, ZoneEndpoint>> zoneEndpoints,
@@ -98,6 +102,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps {
         this.globalServiceId = Objects.requireNonNull(globalServiceId);
         this.athenzService = Objects.requireNonNull(athenzService);
         this.cloudAccount = Objects.requireNonNull(cloudAccount);
+        this.hostTTL = Objects.requireNonNull(hostTTL);
         this.notifications = Objects.requireNonNull(notifications);
         this.endpoints = List.copyOf(Objects.requireNonNull(endpoints));
         Map<ClusterSpec.Id, Map<ZoneId, ZoneEndpoint>> zoneEndpointsCopy =  new HashMap<>();
@@ -108,6 +113,10 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps {
         validateEndpoints(globalServiceId, this.endpoints);
         validateChangeBlockers(changeBlockers, now);
         validateBcp(bcp);
+        hostTTL.ifPresent(ttl -> {
+            if (cloudAccount.isEmpty()) illegal("Host TTL can only be specified with custom cloud accounts");
+            if (ttl.isNegative()) illegal("Host TTL cannot be negative");
+        });
     }
 
     public InstanceName name() { return name; }
@@ -269,6 +278,15 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps {
                       .or(() -> cloudAccount);
     }
 
+    /** Returns the host TTL to use for given environment and region, if any */
+    public Optional<Duration> hostTTL(Environment environment, Optional<RegionName> region) {
+        return zones().stream()
+                      .filter(zone -> zone.concerns(environment, region))
+                      .findFirst()
+                      .flatMap(DeploymentSpec.DeclaredZone::hostTTL)
+                      .or(() -> hostTTL);
+    }
+
     /** Returns the notification configuration of these instances */
     public Notifications notifications() { return notifications; }
 
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java
index 1f44e599e11..43fdb32f3a2 100644
--- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java
+++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java
@@ -1,6 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.config.application.api;
 
+import ai.vespa.validation.Validation;
 import com.yahoo.collections.Comparables;
 import com.yahoo.config.application.api.xml.DeploymentSpecXmlReader;
 import com.yahoo.config.provision.AthenzDomain;
@@ -45,6 +46,7 @@ public class DeploymentSpec {
                                                                   Optional.empty(),
                                                                   Optional.empty(),
                                                                   Optional.empty(),
+                                                                  Optional.empty(),
                                                                   List.of(),
                                                                   "<deployment version='1.0'/>",
                                                                   List.of());
@@ -56,6 +58,7 @@ public class DeploymentSpec {
     private final Optional<AthenzDomain> athenzDomain;
     private final Optional<AthenzService> athenzService;
     private final Optional<CloudAccount> cloudAccount;
+    private final Optional<Duration> hostTTL;
     private final List<Endpoint> endpoints;
     private final List<DeprecatedElement> deprecatedElements;
 
@@ -66,6 +69,7 @@ public class DeploymentSpec {
                           Optional<AthenzDomain> athenzDomain,
                           Optional<AthenzService> athenzService,
                           Optional<CloudAccount> cloudAccount,
+                          Optional<Duration> hostTTL,
                           List<Endpoint> endpoints,
                           String xmlForm,
                           List<DeprecatedElement> deprecatedElements) {
@@ -74,6 +78,7 @@ public class DeploymentSpec {
         this.athenzDomain = Objects.requireNonNull(athenzDomain);
         this.athenzService = Objects.requireNonNull(athenzService);
         this.cloudAccount = Objects.requireNonNull(cloudAccount);
+        this.hostTTL = Objects.requireNonNull(hostTTL);
         this.xmlForm = Objects.requireNonNull(xmlForm);
         this.endpoints = List.copyOf(Objects.requireNonNull(endpoints));
         this.deprecatedElements = List.copyOf(Objects.requireNonNull(deprecatedElements));
@@ -81,6 +86,10 @@ public class DeploymentSpec {
         validateUpgradePoliciesOfIncreasingConservativeness(steps);
         validateAthenz();
         validateApplicationEndpoints();
+        hostTTL.ifPresent(ttl -> {
+            if (cloudAccount.isEmpty()) illegal("Host TTL can only be specified with custom cloud accounts");
+            if (ttl.isNegative()) illegal("Host TTL cannot be negative");
+        });
     }
 
     public boolean isEmpty() { return this == empty; }
@@ -184,6 +193,14 @@ public class DeploymentSpec {
     public Optional<CloudAccount> cloudAccount() { return cloudAccount; }
 
     /**
+     * Additional host time-to-live for this application. Requires a custom cloud account to be set.
+     * This also applies only to zones with dynamic provisioning, and is then the time hosts are
+     * allowed remain empty, before being deprovisioned. This is useful for applications which frequently
+     * deploy to, e.g., test and staging zones, and want to avoid the delay of having to provision hosts.
+     */
+    public Optional<Duration> hostTTL() { return hostTTL; }
+
+    /**
      * Returns the most specific zone endpoint, where specificity is given, in decreasing order:
      * 1. The given instance has declared a zone endpoint for the cluster, for the given region.
      * 2. The given instance has declared a universal zone endpoint for the cluster.
@@ -262,7 +279,7 @@ public class DeploymentSpec {
     }
 
 
-    private static void illegal(String message) {
+    static void illegal(String message) {
         throw new IllegalArgumentException(message);
     }
 
@@ -403,14 +420,15 @@ public class DeploymentSpec {
         private final Optional<AthenzService> athenzService;
         private final Optional<String> testerFlavor;
         private final Optional<CloudAccount> cloudAccount;
+        private final Optional<Duration> hostTTL;
 
         public DeclaredZone(Environment environment) {
-            this(environment, Optional.empty(), false, Optional.empty(), Optional.empty(), Optional.empty());
+            this(environment, Optional.empty(), false, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty());
         }
 
         public DeclaredZone(Environment environment, Optional<RegionName> region, boolean active,
                             Optional<AthenzService> athenzService, Optional<String> testerFlavor,
-                            Optional<CloudAccount> cloudAccount) {
+                            Optional<CloudAccount> cloudAccount, Optional<Duration> hostTTL) {
             if (environment != Environment.prod && region.isPresent())
                 illegal("Non-prod environments cannot specify a region");
             if (environment == Environment.prod && region.isEmpty())
@@ -421,6 +439,11 @@ public class DeploymentSpec {
             this.athenzService = Objects.requireNonNull(athenzService);
             this.testerFlavor = Objects.requireNonNull(testerFlavor);
             this.cloudAccount = Objects.requireNonNull(cloudAccount);
+            this.hostTTL = Objects.requireNonNull(hostTTL);
+            hostTTL.ifPresent(ttl -> {
+                if (cloudAccount.isEmpty()) illegal("Host TTL can only be specified with custom cloud accounts");
+                if (ttl.isNegative()) illegal("Host TTL cannot be negative");
+            });
         }
 
         public Environment environment() { return environment; }
@@ -472,6 +495,10 @@ public class DeploymentSpec {
             return environment + (region.map(regionName -> "." + regionName).orElse(""));
         }
 
+        public Optional<Duration> hostTTL() {
+            return hostTTL;
+        }
+
     }
 
     /** A declared production test */
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java
index 89373d8bca0..db00ad4a421 100644
--- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java
+++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java
@@ -94,6 +94,7 @@ public class DeploymentSpecXmlReader {
     private static final String majorVersionAttribute = "major-version";
     private static final String globalServiceIdAttribute = "global-service-id";
     private static final String cloudAccountAttribute = "cloud-account";
+    private static final String hostTTLAttribute = "empty-host-ttl";
 
     private final boolean validate;
     private final Clock clock;
@@ -165,6 +166,7 @@ public class DeploymentSpecXmlReader {
                                   stringAttribute(athenzDomainAttribute, root).map(AthenzDomain::from),
                                   stringAttribute(athenzServiceAttribute, root).map(AthenzService::from),
                                   stringAttribute(cloudAccountAttribute, root).map(CloudAccount::from),
+                                  stringAttribute(hostTTLAttribute, root).map(s -> toDuration(s, "empty host TTL")),
                                   applicationEndpoints,
                                   xmlForm,
                                   deprecatedElements);
@@ -204,6 +206,7 @@ public class DeploymentSpecXmlReader {
         List<DeploymentSpec.ChangeBlocker> changeBlockers = readChangeBlockers(instanceElement, parentTag);
         Optional<AthenzService> athenzService = mostSpecificAttribute(instanceElement, athenzServiceAttribute).map(AthenzService::from);
         Optional<CloudAccount> cloudAccount = mostSpecificAttribute(instanceElement, cloudAccountAttribute).map(CloudAccount::from);
+        Optional<Duration> hostTTL = mostSpecificAttribute(instanceElement, hostTTLAttribute).map(s -> toDuration(s, "empty host TTL"));
         Notifications notifications = readNotifications(instanceElement, parentTag);
 
         // Values where there is no default
@@ -233,6 +236,7 @@ public class DeploymentSpecXmlReader {
                                                              Optional.ofNullable(prodAttributes.get(globalServiceIdAttribute)),
                                                              athenzService,
                                                              cloudAccount,
+                                                             hostTTL,
                                                              notifications,
                                                              endpoints,
                                                              zoneEndpoints,
@@ -258,6 +262,7 @@ public class DeploymentSpecXmlReader {
     }
 
     // Consume the given tag as 0-N steps. 0 if it is not a step, >1 if it contains multiple nested steps that should be flattened
+    @SuppressWarnings("fallthrough")
     private List<Step> readNonInstanceSteps(Element stepTag, Map<String, String> prodAttributes, Element parentTag, Bcp defaultBcp) {
         Optional<AthenzService> athenzService = mostSpecificAttribute(stepTag, athenzServiceAttribute).map(AthenzService::from);
         Optional<String> testerFlavor = mostSpecificAttribute(stepTag, testerFlavorAttribute);
@@ -272,12 +277,10 @@ public class DeploymentSpecXmlReader {
             case testTag:
                 if (Stream.iterate(stepTag, Objects::nonNull, Node::getParentNode)
                           .anyMatch(node -> prodTag.equals(node.getNodeName()))) {
-                    // A production test
-                    return List.of(new DeclaredTest(RegionName.from(XML.getValue(stepTag).trim())));
+                    return List.of(new DeclaredTest(RegionName.from(XML.getValue(stepTag).trim()))); // A production test
                 }
-                return List.of(new DeclaredZone(Environment.from(stepTag.getTagName()), Optional.empty(), false, athenzService, testerFlavor, readCloudAccount(stepTag)));
-            case devTag, perfTag, stagingTag:
-                return List.of(new DeclaredZone(Environment.from(stepTag.getTagName()), Optional.empty(), false, athenzService, testerFlavor, readCloudAccount(stepTag)));
+            case devTag, perfTag, stagingTag: // Intentional fallthrough from test tag.
+                return List.of(new DeclaredZone(Environment.from(stepTag.getTagName()), Optional.empty(), false, athenzService, testerFlavor, readCloudAccount(stepTag), readHostTTL(stepTag)));
             case prodTag: // regions, delay and parallel may be nested within, but we can flatten them
                 return XML.getChildren(stepTag).stream()
                                                .flatMap(child -> readNonInstanceSteps(child, prodAttributes, stepTag, defaultBcp).stream())
@@ -682,13 +685,17 @@ public class DeploymentSpecXmlReader {
                                           Optional<String> testerFlavor, Element regionTag) {
         return new DeclaredZone(environment, Optional.of(RegionName.from(XML.getValue(regionTag).trim())),
                                 readActive(regionTag), athenzService, testerFlavor,
-                                readCloudAccount(regionTag));
+                                readCloudAccount(regionTag), readHostTTL(regionTag));
     }
 
     private Optional<CloudAccount> readCloudAccount(Element tag) {
         return mostSpecificAttribute(tag, cloudAccountAttribute).map(CloudAccount::from);
     }
 
+    private Optional<Duration> readHostTTL(Element tag) {
+        return mostSpecificAttribute(tag, hostTTLAttribute).map(s -> toDuration(s, "empty host TTL"));
+    }
+
     private Optional<String> readGlobalServiceId(Element environmentTag) {
         String globalServiceId = environmentTag.getAttribute(globalServiceIdAttribute);
         if (globalServiceId.isEmpty()) return Optional.empty();
@@ -804,8 +811,8 @@ public class DeploymentSpecXmlReader {
     }
 
     /**
-     * Returns a string consisting of a number followed by "m" or "h" to a duration given in that unit,
-     * or zero duration if null of blank.
+     * Returns a string consisting of a number followed by "m", "h" or "d" to a duration given in that unit,
+     * or zero duration if null or blank.
      */
     private static Duration toDuration(String durationSpec, String sourceDescription) {
         try {