diff options
10 files changed, 339 insertions, 109 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java index ac36e8e6c4d..fc170db5897 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application.api; +import ai.vespa.validation.Validation; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; @@ -31,6 +32,7 @@ import static ai.vespa.validation.Validation.requireAtLeast; import static ai.vespa.validation.Validation.requireInRange; import static com.yahoo.config.application.api.DeploymentSpec.RevisionChange.whenClear; import static com.yahoo.config.application.api.DeploymentSpec.RevisionTarget.next; +import static com.yahoo.config.application.api.DeploymentSpec.illegal; import static com.yahoo.config.provision.Environment.prod; /** @@ -58,6 +60,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { private final Optional<String> globalServiceId; private final Optional<AthenzService> athenzService; private final Optional<CloudAccount> cloudAccount; + private final Optional<Duration> hostTTL; private final Notifications notifications; private final List<Endpoint> endpoints; private final Map<ClusterSpec.Id, Map<ZoneId, ZoneEndpoint>> zoneEndpoints; @@ -75,6 +78,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { Optional<String> globalServiceId, Optional<AthenzService> athenzService, Optional<CloudAccount> cloudAccount, + Optional<Duration> hostTTL, Notifications notifications, List<Endpoint> endpoints, Map<ClusterSpec.Id, Map<ZoneId, ZoneEndpoint>> zoneEndpoints, @@ -98,6 +102,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { this.globalServiceId = Objects.requireNonNull(globalServiceId); this.athenzService = Objects.requireNonNull(athenzService); this.cloudAccount = Objects.requireNonNull(cloudAccount); + this.hostTTL = Objects.requireNonNull(hostTTL); this.notifications = Objects.requireNonNull(notifications); this.endpoints = List.copyOf(Objects.requireNonNull(endpoints)); Map<ClusterSpec.Id, Map<ZoneId, ZoneEndpoint>> zoneEndpointsCopy = new HashMap<>(); @@ -108,6 +113,10 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { validateEndpoints(globalServiceId, this.endpoints); validateChangeBlockers(changeBlockers, now); validateBcp(bcp); + hostTTL.ifPresent(ttl -> { + if (cloudAccount.isEmpty()) illegal("Host TTL can only be specified with custom cloud accounts"); + if (ttl.isNegative()) illegal("Host TTL cannot be negative"); + }); } public InstanceName name() { return name; } @@ -269,6 +278,15 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { .or(() -> cloudAccount); } + /** Returns the host TTL to use for given environment and region, if any */ + public Optional<Duration> hostTTL(Environment environment, Optional<RegionName> region) { + return zones().stream() + .filter(zone -> zone.concerns(environment, region)) + .findFirst() + .flatMap(DeploymentSpec.DeclaredZone::hostTTL) + .or(() -> hostTTL); + } + /** Returns the notification configuration of these instances */ public Notifications notifications() { return notifications; } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index 1f44e599e11..43fdb32f3a2 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application.api; +import ai.vespa.validation.Validation; import com.yahoo.collections.Comparables; import com.yahoo.config.application.api.xml.DeploymentSpecXmlReader; import com.yahoo.config.provision.AthenzDomain; @@ -45,6 +46,7 @@ public class DeploymentSpec { Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), List.of(), "<deployment version='1.0'/>", List.of()); @@ -56,6 +58,7 @@ public class DeploymentSpec { private final Optional<AthenzDomain> athenzDomain; private final Optional<AthenzService> athenzService; private final Optional<CloudAccount> cloudAccount; + private final Optional<Duration> hostTTL; private final List<Endpoint> endpoints; private final List<DeprecatedElement> deprecatedElements; @@ -66,6 +69,7 @@ public class DeploymentSpec { Optional<AthenzDomain> athenzDomain, Optional<AthenzService> athenzService, Optional<CloudAccount> cloudAccount, + Optional<Duration> hostTTL, List<Endpoint> endpoints, String xmlForm, List<DeprecatedElement> deprecatedElements) { @@ -74,6 +78,7 @@ public class DeploymentSpec { this.athenzDomain = Objects.requireNonNull(athenzDomain); this.athenzService = Objects.requireNonNull(athenzService); this.cloudAccount = Objects.requireNonNull(cloudAccount); + this.hostTTL = Objects.requireNonNull(hostTTL); this.xmlForm = Objects.requireNonNull(xmlForm); this.endpoints = List.copyOf(Objects.requireNonNull(endpoints)); this.deprecatedElements = List.copyOf(Objects.requireNonNull(deprecatedElements)); @@ -81,6 +86,10 @@ public class DeploymentSpec { validateUpgradePoliciesOfIncreasingConservativeness(steps); validateAthenz(); validateApplicationEndpoints(); + hostTTL.ifPresent(ttl -> { + if (cloudAccount.isEmpty()) illegal("Host TTL can only be specified with custom cloud accounts"); + if (ttl.isNegative()) illegal("Host TTL cannot be negative"); + }); } public boolean isEmpty() { return this == empty; } @@ -184,6 +193,14 @@ public class DeploymentSpec { public Optional<CloudAccount> cloudAccount() { return cloudAccount; } /** + * Additional host time-to-live for this application. Requires a custom cloud account to be set. + * This also applies only to zones with dynamic provisioning, and is then the time hosts are + * allowed remain empty, before being deprovisioned. This is useful for applications which frequently + * deploy to, e.g., test and staging zones, and want to avoid the delay of having to provision hosts. + */ + public Optional<Duration> hostTTL() { return hostTTL; } + + /** * Returns the most specific zone endpoint, where specificity is given, in decreasing order: * 1. The given instance has declared a zone endpoint for the cluster, for the given region. * 2. The given instance has declared a universal zone endpoint for the cluster. @@ -262,7 +279,7 @@ public class DeploymentSpec { } - private static void illegal(String message) { + static void illegal(String message) { throw new IllegalArgumentException(message); } @@ -403,14 +420,15 @@ public class DeploymentSpec { private final Optional<AthenzService> athenzService; private final Optional<String> testerFlavor; private final Optional<CloudAccount> cloudAccount; + private final Optional<Duration> hostTTL; public DeclaredZone(Environment environment) { - this(environment, Optional.empty(), false, Optional.empty(), Optional.empty(), Optional.empty()); + this(environment, Optional.empty(), false, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); } public DeclaredZone(Environment environment, Optional<RegionName> region, boolean active, Optional<AthenzService> athenzService, Optional<String> testerFlavor, - Optional<CloudAccount> cloudAccount) { + Optional<CloudAccount> cloudAccount, Optional<Duration> hostTTL) { if (environment != Environment.prod && region.isPresent()) illegal("Non-prod environments cannot specify a region"); if (environment == Environment.prod && region.isEmpty()) @@ -421,6 +439,11 @@ public class DeploymentSpec { this.athenzService = Objects.requireNonNull(athenzService); this.testerFlavor = Objects.requireNonNull(testerFlavor); this.cloudAccount = Objects.requireNonNull(cloudAccount); + this.hostTTL = Objects.requireNonNull(hostTTL); + hostTTL.ifPresent(ttl -> { + if (cloudAccount.isEmpty()) illegal("Host TTL can only be specified with custom cloud accounts"); + if (ttl.isNegative()) illegal("Host TTL cannot be negative"); + }); } public Environment environment() { return environment; } @@ -472,6 +495,10 @@ public class DeploymentSpec { return environment + (region.map(regionName -> "." + regionName).orElse("")); } + public Optional<Duration> hostTTL() { + return hostTTL; + } + } /** A declared production test */ diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java index 89373d8bca0..db00ad4a421 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java @@ -94,6 +94,7 @@ public class DeploymentSpecXmlReader { private static final String majorVersionAttribute = "major-version"; private static final String globalServiceIdAttribute = "global-service-id"; private static final String cloudAccountAttribute = "cloud-account"; + private static final String hostTTLAttribute = "empty-host-ttl"; private final boolean validate; private final Clock clock; @@ -165,6 +166,7 @@ public class DeploymentSpecXmlReader { stringAttribute(athenzDomainAttribute, root).map(AthenzDomain::from), stringAttribute(athenzServiceAttribute, root).map(AthenzService::from), stringAttribute(cloudAccountAttribute, root).map(CloudAccount::from), + stringAttribute(hostTTLAttribute, root).map(s -> toDuration(s, "empty host TTL")), applicationEndpoints, xmlForm, deprecatedElements); @@ -204,6 +206,7 @@ public class DeploymentSpecXmlReader { List<DeploymentSpec.ChangeBlocker> changeBlockers = readChangeBlockers(instanceElement, parentTag); Optional<AthenzService> athenzService = mostSpecificAttribute(instanceElement, athenzServiceAttribute).map(AthenzService::from); Optional<CloudAccount> cloudAccount = mostSpecificAttribute(instanceElement, cloudAccountAttribute).map(CloudAccount::from); + Optional<Duration> hostTTL = mostSpecificAttribute(instanceElement, hostTTLAttribute).map(s -> toDuration(s, "empty host TTL")); Notifications notifications = readNotifications(instanceElement, parentTag); // Values where there is no default @@ -233,6 +236,7 @@ public class DeploymentSpecXmlReader { Optional.ofNullable(prodAttributes.get(globalServiceIdAttribute)), athenzService, cloudAccount, + hostTTL, notifications, endpoints, zoneEndpoints, @@ -258,6 +262,7 @@ public class DeploymentSpecXmlReader { } // Consume the given tag as 0-N steps. 0 if it is not a step, >1 if it contains multiple nested steps that should be flattened + @SuppressWarnings("fallthrough") private List<Step> readNonInstanceSteps(Element stepTag, Map<String, String> prodAttributes, Element parentTag, Bcp defaultBcp) { Optional<AthenzService> athenzService = mostSpecificAttribute(stepTag, athenzServiceAttribute).map(AthenzService::from); Optional<String> testerFlavor = mostSpecificAttribute(stepTag, testerFlavorAttribute); @@ -272,12 +277,10 @@ public class DeploymentSpecXmlReader { case testTag: if (Stream.iterate(stepTag, Objects::nonNull, Node::getParentNode) .anyMatch(node -> prodTag.equals(node.getNodeName()))) { - // A production test - return List.of(new DeclaredTest(RegionName.from(XML.getValue(stepTag).trim()))); + return List.of(new DeclaredTest(RegionName.from(XML.getValue(stepTag).trim()))); // A production test } - return List.of(new DeclaredZone(Environment.from(stepTag.getTagName()), Optional.empty(), false, athenzService, testerFlavor, readCloudAccount(stepTag))); - case devTag, perfTag, stagingTag: - return List.of(new DeclaredZone(Environment.from(stepTag.getTagName()), Optional.empty(), false, athenzService, testerFlavor, readCloudAccount(stepTag))); + case devTag, perfTag, stagingTag: // Intentional fallthrough from test tag. + return List.of(new DeclaredZone(Environment.from(stepTag.getTagName()), Optional.empty(), false, athenzService, testerFlavor, readCloudAccount(stepTag), readHostTTL(stepTag))); case prodTag: // regions, delay and parallel may be nested within, but we can flatten them return XML.getChildren(stepTag).stream() .flatMap(child -> readNonInstanceSteps(child, prodAttributes, stepTag, defaultBcp).stream()) @@ -682,13 +685,17 @@ public class DeploymentSpecXmlReader { Optional<String> testerFlavor, Element regionTag) { return new DeclaredZone(environment, Optional.of(RegionName.from(XML.getValue(regionTag).trim())), readActive(regionTag), athenzService, testerFlavor, - readCloudAccount(regionTag)); + readCloudAccount(regionTag), readHostTTL(regionTag)); } private Optional<CloudAccount> readCloudAccount(Element tag) { return mostSpecificAttribute(tag, cloudAccountAttribute).map(CloudAccount::from); } + private Optional<Duration> readHostTTL(Element tag) { + return mostSpecificAttribute(tag, hostTTLAttribute).map(s -> toDuration(s, "empty host TTL")); + } + private Optional<String> readGlobalServiceId(Element environmentTag) { String globalServiceId = environmentTag.getAttribute(globalServiceIdAttribute); if (globalServiceId.isEmpty()) return Optional.empty(); @@ -804,8 +811,8 @@ public class DeploymentSpecXmlReader { } /** - * Returns a string consisting of a number followed by "m" or "h" to a duration given in that unit, - * or zero duration if null of blank. + * Returns a string consisting of a number followed by "m", "h" or "d" to a duration given in that unit, + * or zero duration if null or blank. */ private static Duration toDuration(String durationSpec, String sourceDescription) { try { diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index 89b7318739e..9cee6c57591 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -32,6 +32,11 @@ import java.util.stream.Collectors; import static com.yahoo.config.application.api.Notifications.Role.author; import static com.yahoo.config.application.api.Notifications.When.failing; import static com.yahoo.config.application.api.Notifications.When.failingCommit; +import static com.yahoo.config.provision.Environment.dev; +import static com.yahoo.config.provision.Environment.perf; +import static com.yahoo.config.provision.Environment.prod; +import static com.yahoo.config.provision.Environment.staging; +import static com.yahoo.config.provision.Environment.test; import static com.yahoo.config.provision.zone.ZoneId.defaultId; import static com.yahoo.config.provision.zone.ZoneId.from; import static org.junit.Assert.assertEquals; @@ -60,11 +65,11 @@ public class DeploymentSpecTest { assertEquals(specXml, spec.xmlForm()); assertEquals(1, spec.requireInstance("default").steps().size()); assertFalse(spec.majorVersion().isPresent()); - assertTrue(spec.requireInstance("default").steps().get(0).concerns(Environment.test)); - assertTrue(spec.requireInstance("default").concerns(Environment.test, Optional.empty())); - assertTrue(spec.requireInstance("default").concerns(Environment.test, Optional.of(RegionName.from("region1")))); // test steps specify no region - assertFalse(spec.requireInstance("default").concerns(Environment.staging, Optional.empty())); - assertFalse(spec.requireInstance("default").concerns(Environment.prod, Optional.empty())); + assertTrue(spec.requireInstance("default").steps().get(0).concerns(test)); + assertTrue(spec.requireInstance("default").concerns(test, Optional.empty())); + assertTrue(spec.requireInstance("default").concerns(test, Optional.of(RegionName.from("region1")))); // test steps specify no region + assertFalse(spec.requireInstance("default").concerns(staging, Optional.empty())); + assertFalse(spec.requireInstance("default").concerns(prod, Optional.empty())); assertFalse(spec.requireInstance("default").globalServiceId().isPresent()); } @@ -97,10 +102,10 @@ public class DeploymentSpecTest { DeploymentSpec spec = DeploymentSpec.fromXml(r); assertEquals(1, spec.steps().size()); assertEquals(1, spec.requireInstance("default").steps().size()); - assertTrue(spec.requireInstance("default").steps().get(0).concerns(Environment.staging)); - assertFalse(spec.requireInstance("default").concerns(Environment.test, Optional.empty())); - assertTrue(spec.requireInstance("default").concerns(Environment.staging, Optional.empty())); - assertFalse(spec.requireInstance("default").concerns(Environment.prod, Optional.empty())); + assertTrue(spec.requireInstance("default").steps().get(0).concerns(staging)); + assertFalse(spec.requireInstance("default").concerns(test, Optional.empty())); + assertTrue(spec.requireInstance("default").concerns(staging, Optional.empty())); + assertFalse(spec.requireInstance("default").concerns(prod, Optional.empty())); assertFalse(spec.requireInstance("default").globalServiceId().isPresent()); } @@ -121,17 +126,17 @@ public class DeploymentSpecTest { assertEquals(1, spec.steps().size()); assertEquals(2, spec.requireInstance("default").steps().size()); - assertTrue(spec.requireInstance("default").steps().get(0).concerns(Environment.prod, Optional.of(RegionName.from("us-east1")))); + assertTrue(spec.requireInstance("default").steps().get(0).concerns(prod, Optional.of(RegionName.from("us-east1")))); assertFalse(((DeploymentSpec.DeclaredZone)spec.requireInstance("default").steps().get(0)).active()); - assertTrue(spec.requireInstance("default").steps().get(1).concerns(Environment.prod, Optional.of(RegionName.from("us-west1")))); + assertTrue(spec.requireInstance("default").steps().get(1).concerns(prod, Optional.of(RegionName.from("us-west1")))); assertTrue(((DeploymentSpec.DeclaredZone)spec.requireInstance("default").steps().get(1)).active()); - assertFalse(spec.requireInstance("default").concerns(Environment.test, Optional.empty())); - assertFalse(spec.requireInstance("default").concerns(Environment.staging, Optional.empty())); - assertTrue(spec.requireInstance("default").concerns(Environment.prod, Optional.of(RegionName.from("us-east1")))); - assertTrue(spec.requireInstance("default").concerns(Environment.prod, Optional.of(RegionName.from("us-west1")))); - assertFalse(spec.requireInstance("default").concerns(Environment.prod, Optional.of(RegionName.from("no-such-region")))); + assertFalse(spec.requireInstance("default").concerns(test, Optional.empty())); + assertFalse(spec.requireInstance("default").concerns(staging, Optional.empty())); + assertTrue(spec.requireInstance("default").concerns(prod, Optional.of(RegionName.from("us-east1")))); + assertTrue(spec.requireInstance("default").concerns(prod, Optional.of(RegionName.from("us-west1")))); + assertFalse(spec.requireInstance("default").concerns(prod, Optional.of(RegionName.from("no-such-region")))); assertFalse(spec.requireInstance("default").globalServiceId().isPresent()); assertEquals(DeploymentSpec.UpgradePolicy.defaultPolicy, spec.requireInstance("default").upgradePolicy()); @@ -293,7 +298,7 @@ public class DeploymentSpecTest { assertEquals(1, instance2.steps().size()); assertEquals(1, instance2.zones().size()); - assertTrue(instance2.steps().get(0).concerns(Environment.prod, Optional.of(RegionName.from("us-central1")))); + assertTrue(instance2.steps().get(0).concerns(prod, Optional.of(RegionName.from("us-central1")))); } @Test @@ -322,25 +327,25 @@ public class DeploymentSpecTest { assertEquals(5, instance.steps().size()); assertEquals(4, instance.zones().size()); - assertTrue(instance.steps().get(0).concerns(Environment.test)); + assertTrue(instance.steps().get(0).concerns(test)); - assertTrue(instance.steps().get(1).concerns(Environment.staging)); + assertTrue(instance.steps().get(1).concerns(staging)); - assertTrue(instance.steps().get(2).concerns(Environment.prod, Optional.of(RegionName.from("us-east1")))); + assertTrue(instance.steps().get(2).concerns(prod, Optional.of(RegionName.from("us-east1")))); assertFalse(((DeploymentSpec.DeclaredZone)instance.steps().get(2)).active()); assertTrue(instance.steps().get(3) instanceof DeploymentSpec.Delay); assertEquals(3 * 60 * 60 + 30 * 60, instance.steps().get(3).delay().getSeconds()); - assertTrue(instance.steps().get(4).concerns(Environment.prod, Optional.of(RegionName.from("us-west1")))); + assertTrue(instance.steps().get(4).concerns(prod, Optional.of(RegionName.from("us-west1")))); assertTrue(((DeploymentSpec.DeclaredZone)instance.steps().get(4)).active()); - assertTrue(instance.concerns(Environment.test, Optional.empty())); - assertTrue(instance.concerns(Environment.test, Optional.of(RegionName.from("region1")))); // test steps specify no region - assertTrue(instance.concerns(Environment.staging, Optional.empty())); - assertTrue(instance.concerns(Environment.prod, Optional.of(RegionName.from("us-east1")))); - assertTrue(instance.concerns(Environment.prod, Optional.of(RegionName.from("us-west1")))); - assertFalse(instance.concerns(Environment.prod, Optional.of(RegionName.from("no-such-region")))); + assertTrue(instance.concerns(test, Optional.empty())); + assertTrue(instance.concerns(test, Optional.of(RegionName.from("region1")))); // test steps specify no region + assertTrue(instance.concerns(staging, Optional.empty())); + assertTrue(instance.concerns(prod, Optional.of(RegionName.from("us-east1")))); + assertTrue(instance.concerns(prod, Optional.of(RegionName.from("us-west1")))); + assertFalse(instance.concerns(prod, Optional.of(RegionName.from("no-such-region")))); assertFalse(instance.globalServiceId().isPresent()); } @@ -563,7 +568,7 @@ public class DeploymentSpecTest { DeploymentInstanceSpec instance = spec.instances().get(0); assertEquals("default", instance.name().value()); - assertEquals("service", instance.athenzService(Environment.prod, RegionName.defaultName()).get().value()); + assertEquals("service", instance.athenzService(prod, RegionName.defaultName()).get().value()); } @Test @@ -695,9 +700,9 @@ public class DeploymentSpecTest { List<DeploymentSpec.Step> innerParallelSteps = secondSerialSteps.get(2).steps(); assertEquals(3, innerParallelSteps.size()); assertEquals("prod.ap-northeast-1", innerParallelSteps.get(0).toString()); - assertEquals("no-service", spec.requireInstance("instance").athenzService(Environment.prod, RegionName.from("ap-northeast-1")).get().value()); + assertEquals("no-service", spec.requireInstance("instance").athenzService(prod, RegionName.from("ap-northeast-1")).get().value()); assertEquals("prod.ap-southeast-2", innerParallelSteps.get(1).toString()); - assertEquals("in-service", spec.requireInstance("instance").athenzService(Environment.prod, RegionName.from("ap-southeast-2")).get().value()); + assertEquals("in-service", spec.requireInstance("instance").athenzService(prod, RegionName.from("ap-southeast-2")).get().value()); assertEquals("tests for prod.aws-us-east-1a", innerParallelSteps.get(2).toString()); } @@ -956,7 +961,7 @@ public class DeploymentSpecTest { DeploymentSpec spec = DeploymentSpec.fromXml(r); assertEquals("domain", spec.athenzDomain().get().value()); assertEquals("service", spec.athenzService().get().value()); - assertEquals("service", spec.requireInstance("instance1").athenzService(Environment.prod, + assertEquals("service", spec.requireInstance("instance1").athenzService(prod, RegionName.from("us-west-1")).get().value()); } @@ -979,11 +984,11 @@ public class DeploymentSpecTest { assertEquals("domain", spec.athenzDomain().get().value()); assertEquals("service", spec.athenzService().get().value()); - assertEquals("prod-service", spec.requireInstance("instance1").athenzService(Environment.prod, + assertEquals("prod-service", spec.requireInstance("instance1").athenzService(prod, RegionName.from("us-central-1")).get().value()); - assertEquals("prod-service", spec.requireInstance("instance1").athenzService(Environment.prod, + assertEquals("prod-service", spec.requireInstance("instance1").athenzService(prod, RegionName.from("us-west-1")).get().value()); - assertEquals("prod-service", spec.requireInstance("instance1").athenzService(Environment.prod, + assertEquals("prod-service", spec.requireInstance("instance1").athenzService(prod, RegionName.from("us-east-3")).get().value()); } @@ -1014,11 +1019,11 @@ public class DeploymentSpecTest { """; DeploymentSpec spec = DeploymentSpec.fromXml(r); assertEquals("domain", spec.athenzDomain().get().value()); - assertEquals("service", spec.requireInstance("instance1").athenzService(Environment.prod, + assertEquals("service", spec.requireInstance("instance1").athenzService(prod, RegionName.from("us-west-1")).get().value()); - assertEquals("service", spec.requireInstance("instance1").athenzService(Environment.prod, + assertEquals("service", spec.requireInstance("instance1").athenzService(prod, RegionName.from("us-east-3")).get().value()); - assertEquals("service", spec.requireInstance("instance2").athenzService(Environment.prod, + assertEquals("service", spec.requireInstance("instance2").athenzService(prod, RegionName.from("us-east-3")).get().value()); } @@ -1036,7 +1041,7 @@ public class DeploymentSpecTest { DeploymentSpec spec = DeploymentSpec.fromXml(r); assertEquals("domain", spec.athenzDomain().get().value()); assertEquals(Optional.empty(), spec.athenzService()); - assertEquals("service", spec.requireInstance("default").athenzService(Environment.prod, RegionName.from("us-west-1")).get().value()); + assertEquals("service", spec.requireInstance("default").athenzService(prod, RegionName.from("us-west-1")).get().value()); } @Test @@ -1054,13 +1059,13 @@ public class DeploymentSpecTest { ); DeploymentSpec spec = DeploymentSpec.fromXml(r); assertEquals("service", - spec.requireInstance("default").athenzService(Environment.test, + spec.requireInstance("default").athenzService(test, RegionName.from("us-east-1")).get().value()); assertEquals("staging-service", - spec.requireInstance("default").athenzService(Environment.staging, + spec.requireInstance("default").athenzService(staging, RegionName.from("us-north-1")).get().value()); assertEquals("prod-service", - spec.requireInstance("default").athenzService(Environment.prod, + spec.requireInstance("default").athenzService(prod, RegionName.from("us-west-1")).get().value()); } @@ -1273,8 +1278,8 @@ public class DeploymentSpecTest { assertEquals(List.of(RegionName.from("us-east")), spec.requireInstance("default").endpoints().get(0).regions()); - var zone = from(Environment.prod, RegionName.from("us-east")); - var testZone = from(Environment.test, RegionName.from("us-east")); + var zone = from(prod, RegionName.from("us-east")); + var testZone = from(test, RegionName.from("us-east")); assertEquals(ZoneEndpoint.defaultEndpoint, spec.zoneEndpoint(InstanceName.from("custom"), zone, ClusterSpec.Id.from("bax"))); assertEquals(ZoneEndpoint.defaultEndpoint, @@ -1778,23 +1783,142 @@ public class DeploymentSpecTest { """; DeploymentSpec spec = DeploymentSpec.fromXml(r); assertEquals(Optional.of(CloudAccount.from("100000000000")), spec.cloudAccount()); - assertCloudAccount("800000000000", spec.requireInstance("alpha"), Environment.prod, "us-east-1"); - assertCloudAccount("200000000000", spec.requireInstance("beta"), Environment.prod, "us-west-1"); - assertCloudAccount("600000000000", spec.requireInstance("beta"), Environment.staging, ""); - assertCloudAccount("700000000000", spec.requireInstance("beta"), Environment.perf, ""); - assertCloudAccount("200000000000", spec.requireInstance("beta"), Environment.dev, ""); - assertCloudAccount("300000000000", spec.requireInstance("main"), Environment.prod, "us-east-1"); - assertCloudAccount("100000000000", spec.requireInstance("main"), Environment.prod, "eu-west-1"); - assertCloudAccount("400000000000", spec.requireInstance("main"), Environment.dev, ""); - assertCloudAccount("500000000000", spec.requireInstance("main"), Environment.test, ""); - assertCloudAccount("100000000000", spec.requireInstance("main"), Environment.staging, ""); - assertCloudAccount("default", spec.requireInstance("beta"), Environment.prod, "us-west-2"); + assertCloudAccount("800000000000", spec.requireInstance("alpha"), prod, "us-east-1"); + assertCloudAccount("200000000000", spec.requireInstance("beta"), prod, "us-west-1"); + assertCloudAccount("600000000000", spec.requireInstance("beta"), staging, ""); + assertCloudAccount("700000000000", spec.requireInstance("beta"), perf, ""); + assertCloudAccount("200000000000", spec.requireInstance("beta"), dev, ""); + assertCloudAccount("300000000000", spec.requireInstance("main"), prod, "us-east-1"); + assertCloudAccount("100000000000", spec.requireInstance("main"), prod, "eu-west-1"); + assertCloudAccount("400000000000", spec.requireInstance("main"), dev, ""); + assertCloudAccount("500000000000", spec.requireInstance("main"), test, ""); + assertCloudAccount("100000000000", spec.requireInstance("main"), staging, ""); + assertCloudAccount("default", spec.requireInstance("beta"), prod, "us-west-2"); + } + + @Test + public void hostTTL() { + String r = + """ + <deployment version='1.0' cloud-account='100000000000' empty-host-ttl='1h'> + <instance id='alpha'> + <staging /> + <prod empty-host-ttl='1m'> + <region>us-east</region> + <region empty-host-ttl='2m'>us-west</region> + </prod> + </instance> + <instance id='beta'> + <staging empty-host-ttl='3d'/> + <perf empty-host-ttl='4h'/> + <prod> + <region>us-east</region> + <region empty-host-ttl='0d'>us-west</region> + </prod> + </instance> + <instance id='gamma' empty-host-ttl='6h'> + <dev empty-host-ttl='7d'/> + <prod> + <region>us-east</region> + </prod> + </instance> + </deployment> + """; + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(Optional.of(CloudAccount.from("100000000000")), spec.cloudAccount()); + + assertHostTTL(Duration.ofHours(1), spec, "alpha", test, null); + assertHostTTL(Duration.ofHours(1), spec, "alpha", staging, null); + assertHostTTL(Duration.ofHours(1), spec, "alpha", dev, null); + assertHostTTL(Duration.ofHours(1), spec, "alpha", perf, null); + assertHostTTL(Duration.ofMinutes(1), spec, "alpha", prod, "us-east"); + assertHostTTL(Duration.ofMinutes(2), spec, "alpha", prod, "us-west"); + + assertHostTTL(Duration.ofHours(1), spec, "beta", test, null); + assertHostTTL(Duration.ofDays(3), spec, "beta", staging, null); + assertHostTTL(Duration.ofHours(1), spec, "beta", dev, null); + assertHostTTL(Duration.ofHours(4), spec, "beta", perf, null); + assertHostTTL(Duration.ofHours(1), spec, "beta", prod, "us-east"); + assertHostTTL(Duration.ZERO, spec, "beta", prod, "us-west"); + + assertHostTTL(Duration.ofHours(6), spec, "gamma", test, null); + assertHostTTL(Duration.ofHours(6), spec, "gamma", staging, null); + assertHostTTL(Duration.ofDays(7), spec, "gamma", dev, null); + assertHostTTL(Duration.ofHours(6), spec, "gamma", perf, null); + assertHostTTL(Duration.ofHours(6), spec, "gamma", prod, "us-east"); + assertHostTTL(Duration.ofHours(6), spec, "gamma", prod, "us-west"); + + assertHostTTL(Duration.ofHours(1), spec, "nope", test, null); + assertHostTTL(Duration.ofHours(1), spec, "nope", staging, null); + assertHostTTL(Duration.ofHours(1), spec, "nope", dev, null); + assertHostTTL(Duration.ofHours(1), spec, "nope", perf, null); + assertHostTTL(Duration.ofHours(1), spec, "nope", prod, "us-east"); + assertHostTTL(Duration.ofHours(1), spec, "nope", prod, "us-west"); + + assertEquals("Host TTL can only be specified with custom cloud accounts", + assertThrows(IllegalArgumentException.class, + () -> DeploymentSpec.fromXml(""" + <deployment version='1.0' empty-host-ttl='0m'> + <instance id='main'> + <prod> + <region>us-east-1</region> + </prod> + </instance> + </deployment> + """)) + .getMessage()); + + assertEquals("Host TTL can only be specified with custom cloud accounts", + assertThrows(IllegalArgumentException.class, + () -> DeploymentSpec.fromXml(""" + <deployment version='1.0'> + <instance id='main' empty-host-ttl='0m'> + <prod> + <region>us-east-1</region> + </prod> + </instance> + </deployment> + """)) + .getMessage()); + + assertEquals("Host TTL can only be specified with custom cloud accounts", + assertThrows(IllegalArgumentException.class, + () -> DeploymentSpec.fromXml(""" + <deployment version='1.0'> + <instance id='main'> + <prod empty-host-ttl='0m'> + <region>us-east-1</region> + </prod> + </instance> + </deployment> + """)) + .getMessage()); + + assertEquals("Host TTL can only be specified with custom cloud accounts", + assertThrows(IllegalArgumentException.class, + () -> DeploymentSpec.fromXml(""" + <deployment version='1.0'> + <instance id='main'> + <prod> + <region empty-host-ttl='0m'>us-east-1</region> + </prod> + </instance> + </deployment> + """)) + .getMessage()); + } private void assertCloudAccount(String expected, DeploymentInstanceSpec instance, Environment environment, String region) { assertEquals(Optional.of(expected).map(CloudAccount::from), instance.cloudAccount(environment, Optional.of(region).filter(s -> !s.isEmpty()).map(RegionName::from))); } + private void assertHostTTL(Duration expected, DeploymentSpec spec, String instance, Environment environment, String region) { + assertEquals(Optional.of(expected), spec.instance(InstanceName.from(instance)) + .flatMap(iSpec -> iSpec.hostTTL(environment, Optional.ofNullable(region).map(RegionName::from))) + .or(spec::hostTTL)); + } + private static void assertInvalid(String deploymentSpec, String errorMessagePart) { assertInvalid(deploymentSpec, errorMessagePart, new ManualClock()); } diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java index 38410cc5b37..ed8074ddc61 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java @@ -25,6 +25,9 @@ import java.util.stream.Collectors; import static com.yahoo.config.application.api.Notifications.Role.author; import static com.yahoo.config.application.api.Notifications.When.failing; import static com.yahoo.config.application.api.Notifications.When.failingCommit; +import static com.yahoo.config.provision.Environment.dev; +import static com.yahoo.config.provision.Environment.prod; +import static com.yahoo.config.provision.Environment.test; import static com.yahoo.config.provision.zone.ZoneId.defaultId; import static com.yahoo.config.provision.zone.ZoneId.from; import static org.junit.Assert.assertEquals; @@ -49,11 +52,11 @@ public class DeploymentSpecWithoutInstanceTest { assertEquals(specXml, spec.xmlForm()); assertEquals(1, spec.steps().size()); assertFalse(spec.majorVersion().isPresent()); - assertTrue(spec.steps().get(0).concerns(Environment.test)); - assertTrue(spec.requireInstance("default").concerns(Environment.test, Optional.empty())); - assertTrue(spec.requireInstance("default").concerns(Environment.test, Optional.of(RegionName.from("region1")))); // test steps specify no region + assertTrue(spec.steps().get(0).concerns(test)); + assertTrue(spec.requireInstance("default").concerns(test, Optional.empty())); + assertTrue(spec.requireInstance("default").concerns(test, Optional.of(RegionName.from("region1")))); // test steps specify no region assertFalse(spec.requireInstance("default").concerns(Environment.staging, Optional.empty())); - assertFalse(spec.requireInstance("default").concerns(Environment.prod, Optional.empty())); + assertFalse(spec.requireInstance("default").concerns(prod, Optional.empty())); assertFalse(spec.requireInstance("default").globalServiceId().isPresent()); } @@ -83,9 +86,9 @@ public class DeploymentSpecWithoutInstanceTest { assertEquals(1, spec.steps().size()); assertEquals(1, spec.requireInstance("default").steps().size()); assertTrue(spec.requireInstance("default").steps().get(0).concerns(Environment.staging)); - assertFalse(spec.requireInstance("default").concerns(Environment.test, Optional.empty())); + assertFalse(spec.requireInstance("default").concerns(test, Optional.empty())); assertTrue(spec.requireInstance("default").concerns(Environment.staging, Optional.empty())); - assertFalse(spec.requireInstance("default").concerns(Environment.prod, Optional.empty())); + assertFalse(spec.requireInstance("default").concerns(prod, Optional.empty())); assertFalse(spec.requireInstance("default").globalServiceId().isPresent()); } @@ -104,17 +107,17 @@ public class DeploymentSpecWithoutInstanceTest { assertEquals(1, spec.steps().size()); assertEquals(2, spec.requireInstance("default").steps().size()); - assertTrue(spec.requireInstance("default").steps().get(0).concerns(Environment.prod, Optional.of(RegionName.from("us-east1")))); + assertTrue(spec.requireInstance("default").steps().get(0).concerns(prod, Optional.of(RegionName.from("us-east1")))); assertFalse(((DeploymentSpec.DeclaredZone)spec.requireInstance("default").steps().get(0)).active()); - assertTrue(spec.requireInstance("default").steps().get(1).concerns(Environment.prod, Optional.of(RegionName.from("us-west1")))); + assertTrue(spec.requireInstance("default").steps().get(1).concerns(prod, Optional.of(RegionName.from("us-west1")))); assertTrue(((DeploymentSpec.DeclaredZone)spec.requireInstance("default").steps().get(1)).active()); - assertFalse(spec.requireInstance("default").concerns(Environment.test, Optional.empty())); + assertFalse(spec.requireInstance("default").concerns(test, Optional.empty())); assertFalse(spec.requireInstance("default").concerns(Environment.staging, Optional.empty())); - assertTrue(spec.requireInstance("default").concerns(Environment.prod, Optional.of(RegionName.from("us-east1")))); - assertTrue(spec.requireInstance("default").concerns(Environment.prod, Optional.of(RegionName.from("us-west1")))); - assertFalse(spec.requireInstance("default").concerns(Environment.prod, Optional.of(RegionName.from("no-such-region")))); + assertTrue(spec.requireInstance("default").concerns(prod, Optional.of(RegionName.from("us-east1")))); + assertTrue(spec.requireInstance("default").concerns(prod, Optional.of(RegionName.from("us-west1")))); + assertFalse(spec.requireInstance("default").concerns(prod, Optional.of(RegionName.from("no-such-region")))); assertFalse(spec.requireInstance("default").globalServiceId().isPresent()); assertEquals(DeploymentSpec.UpgradePolicy.defaultPolicy, spec.requireInstance("default").upgradePolicy()); @@ -139,25 +142,25 @@ public class DeploymentSpecWithoutInstanceTest { assertEquals(5, spec.requireInstance("default").steps().size()); assertEquals(4, spec.requireInstance("default").zones().size()); - assertTrue(spec.requireInstance("default").steps().get(0).concerns(Environment.test)); + assertTrue(spec.requireInstance("default").steps().get(0).concerns(test)); assertTrue(spec.requireInstance("default").steps().get(1).concerns(Environment.staging)); - assertTrue(spec.requireInstance("default").steps().get(2).concerns(Environment.prod, Optional.of(RegionName.from("us-east1")))); + assertTrue(spec.requireInstance("default").steps().get(2).concerns(prod, Optional.of(RegionName.from("us-east1")))); assertFalse(((DeploymentSpec.DeclaredZone)spec.requireInstance("default").steps().get(2)).active()); assertTrue(spec.requireInstance("default").steps().get(3) instanceof DeploymentSpec.Delay); assertEquals(3 * 60 * 60 + 30 * 60, spec.requireInstance("default").steps().get(3).delay().getSeconds()); - assertTrue(spec.requireInstance("default").steps().get(4).concerns(Environment.prod, Optional.of(RegionName.from("us-west1")))); + assertTrue(spec.requireInstance("default").steps().get(4).concerns(prod, Optional.of(RegionName.from("us-west1")))); assertTrue(((DeploymentSpec.DeclaredZone)spec.requireInstance("default").steps().get(4)).active()); - assertTrue(spec.requireInstance("default").concerns(Environment.test, Optional.empty())); - assertTrue(spec.requireInstance("default").concerns(Environment.test, Optional.of(RegionName.from("region1")))); // test steps specify no region + assertTrue(spec.requireInstance("default").concerns(test, Optional.empty())); + assertTrue(spec.requireInstance("default").concerns(test, Optional.of(RegionName.from("region1")))); // test steps specify no region assertTrue(spec.requireInstance("default").concerns(Environment.staging, Optional.empty())); - assertTrue(spec.requireInstance("default").concerns(Environment.prod, Optional.of(RegionName.from("us-east1")))); - assertTrue(spec.requireInstance("default").concerns(Environment.prod, Optional.of(RegionName.from("us-west1")))); - assertFalse(spec.requireInstance("default").concerns(Environment.prod, Optional.of(RegionName.from("no-such-region")))); + assertTrue(spec.requireInstance("default").concerns(prod, Optional.of(RegionName.from("us-east1")))); + assertTrue(spec.requireInstance("default").concerns(prod, Optional.of(RegionName.from("us-west1")))); + assertFalse(spec.requireInstance("default").concerns(prod, Optional.of(RegionName.from("no-such-region")))); assertFalse(spec.requireInstance("default").globalServiceId().isPresent()); } @@ -436,9 +439,9 @@ public class DeploymentSpecWithoutInstanceTest { List<DeploymentSpec.Step> innerParallelSteps = secondSerialSteps.get(2).steps(); assertEquals(3, innerParallelSteps.size()); assertEquals("prod.ap-northeast-1", innerParallelSteps.get(0).toString()); - assertEquals("no-service", spec.requireInstance("default").athenzService(Environment.prod, RegionName.from("ap-northeast-1")).get().value()); + assertEquals("no-service", spec.requireInstance("default").athenzService(prod, RegionName.from("ap-northeast-1")).get().value()); assertEquals("prod.ap-southeast-2", innerParallelSteps.get(1).toString()); - assertEquals("service", spec.requireInstance("default").athenzService(Environment.prod, RegionName.from("ap-southeast-2")).get().value()); + assertEquals("service", spec.requireInstance("default").athenzService(prod, RegionName.from("ap-southeast-2")).get().value()); assertEquals("tests for prod.aws-us-east-1a", innerParallelSteps.get(2).toString()); } @@ -534,7 +537,7 @@ public class DeploymentSpecWithoutInstanceTest { ); DeploymentSpec spec = DeploymentSpec.fromXml(r); assertEquals(spec.athenzDomain().get().value(), "domain"); - assertEquals(spec.requireInstance("default").athenzService(Environment.prod, RegionName.from("us-west-1")).get().value(), "service"); + assertEquals(spec.requireInstance("default").athenzService(prod, RegionName.from("us-west-1")).get().value(), "service"); } @Test @@ -553,11 +556,11 @@ public class DeploymentSpecWithoutInstanceTest { DeploymentSpec spec = DeploymentSpec.fromXml(r); assertEquals("domain", spec.athenzDomain().get().value()); assertEquals("service", spec.athenzService().get().value()); - assertEquals("prod-service", spec.requireInstance("default").athenzService(Environment.prod, RegionName.from("us-central-1")) + assertEquals("prod-service", spec.requireInstance("default").athenzService(prod, RegionName.from("us-central-1")) .get().value()); - assertEquals("prod-service", spec.requireInstance("default").athenzService(Environment.prod, RegionName.from("us-west-1")) + assertEquals("prod-service", spec.requireInstance("default").athenzService(prod, RegionName.from("us-west-1")) .get().value()); - assertEquals("prod-service", spec.requireInstance("default").athenzService(Environment.prod, RegionName.from("us-east-3")) + assertEquals("prod-service", spec.requireInstance("default").athenzService(prod, RegionName.from("us-east-3")) .get().value()); } @@ -575,9 +578,9 @@ public class DeploymentSpecWithoutInstanceTest { DeploymentSpec spec = DeploymentSpec.fromXml(r); assertEquals("service", spec.athenzService().get().value()); assertEquals(spec.athenzDomain().get().value(), "domain"); - assertEquals(spec.requireInstance("default").athenzService(Environment.test, RegionName.from("us-east-1")).get().value(), "service"); + assertEquals(spec.requireInstance("default").athenzService(test, RegionName.from("us-east-1")).get().value(), "service"); assertEquals(spec.requireInstance("default").athenzService(Environment.staging, RegionName.from("us-north-1")).get().value(), "staging-service"); - assertEquals(spec.requireInstance("default").athenzService(Environment.prod, RegionName.from("us-west-1")).get().value(), "prod-service"); + assertEquals(spec.requireInstance("default").athenzService(prod, RegionName.from("us-west-1")).get().value(), "prod-service"); } @Test(expected = IllegalArgumentException.class) @@ -694,7 +697,7 @@ public class DeploymentSpecWithoutInstanceTest { assertEquals(List.of(RegionName.from("us-east")), spec.requireInstance("default").endpoints().get(0).regions()); - var zone = from(Environment.prod, RegionName.from("us-east")); + var zone = from(prod, RegionName.from("us-east")); assertEquals(ZoneEndpoint.defaultEndpoint, spec.zoneEndpoint(InstanceName.from("custom"), zone, ClusterSpec.Id.from("bax"))); assertEquals(ZoneEndpoint.defaultEndpoint, @@ -742,8 +745,8 @@ public class DeploymentSpecWithoutInstanceTest { DeploymentSpec spec = DeploymentSpec.fromXml(r); DeploymentInstanceSpec instance = spec.requireInstance("default"); assertEquals(Optional.of(CloudAccount.from("012345678912")), spec.cloudAccount()); - assertEquals(Optional.of(CloudAccount.from("219876543210")), instance.cloudAccount(Environment.prod, Optional.of(RegionName.from("us-east-1")))); - assertEquals(Optional.of(CloudAccount.from("012345678912")), instance.cloudAccount(Environment.prod, Optional.of(RegionName.from("us-west-1")))); + assertEquals(Optional.of(CloudAccount.from("219876543210")), instance.cloudAccount(prod, Optional.of(RegionName.from("us-east-1")))); + assertEquals(Optional.of(CloudAccount.from("012345678912")), instance.cloudAccount(prod, Optional.of(RegionName.from("us-west-1")))); assertEquals(Optional.of(CloudAccount.from("012345678912")), instance.cloudAccount(Environment.staging, Optional.empty())); r = new StringReader( @@ -756,8 +759,41 @@ public class DeploymentSpecWithoutInstanceTest { ); spec = DeploymentSpec.fromXml(r); assertEquals(Optional.empty(), spec.cloudAccount()); - assertEquals(Optional.of(CloudAccount.from("219876543210")), spec.requireInstance("default").cloudAccount(Environment.prod, Optional.of(RegionName.from("us-east-1")))); - assertEquals(Optional.empty(), spec.requireInstance("default").cloudAccount(Environment.prod, Optional.of(RegionName.from("us-west-1")))); + assertEquals(Optional.of(CloudAccount.from("219876543210")), spec.requireInstance("default").cloudAccount(prod, Optional.of(RegionName.from("us-east-1")))); + assertEquals(Optional.empty(), spec.requireInstance("default").cloudAccount(prod, Optional.of(RegionName.from("us-west-1")))); + } + + @Test + public void productionSpecWithHostTTL() { + String r = """ + <deployment version='1.0' cloud-account='012345678912' empty-host-ttl='1d'> + <prod> + <region empty-host-ttl='1m'>us-east-1</region> + <region>us-west-1</region> + </prod> + </deployment> + """; + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(Optional.of(Duration.ofDays(1)), spec.hostTTL()); + DeploymentInstanceSpec instance = spec.requireInstance("default"); + assertEquals(Optional.of(Duration.ofMinutes(1)), instance.hostTTL(prod, Optional.of(RegionName.from("us-east-1")))); + assertEquals(Optional.of(Duration.ofDays(1)), instance.hostTTL(prod, Optional.of(RegionName.from("us-west-1")))); + assertEquals(Optional.of(Duration.ofDays(1)), instance.hostTTL(test, Optional.empty())); + + r = """ + <deployment version='1.0' cloud-account='012345678912'> + <prod empty-host-ttl='1d'> + <region empty-host-ttl='1m'>us-east-1</region> + <region>us-west-1</region> + </prod> + </deployment> + """; + spec = DeploymentSpec.fromXml(r); + assertEquals(Optional.empty(), spec.hostTTL()); + instance = spec.requireInstance("default"); + assertEquals(Optional.of(Duration.ofMinutes(1)), instance.hostTTL(prod, Optional.of(RegionName.from("us-east-1")))); + assertEquals(Optional.of(Duration.ofDays(1)), instance.hostTTL(prod, Optional.of(RegionName.from("us-west-1")))); + assertEquals(Optional.empty(), instance.hostTTL(test, Optional.empty())); } private static Set<String> endpointRegions(String endpointId, DeploymentSpec spec) { diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java index 60592a7951f..a9ee884e977 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java @@ -3,15 +3,18 @@ package com.yahoo.config.model; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.provision.ClusterInfo; +import com.yahoo.config.provision.ClusterInfo.Builder; import com.yahoo.vespa.model.VespaModel; import java.time.Duration; import java.util.Comparator; +import java.util.Optional; import java.util.stream.Stream; /** @@ -72,15 +75,20 @@ public final class ConfigModelContext { /** Returns a cluster info builder pre-populated with info known in this context. */ public ClusterInfo.Builder clusterInfo() { - // TODO jonmv: get TTL - var instance = getApplicationPackage().getDeploymentSpec().instance(properties().applicationId().instance()); - if ( ! instance.isPresent()) return new ClusterInfo.Builder(); + DeploymentSpec spec = getApplicationPackage().getDeploymentSpec(); + var instance = spec.instance(properties().applicationId().instance()); + ClusterInfo.Builder builder = new ClusterInfo.Builder(); + spec.hostTTL().ifPresent(builder::hostTTL); + if (instance.isEmpty()) return builder; + instance.get() + .hostTTL(deployState.zone().environment(), Optional.of(deployState.zone().region())) + .ifPresent(builder::hostTTL); var maxDeadline = instance.get().bcp().groups().stream() .filter(group -> group.memberRegions().contains(properties().zone().region())) .map(group -> group.deadline()) .min(Comparator.comparing(deadline -> deadline)) .orElse(Duration.ofMinutes(0)); - return new ClusterInfo.Builder().bcpDeadline(maxDeadline); + return builder.bcpDeadline(maxDeadline); } /** diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java index 3883e4f70c6..d9076557ac7 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java @@ -3,6 +3,8 @@ package com.yahoo.config.provision; import java.time.Duration; import java.util.Objects; +import static ai.vespa.validation.Validation.requireAtLeast; + /** * Auxiliary information about a cluster, provided by the config model to the node repo during a * capacity request. @@ -19,6 +21,7 @@ public class ClusterInfo { private ClusterInfo(Builder builder) { this.bcpDeadline = builder.bcpDeadline; this.hostTTL = builder.hostTTL; + requireAtLeast(hostTTL, "host TTL", Duration.ZERO); } public Duration bcpDeadline() { return bcpDeadline; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java index 1e378c80f90..f62e019e408 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java @@ -58,6 +58,7 @@ public class ApplicationSerializer { private static final String suggestedKey = "suggested"; private static final String clusterInfoKey = "clusterInfo"; private static final String bcpDeadlineKey = "bcpDeadline"; + private static final String hostTTLKey = "hostTTL"; private static final String bcpGroupInfoKey = "bcpGroupInfo"; private static final String queryRateKey = "queryRateKey"; private static final String growthRateHeadroomKey = "growthRateHeadroomKey"; @@ -234,12 +235,14 @@ public class ApplicationSerializer { private static void toSlime(ClusterInfo clusterInfo, Cursor clusterInfoObject) { clusterInfoObject.setLong(bcpDeadlineKey, clusterInfo.bcpDeadline().toMinutes()); + if ( ! clusterInfo.hostTTL().isZero()) clusterInfoObject.setLong(hostTTLKey, clusterInfo.hostTTL().toMillis()); } private static ClusterInfo clusterInfoFromSlime(Inspector clusterInfoObject) { if ( ! clusterInfoObject.valid()) return ClusterInfo.empty(); ClusterInfo.Builder builder = new ClusterInfo.Builder(); builder.bcpDeadline(Duration.ofMinutes(clusterInfoObject.field(bcpDeadlineKey).asLong())); + builder.hostTTL(Duration.ofMillis(clusterInfoObject.field(hostTTLKey).asLong())); return builder.build(); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java index 9de4b8768b2..2a6e5a0d1e7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java @@ -510,11 +510,10 @@ public class HostCapacityMaintainerTest { tester.provisioningTester.makeReadyHosts(2, new NodeResources(1, 1, 1, 1)).activateTenantHosts(); List<Node> hosts = tester.nodeRepository.nodes().list(Node.State.active).asList(); Node host1 = hosts.get(0); + Node host2 = hosts.get(1); tester.nodeRepository.nodes().write(host1.withHostTTL(Duration.ofDays(1)), () -> { }); + tester.nodeRepository.nodes().write(host2.withHostTTL(Duration.ofHours(1)), () -> { }); Node host11 = tester.addNode("host1-1", Optional.of(host1.hostname()), NodeType.tenant, State.active, DynamicProvisioningTester.tenantApp); - Node host2 = hosts.get(1); - tester.nodeRepository.nodes().write(host2.withHostTTL(Duration.ofDays(1)), () -> { }); - Node host21 = tester.addNode("host2-1", Optional.of(host2.hostname()), NodeType.tenant, State.active, ApplicationId.from("t", "a", "i")); // Host is not marked for deprovisioning by maintainer, because child is present tester.maintain(); @@ -537,14 +536,19 @@ public class HostCapacityMaintainerTest { assertEquals(Optional.of(clock.instant().truncatedTo(ChronoUnit.MILLIS)), tester.nodeRepository.nodes().node(host1.hostname()).get().hostEmptyAt()); - // Some time passes, but not enough for host to be deprovisioned + // Some time passes, but not enough for host1 to be deprovisioned clock.advance(Duration.ofDays(1).minusSeconds(1)); tester.maintain(); assertFalse(tester.nodeRepository.nodes().node(host1.hostname()).get().status().wantToDeprovision()); assertEquals(Optional.of(clock.instant().minus(Duration.ofDays(1).minusSeconds(1)).truncatedTo(ChronoUnit.MILLIS)), tester.nodeRepository.nodes().node(host1.hostname()).get().hostEmptyAt()); + assertTrue(tester.nodeRepository.nodes().node(host2.hostname()).get().status().wantToDeprovision()); + assertTrue(tester.nodeRepository.nodes().node(host2.hostname()).get().status().wantToRetire()); + assertEquals(State.active, tester.nodeRepository.nodes().node(host2.hostname()).get().state()); + assertEquals(Optional.of(clock.instant().minus(Duration.ofDays(1).minusSeconds(1)).truncatedTo(ChronoUnit.MILLIS)), + tester.nodeRepository.nodes().node(host2.hostname()).get().hostEmptyAt()); - // Some more time passes, but child is reactivated, rendering the host non-empty again + // Some more time passes, but child is reactivated on host1, rendering the host non-empty again clock.advance(Duration.ofDays(1)); tester.nodeRepository.nodes().reactivate(host11.hostname(), Agent.operator, "all good"); tester.maintain(); @@ -581,7 +585,7 @@ public class HostCapacityMaintainerTest { // Host and children can now be removed. tester.provisioningTester.activateTenantHosts(); tester.maintain(); - assertEquals(Set.of(host2, host21), Set.copyOf(tester.nodeRepository.nodes().list().asList())); + assertEquals(List.of(), tester.nodeRepository.nodes().list().asList()); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java index c429f88cfa1..c997da543ea 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java @@ -68,7 +68,7 @@ public class ApplicationSerializerTest { Load.zero(), Load.one(), Autoscaling.Metrics.zero()), - new ClusterInfo.Builder().bcpDeadline(Duration.ofMinutes(33)).build(), + new ClusterInfo.Builder().bcpDeadline(Duration.ofMinutes(33)).hostTTL(Duration.ofSeconds(321)).build(), new BcpGroupInfo(0.1, 0.2, 0.3), List.of(new ScalingEvent(new ClusterResources(10, 5, minResources), new ClusterResources(12, 6, minResources), |