diff options
author | Jon Bratseth <bratseth@verizonmedia.com> | 2019-10-04 15:03:23 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@verizonmedia.com> | 2019-10-04 15:03:23 +0200 |
commit | 5188586ce3314efd0fffd52a62403e1b371890dd (patch) | |
tree | e5b43fe64cb851b240c78e5cca9809c8450c82af | |
parent | 2f95c6b456c6fe12b0f488394439193103c6e1f2 (diff) |
Avoid deprecated methods
21 files changed, 305 insertions, 220 deletions
diff --git a/abi-check-plugin/src/main/java/com/yahoo/abicheck/mojo/AbiCheck.java b/abi-check-plugin/src/main/java/com/yahoo/abicheck/mojo/AbiCheck.java index a09e23bbe9e..c5452ecdde3 100644 --- a/abi-check-plugin/src/main/java/com/yahoo/abicheck/mojo/AbiCheck.java +++ b/abi-check-plugin/src/main/java/com/yahoo/abicheck/mojo/AbiCheck.java @@ -192,7 +192,7 @@ public class AbiCheck extends AbstractMojo { } else { Map<String, JavaClassSignature> abiSpec = readSpec(specFile); if (!compareSignatures(abiSpec, signatures, getLog())) { - throw new MojoFailureException("ABI spec mismatch"); + throw new MojoFailureException("ABI spec mismatch. To update run 'mvn package -Dabicheck.writeSpec'"); } } } catch (IOException e) { diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 32c9e433157..5004aebc593 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -187,6 +187,35 @@ ], "fields": [] }, + "com.yahoo.config.application.api.DeploymentInstanceSpec": { + "superClass": "com.yahoo.config.application.api.DeploymentSpec$Step", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.config.provision.InstanceName, java.util.List, com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy, java.util.List, java.util.Optional, java.util.Optional, java.util.Optional, com.yahoo.config.application.api.Notifications, java.util.List)", + "public com.yahoo.config.provision.InstanceName name()", + "public java.time.Duration delay()", + "public java.util.List steps()", + "public com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy upgradePolicy()", + "public java.util.List changeBlocker()", + "public java.util.Optional globalServiceId()", + "public boolean canUpgradeAt(java.time.Instant)", + "public boolean canChangeRevisionAt(java.time.Instant)", + "public java.util.List zones()", + "public boolean deploysTo(com.yahoo.config.provision.Environment, java.util.Optional)", + "public java.util.Optional athenzDomain()", + "public java.util.Optional athenzService(com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", + "public com.yahoo.config.application.api.Notifications notifications()", + "public java.util.List endpoints()", + "public boolean includes(com.yahoo.config.provision.Environment, java.util.Optional)", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields": [] + }, "com.yahoo.config.application.api.DeploymentSpec$ChangeBlocker": { "superClass": "java.lang.Object", "interfaces": [], @@ -234,6 +263,7 @@ "methods": [ "public void <init>(java.time.Duration)", "public java.time.Duration duration()", + "public java.time.Duration delay()", "public boolean deploysTo(com.yahoo.config.provision.Environment, java.util.Optional)" ], "fields": [] @@ -247,6 +277,7 @@ "methods": [ "public void <init>(java.util.List)", "public java.util.List zones()", + "public java.util.List steps()", "public boolean deploysTo(com.yahoo.config.provision.Environment, java.util.Optional)", "public boolean equals(java.lang.Object)", "public int hashCode()" @@ -264,7 +295,8 @@ "public void <init>()", "public final boolean deploysTo(com.yahoo.config.provision.Environment)", "public abstract boolean deploysTo(com.yahoo.config.provision.Environment, java.util.Optional)", - "public java.util.List zones()" + "public java.util.List zones()", + "public java.time.Duration delay()" ], "fields": [] }, @@ -293,6 +325,7 @@ "public" ], "methods": [ + "public void <init>(java.util.List, java.util.Optional, java.lang.String)", "public void <init>(java.util.Optional, com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy, java.util.Optional, java.util.List, java.util.List, java.lang.String, java.util.Optional, java.util.Optional, com.yahoo.config.application.api.Notifications, java.util.List)", "public java.util.Optional globalServiceId()", "public com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy upgradePolicy()", @@ -302,16 +335,21 @@ "public java.util.List changeBlocker()", "public java.util.List steps()", "public java.util.List zones()", + "public java.util.Optional athenzDomain()", + "public java.util.Optional athenzService(com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", "public com.yahoo.config.application.api.Notifications notifications()", "public java.util.List endpoints()", "public java.lang.String xmlForm()", "public boolean includes(com.yahoo.config.provision.Environment, java.util.Optional)", + "public com.yahoo.config.application.api.DeploymentInstanceSpec instance(java.lang.String)", + "public com.yahoo.config.application.api.DeploymentInstanceSpec instance(com.yahoo.config.provision.InstanceName)", + "public com.yahoo.config.application.api.DeploymentInstanceSpec requireInstance(java.lang.String)", + "public com.yahoo.config.application.api.DeploymentInstanceSpec requireInstance(com.yahoo.config.provision.InstanceName)", + "public java.util.List instances()", "public static com.yahoo.config.application.api.DeploymentSpec fromXml(java.io.Reader)", "public static com.yahoo.config.application.api.DeploymentSpec fromXml(java.lang.String)", "public static com.yahoo.config.application.api.DeploymentSpec fromXml(java.lang.String, boolean)", "public static java.lang.String toMessageString(java.lang.Throwable)", - "public java.util.Optional athenzDomain()", - "public java.util.Optional athenzService(com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", "public boolean equals(java.lang.Object)", "public int hashCode()", "public static void main(java.lang.String[])" diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstancesSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java index 344f3438804..1fc3a8af2eb 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstancesSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java @@ -18,14 +18,14 @@ import java.util.Set; import java.util.stream.Collectors; /** - * The deployment spec for some specified application instances + * The deployment spec for an application instance * * @author bratseth */ -public class DeploymentInstancesSpec extends DeploymentSpec.Step { +public class DeploymentInstanceSpec extends DeploymentSpec.Step { - /** The instances deployed in this step */ - private final List<InstanceName> ids; + /** The name of the instance this step deploys */ + private final InstanceName name; private final List<DeploymentSpec.Step> steps; private final DeploymentSpec.UpgradePolicy upgradePolicy; @@ -36,16 +36,16 @@ public class DeploymentInstancesSpec extends DeploymentSpec.Step { private final Notifications notifications; private final List<Endpoint> endpoints; - public DeploymentInstancesSpec(List<InstanceName> ids, - List<DeploymentSpec.Step> steps, - DeploymentSpec.UpgradePolicy upgradePolicy, - List<DeploymentSpec.ChangeBlocker> changeBlockers, - Optional<String> globalServiceId, - Optional<AthenzDomain> athenzDomain, - Optional<AthenzService> athenzService, - Notifications notifications, - List<Endpoint> endpoints) { - this.ids = List.copyOf(ids); + public DeploymentInstanceSpec(InstanceName name, + List<DeploymentSpec.Step> steps, + DeploymentSpec.UpgradePolicy upgradePolicy, + List<DeploymentSpec.ChangeBlocker> changeBlockers, + Optional<String> globalServiceId, + Optional<AthenzDomain> athenzDomain, + Optional<AthenzService> athenzService, + Notifications notifications, + List<Endpoint> endpoints) { + this.name = name; this.steps = List.copyOf(completeSteps(new ArrayList<>(steps))); this.upgradePolicy = upgradePolicy; this.changeBlockers = changeBlockers; @@ -59,7 +59,7 @@ public class DeploymentInstancesSpec extends DeploymentSpec.Step { validateAthenz(); } - public List<InstanceName> names() { return ids; } + public InstanceName name() { return name; } /** Adds missing required steps and reorders steps to a permissible order */ private static List<DeploymentSpec.Step> completeSteps(List<DeploymentSpec.Step> steps) { @@ -255,7 +255,7 @@ public class DeploymentInstancesSpec extends DeploymentSpec.Step { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - DeploymentInstancesSpec other = (DeploymentInstancesSpec) o; + DeploymentInstanceSpec other = (DeploymentInstanceSpec) o; return globalServiceId.equals(other.globalServiceId) && upgradePolicy == other.upgradePolicy && changeBlockers.equals(other.changeBlockers) && @@ -273,7 +273,7 @@ public class DeploymentInstancesSpec extends DeploymentSpec.Step { @Override public String toString() { - return "instance" + ( ids.size() < 1 ? " " : "s " ) + ids; + return "instance '" + name + "'"; } } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index e54371e714d..db383a1bea5 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -13,13 +13,10 @@ import java.io.FileReader; import java.io.Reader; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; /** @@ -67,15 +64,15 @@ public class DeploymentSpec { Optional<AthenzDomain> athenzDomain, Optional<AthenzService> athenzService, Notifications notifications, List<Endpoint> endpoints) { - this(List.of(new DeploymentInstancesSpec(List.of(InstanceName.from("default")), - steps, - upgradePolicy, - changeBlockers, - globalServiceId, - athenzDomain, - athenzService, - notifications, - endpoints)), + this(List.of(new DeploymentInstanceSpec(InstanceName.from("default"), + steps, + upgradePolicy, + changeBlockers, + globalServiceId, + athenzDomain, + athenzService, + notifications, + endpoints)), majorVersion, xmlForm); } @@ -89,18 +86,11 @@ public class DeploymentSpec { } // TODO: Remove after October 2019 - private DeploymentInstancesSpec defaultInstance() { - if (hasDefaultInstanceStepOnly()) return (DeploymentInstancesSpec)steps.get(0); + private DeploymentInstanceSpec defaultInstance() { + if (instances().size() == 1) return (DeploymentInstanceSpec)steps.get(0); throw new IllegalArgumentException("This deployment spec does not support the legacy API " + - "as it does not consist only of a default instance. Content: " + - steps.stream().map(Step::toString).collect(Collectors.joining(","))); - } - - // TODO: Remove after October 2019 - private boolean hasDefaultInstanceStepOnly() { - return steps.size() == 1 - && (steps.get(0) instanceof DeploymentInstancesSpec) - && ((DeploymentInstancesSpec)steps.get(0)).names().equals(List.of(InstanceName.from("default"))); + "as it has multiple instances: " + + instances().stream().map(Step::toString).collect(Collectors.joining(","))); } // TODO: Remove after October 2019 @@ -112,61 +102,86 @@ public class DeploymentSpec { /** Returns the major version this application is pinned to, or empty (default) to allow all major versions */ public Optional<Integer> majorVersion() { return majorVersion; } - // TODO: Remove after October 2019 + // TODO: Remove after November 2019 public boolean canUpgradeAt(Instant instant) { return defaultInstance().canUpgradeAt(instant); } - // TODO: Remove after October 2019 + // TODO: Remove after November 2019 public boolean canChangeRevisionAt(Instant instant) { return defaultInstance().canChangeRevisionAt(instant); } - // TODO: Remove after October 2019 + // TODO: Remove after November 2019 public List<ChangeBlocker> changeBlocker() { return defaultInstance().changeBlocker(); } /** Returns the deployment steps of this in the order they will be performed */ public List<Step> steps() { - if (hasDefaultInstanceStepOnly()) return defaultInstance().steps(); // TODO: Remove line after October 2019 + if (steps.size() == 1) return defaultInstance().steps(); // TODO: Remove line after November 2019 return steps; } - // TODO: Remove after October 2019 + // TODO: Remove after November 2019 public List<DeclaredZone> zones() { return defaultInstance().steps().stream() .flatMap(step -> step.zones().stream()) .collect(Collectors.toList()); } - // TODO: Remove after October 2019 + // TODO: Remove after November 2019 public Optional<AthenzDomain> athenzDomain() { return defaultInstance().athenzDomain(); } - // TODO: Remove after October 2019 + // TODO: Remove after November 2019 public Optional<AthenzService> athenzService(Environment environment, RegionName region) { return defaultInstance().athenzService(environment, region); } - // TODO: Remove after October 2019 + // TODO: Remove after November 2019 public Notifications notifications() { return defaultInstance().notifications(); } - // TODO: Remove after October 2019 + // TODO: Remove after November 2019 public List<Endpoint> endpoints() { return defaultInstance().endpoints(); } /** Returns the XML form of this spec, or null if it was not created by fromXml, nor is empty */ public String xmlForm() { return xmlForm; } - // TODO: Remove after October 2019 + // TODO: Remove after November 2019 public boolean includes(Environment environment, Optional<RegionName> region) { return defaultInstance().deploysTo(environment, region); } /** Returns the instance step containing the given instance name, or null if not present */ - public DeploymentInstancesSpec instance(String name) { + public DeploymentInstanceSpec instance(String name) { + return instance(InstanceName.from(name)); + } + + /** Returns the instance step containing the given instance name, or null if not present */ + public DeploymentInstanceSpec instance(InstanceName name) { for (Step step : steps) { - if ( ! (step instanceof DeploymentInstancesSpec)) continue; - DeploymentInstancesSpec instanceStep = (DeploymentInstancesSpec)step; - if (instanceStep.names().contains(InstanceName.from(name))) + if ( ! (step instanceof DeploymentInstanceSpec)) continue; + DeploymentInstanceSpec instanceStep = (DeploymentInstanceSpec)step; + if (instanceStep.name().equals(name)) return instanceStep; } return null; } + /** Returns the instance step containing the given instance name, or throws an IllegalArgumentException if not present */ + public DeploymentInstanceSpec requireInstance(String name) { + return requireInstance(InstanceName.from(name)); + } + + public DeploymentInstanceSpec requireInstance(InstanceName name) { + DeploymentInstanceSpec instance = instance(name); + if (instance == null) + throw new IllegalArgumentException("No instance '" + name + "' in deployment.xml'. Instances: " + + instances().stream().map(spec -> spec.name().toString()).collect(Collectors.joining(","))); + return instance; + } + + /** Returns the steps of this which are instances */ + public List<DeploymentInstanceSpec> instances() { + return steps.stream() + .filter(step -> step instanceof DeploymentInstanceSpec).map(DeploymentInstanceSpec.class::cast) + .collect(Collectors.toList()); + } + /** * Creates a deployment spec from XML. * diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java index acecb320b83..4d6495482ea 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application.api.xml; -import com.yahoo.config.application.api.DeploymentInstancesSpec; +import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.DeploymentSpec.DeclaredZone; import com.yahoo.config.application.api.DeploymentSpec.Delay; @@ -53,6 +53,7 @@ public class DeploymentSpecXmlReader { private static final String endpointTag = "endpoint"; private static final String athenzServiceAttribute = "athenz-service"; + private static final String athenzDomainAttribute = "athenz-domain"; private static final String testerFlavorAttribute = "tester-flavor"; private final boolean validate; @@ -86,7 +87,7 @@ public class DeploymentSpecXmlReader { List<Step> steps = new ArrayList<>(); if ( ! hasChildTag(instanceTag, root)) { // deployment spec skipping explicit instance -> "default" instance - steps.add(readInstanceContent("default", root)); + steps.addAll(readInstanceContent("default", root, root)); } else { if (hasChildTag(prodTag, root)) @@ -96,7 +97,7 @@ public class DeploymentSpecXmlReader { for (Element topLevelTag : XML.getChildren(root)) { if (topLevelTag.getTagName().equals(instanceTag)) - steps.add(readInstanceContent(topLevelTag.getAttribute("id"), topLevelTag)); + steps.addAll(readInstanceContent(topLevelTag.getAttribute("id"), topLevelTag, root)); else steps.addAll(readNonInstanceSteps(topLevelTag, new MutableOptional<>(), topLevelTag)); // (No global service id here) } @@ -110,33 +111,38 @@ public class DeploymentSpecXmlReader { /** * Reads the content of an (implicit or explicit) instance tag producing an instances step * - * @param instanceIdString a comma-separated list of the ids of the instance id(s) this is for - * @param instanceElement the element having the content of this instance + * @param instanceNameString a comma-separated list of the names of the instances this is for + * @param instanceTag the element having the content of this instance + * @param parentTag the parent of instanceTag (or the same, if this instances is implicitly defined which means instanceTag is the root) + * @return the instances specified, one for each instance name element */ - private DeploymentInstancesSpec readInstanceContent(String instanceIdString, Element instanceElement) { + private List<DeploymentInstanceSpec> readInstanceContent(String instanceNameString, Element instanceTag, Element parentTag) { if (validate) - validateTagOrder(instanceElement); + validateTagOrder(instanceTag); - List<InstanceName> instanceNames = Arrays.stream(instanceIdString.split("'")) - .map(InstanceName::from) - .collect(Collectors.toList()); MutableOptional<String> globalServiceId = new MutableOptional<>(); // Deprecated: Set of prod, but belongs to instance - Optional<AthenzDomain> athenzDomain = stringAttribute("athenz-domain", instanceElement).map(AthenzDomain::from); - Optional<AthenzService> athenzService = stringAttribute("athenz-service", instanceElement).map(AthenzService::from); + Optional<AthenzDomain> athenzDomain = stringAttribute(athenzDomainAttribute, instanceTag) + .or(() -> stringAttribute(athenzDomainAttribute, parentTag)) + .map(AthenzDomain::from); + Optional<AthenzService> athenzService = stringAttribute(athenzServiceAttribute, instanceTag) + .or(() -> stringAttribute(athenzServiceAttribute, parentTag)) + .map(AthenzService::from); List<Step> steps = new ArrayList<>(); - for (Element instanceChild : XML.getChildren(instanceElement)) + for (Element instanceChild : XML.getChildren(instanceTag)) steps.addAll(readNonInstanceSteps(instanceChild, globalServiceId, instanceChild)); - return new DeploymentInstancesSpec(instanceNames, - steps, - readUpgradePolicy(instanceElement), - readChangeBlockers(instanceElement), - globalServiceId.asOptional(), - athenzDomain, - athenzService, - readNotifications(instanceElement), - readEndpoints(instanceElement)); + return Arrays.stream(instanceNameString.split("'")) + .map(name -> new DeploymentInstanceSpec(InstanceName.from(name), + steps, + readUpgradePolicy(instanceTag), + readChangeBlockers(instanceTag), + globalServiceId.asOptional(), + athenzDomain, + athenzService, + readNotifications(instanceTag), + readEndpoints(instanceTag))) + .collect(Collectors.toList()); } // Consume the give tag as 0-N steps. 0 if it is not a step, >1 if it contains multiple nested steps that should be flattened diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index a7625b42e8d..4923ea396a0 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -152,7 +152,7 @@ public class DeploymentSpecTest { assertFalse(((DeploymentSpec.DeclaredZone)spec.instance("default").steps().get(2)).active()); assertTrue(spec.instance("default").steps().get(3) instanceof DeploymentSpec.Delay); - assertEquals(3 * 60 * 60 + 30 * 60, ((DeploymentSpec.Delay)spec.instance("default").steps().get(3)).duration().getSeconds()); + assertEquals(3 * 60 * 60 + 30 * 60, spec.instance("default").steps().get(3).delay().getSeconds()); assertTrue(spec.instance("default").steps().get(4).deploysTo(Environment.prod, Optional.of(RegionName.from("us-west1")))); assertTrue(((DeploymentSpec.DeclaredZone)spec.instance("default").steps().get(4)).active()); @@ -274,7 +274,7 @@ public class DeploymentSpecTest { @Test public void testEmpty() { - assertFalse(DeploymentSpec.empty.globalServiceId().isPresent()); + assertFalse(DeploymentSpec.empty.instance("default").globalServiceId().isPresent()); assertEquals(DeploymentSpec.UpgradePolicy.defaultPolicy, DeploymentSpec.empty.upgradePolicy()); assertTrue(DeploymentSpec.empty.steps().isEmpty()); assertEquals("<deployment version='1.0'/>", DeploymentSpec.empty.xmlForm()); @@ -391,18 +391,20 @@ public class DeploymentSpecTest { assertTrue(spec.instance("default").canUpgradeAt(Instant.parse("2017-09-23T10:15:30.00Z"))); } - @Test(expected = IllegalArgumentException.class) - public void athenz_config_is_disallowed_on_deployment_if_instances() { + @Test + public void athenz_config_is_read_from_deployment() { StringReader r = new StringReader( - "<deployment athenz-domain='domain' athenz-service='service''>" + - " <instance id='default'>" + + "<deployment athenz-domain='domain' athenz-service='service'>" + + " <instance id='instance1'>" + " <prod>" + " <region active='true'>us-west-1</region>" + " </prod>" + " </instance>" + "</deployment>" ); - DeploymentSpec.fromXml(r); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(spec.instance("instance1").athenzDomain().get().value(), "domain"); + assertEquals(spec.instance("instance1").athenzService(Environment.prod, RegionName.from("us-west-1")).get().value(), "service"); } @Test diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java index 732e1196995..33ef3f4bea8 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java @@ -141,7 +141,7 @@ public class DeploymentSpecWithoutInstanceTest { assertFalse(((DeploymentSpec.DeclaredZone)spec.instance("default").steps().get(2)).active()); assertTrue(spec.instance("default").steps().get(3) instanceof DeploymentSpec.Delay); - assertEquals(3 * 60 * 60 + 30 * 60, ((DeploymentSpec.Delay)spec.instance("default").steps().get(3)).duration().getSeconds()); + assertEquals(3 * 60 * 60 + 30 * 60, spec.instance("default").steps().get(3).delay().getSeconds()); assertTrue(spec.instance("default").steps().get(4).deploysTo(Environment.prod, Optional.of(RegionName.from("us-west1")))); assertTrue(((DeploymentSpec.DeclaredZone)spec.instance("default").steps().get(4)).active()); @@ -251,7 +251,7 @@ public class DeploymentSpecWithoutInstanceTest { @Test public void testEmpty() { - assertFalse(DeploymentSpec.empty.globalServiceId().isPresent()); + assertFalse(DeploymentSpec.empty.instance("default").globalServiceId().isPresent()); assertEquals(DeploymentSpec.UpgradePolicy.defaultPolicy, DeploymentSpec.empty.upgradePolicy()); assertTrue(DeploymentSpec.empty.steps().isEmpty()); assertEquals("<deployment version='1.0'/>", DeploymentSpec.empty.xmlForm()); @@ -414,7 +414,7 @@ public class DeploymentSpecWithoutInstanceTest { @Test public void noNotifications() { assertEquals(Notifications.none(), - DeploymentSpec.fromXml("<deployment />").notifications()); + DeploymentSpec.fromXml("<deployment />").instance("default").notifications()); } @Test @@ -455,7 +455,7 @@ public class DeploymentSpecWithoutInstanceTest { @Test public void noEndpoints() { - assertEquals(Collections.emptyList(), DeploymentSpec.fromXml("<deployment />").endpoints()); + assertEquals(Collections.emptyList(), DeploymentSpec.fromXml("<deployment />").instance("default").endpoints()); } @Test diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidator.java deleted file mode 100644 index 7757a8d4748..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidator.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation; - -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.container.ContainerCluster; -import com.yahoo.vespa.model.container.ContainerModel; - -import java.io.Reader; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Validates that deployment file (deployment.xml) has valid values (for now - * only global-service-id is validated) - * - * @author hmusum - */ -public class DeploymentFileValidator extends Validator { - - @Override - public void validate(VespaModel model, DeployState deployState) { - Optional<Reader> deployment = deployState.getApplicationPackage().getDeployment(); - - if (deployment.isPresent()) { - Reader deploymentReader = deployment.get(); - DeploymentSpec deploymentSpec = DeploymentSpec.fromXml(deploymentReader); - final Optional<String> globalServiceId = deploymentSpec.globalServiceId(); - if (globalServiceId.isPresent()) { - Set<ContainerCluster> containerClusters = model.getRoot().configModelRepo().getModels(ContainerModel.class).stream(). - map(ContainerModel::getCluster).filter(cc -> cc.getName().equals(globalServiceId.get())).collect(Collectors.toSet()); - if (containerClusters.size() != 1) { - throw new IllegalArgumentException("global-service-id '" + globalServiceId.get() + "' specified in deployment.xml does not match any container cluster id"); - } - } - } - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java new file mode 100644 index 00000000000..ac38336a405 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java @@ -0,0 +1,40 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.application.api.DeploymentInstanceSpec; +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.ContainerModel; + +import java.io.Reader; +import java.util.List; +import java.util.Optional; + +/** + * Validates that deployment spec (deployment.xml) has valid values (for now + * only global-service-id is validated) + * + * @author hmusum + * @author bratseth + */ +public class DeploymentSpecValidator extends Validator { + + @Override + public void validate(VespaModel model, DeployState deployState) { + Optional<Reader> deployment = deployState.getApplicationPackage().getDeployment(); + if ( deployment.isEmpty()) return; + + Reader deploymentReader = deployment.get(); + DeploymentSpec deploymentSpec = DeploymentSpec.fromXml(deploymentReader); + List<ContainerModel> containers = model.getRoot().configModelRepo().getModels(ContainerModel.class); + for (DeploymentInstanceSpec instance : deploymentSpec.instances()) { + instance.globalServiceId().ifPresent(globalServiceId -> { + if ( containers.stream().noneMatch(container -> container.getCluster().getName().equals(globalServiceId))) + throw new IllegalArgumentException("The global-service-id in " + instance + ", '" + globalServiceId + + "' specified in deployment.xml does not match any container cluster id"); + }); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 042c7cc867c..7d0d068f9d6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -53,7 +53,7 @@ public class Validation { new StreamingValidator().validate(model, deployState); new RankSetupValidator(validationParameters.ignoreValidationErrors()).validate(model, deployState); new NoPrefixForIndexes().validate(model, deployState); - new DeploymentFileValidator().validate(model, deployState); + new DeploymentSpecValidator().validate(model, deployState); new RankingConstantsValidator().validate(model, deployState); new SecretStoreValidator().validate(model, deployState); new TlsSecretsValidator().validate(model, deployState); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index f4c7f49a9a0..caf84d88cf4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -197,7 +197,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addClientProviders(deployState, spec, cluster); addServerProviders(deployState, spec, cluster); - addAthensCopperArgos(cluster, context); // Must be added after nodes. } @@ -228,14 +227,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } private void addRotationProperties(ApplicationContainerCluster cluster, Zone zone, Set<Rotation> rotations, Set<ContainerEndpoint> endpoints, DeploymentSpec spec) { + Optional<String> globalServiceId = spec.requireInstance(app.getApplicationId().instance()).globalServiceId(); cluster.getContainers().forEach(container -> { - setRotations(container, rotations, endpoints, spec.globalServiceId(), cluster.getName()); + setRotations(container, rotations, endpoints, globalServiceId, cluster.getName()); container.setProp("activeRotation", Boolean.toString(zoneHasActiveRotation(zone, spec))); }); } private boolean zoneHasActiveRotation(Zone zone, DeploymentSpec spec) { - return spec.zones().stream() + return spec.requireInstance(app.getApplicationId().instance()).zones().stream() .anyMatch(declaredZone -> declaredZone.deploysTo(zone.environment(), Optional.of(zone.region())) && declaredZone.active()); } @@ -893,8 +893,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Zone zone, DeploymentSpec spec) { spec.athenzDomain().ifPresent(domain -> { - AthenzService service = spec.athenzService(zone.environment(), zone.region()) - .orElseThrow(() -> new RuntimeException("Missing Athenz service configuration")); + AthenzService service = spec.requireInstance(app.getApplicationId().instance()).athenzService(zone.environment(), zone.region()) + .orElseThrow(() -> new RuntimeException("Missing Athenz service configuration in instance '" + app.getApplicationId().instance() + "'")); String zoneDnsSuffix = zone.environment().value() + "-" + zone.region().value() + "." + athenzDnsSuffix; IdentityProvider identityProvider = new IdentityProvider(domain, service, getLoadBalancerName(loadBalancerName, configServerSpecs), ztsUrl, zoneDnsSuffix, zone); cluster.addComponent(identityProvider); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidatorTest.java index 5fc3f815b09..c6d56455d44 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidatorTest.java @@ -18,7 +18,7 @@ import static org.junit.Assert.fail; /** * @author hmusum */ -public class DeploymentFileValidatorTest { +public class DeploymentSpecValidatorTest { @Test public void testDeploymentWithNonExistentGlobalId() throws IOException, SAXException { @@ -58,7 +58,7 @@ public class DeploymentFileValidatorTest { try { final DeployState deployState = builder.build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new DeploymentFileValidator().validate(model, deployState); + new DeploymentSpecValidator().validate(model, deployState); fail("Did not get expected exception"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), containsString("specified in deployment.xml does not match any container cluster id")); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 54c96c0461d..96df067843d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -9,6 +9,7 @@ import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ConfigDefinitionRepo; @@ -121,7 +122,8 @@ public class SessionPreparer { preparation.writeTlsZK(); var globalServiceId = context.getApplicationPackage().getDeployment() .map(DeploymentSpec::fromXml) - .flatMap(DeploymentSpec::globalServiceId); + .map(spec -> spec.requireInstance(context.getApplicationPackage().getApplicationId().instance())) + .flatMap(DeploymentInstanceSpec::globalServiceId); preparation.writeContainerEndpointsZK(globalServiceId); preparation.distribute(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index 8b8be1a27d7..f565111c363 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -81,7 +81,7 @@ public class ZKApplicationPackageTest { assertThat(readInfo.getHosts().iterator().next().flavor(), is(TEST_FLAVOR)); assertEquals("6.0.1", readInfo.getHosts().iterator().next().version().get().toString()); assertTrue(zkApp.getDeployment().isPresent()); - assertThat(DeploymentSpec.fromXml(zkApp.getDeployment().get()).globalServiceId().get(), is("mydisc")); + assertEquals("mydisc", DeploymentSpec.fromXml(zkApp.getDeployment().get()).instance("default").globalServiceId().get()); } private void feed(ConfigCurator zk, File dirToFeed) throws IOException { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index c012abff670..43e0a07eae1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -364,10 +364,9 @@ public class ApplicationController { verifyApplicationIdentityConfiguration(applicationId.tenant(), applicationPackage, deployingIdentity); if (zone.environment().isProduction()) // Assign and register endpoints - application = withRotation(application, instance); - - endpoints = registerEndpointsInDns(application.get().deploymentSpec(), application.get().require(applicationId.instance()), zone); + application = withRotation(applicationPackage.deploymentSpec(), application, instance); + endpoints = registerEndpointsInDns(applicationPackage.deploymentSpec(), application.get().require(applicationId.instance()), zone); if (controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) { // Provisions a new certificate if missing @@ -497,9 +496,9 @@ public class ApplicationController { } /** Makes sure the application has a global rotation, if eligible. */ - private LockedApplication withRotation(LockedApplication application, InstanceName instanceName) { + private LockedApplication withRotation(DeploymentSpec deploymentSpec, LockedApplication application, InstanceName instanceName) { try (RotationLock rotationLock = rotationRepository.lock()) { - var rotations = rotationRepository.getOrAssignRotations(application.get().deploymentSpec(), + var rotations = rotationRepository.getOrAssignRotations(deploymentSpec, application.get().require(instanceName), rotationLock); application = application.with(instanceName, instance -> instance.with(rotations)); @@ -515,7 +514,7 @@ public class ApplicationController { */ private Set<ContainerEndpoint> registerEndpointsInDns(DeploymentSpec deploymentSpec, Instance instance, ZoneId zone) { var containerEndpoints = new HashSet<ContainerEndpoint>(); - var registerLegacyNames = deploymentSpec.globalServiceId().isPresent(); + var registerLegacyNames = deploymentSpec.requireInstance(instance.name()).globalServiceId().isPresent(); for (var assignedRotation : instance.rotations()) { var names = new ArrayList<String>(); var endpoints = instance.endpointsIn(controller.system(), assignedRotation.endpointId()) @@ -607,8 +606,8 @@ public class ApplicationController { private LockedApplication withoutDeletedDeployments(LockedApplication application, InstanceName instance) { DeploymentSpec deploymentSpec = application.get().deploymentSpec(); List<Deployment> deploymentsToRemove = application.get().require(instance).productionDeployments().values().stream() - .filter(deployment -> ! deploymentSpec.includes(deployment.zone().environment(), - Optional.of(deployment.zone().region()))) + .filter(deployment -> ! deploymentSpec.requireInstance(instance).includes(deployment.zone().environment(), + Optional.of(deployment.zone().region()))) .collect(Collectors.toList()); if (deploymentsToRemove.isEmpty()) return application; @@ -632,7 +631,7 @@ public class ApplicationController { private Instance withoutUnreferencedDeploymentJobs(DeploymentSpec deploymentSpec, Instance instance) { for (JobType job : JobList.from(instance).production().mapToList(JobStatus::type)) { ZoneId zone = job.zone(controller.system()); - if (deploymentSpec.includes(zone.environment(), Optional.of(zone.region()))) + if (deploymentSpec.requireInstance(instance.name()).includes(zone.environment(), Optional.of(zone.region()))) continue; instance = instance.withoutDeploymentJob(job); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentSpecValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentSpecValidator.java index ce7904dc829..5c4d5874e53 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentSpecValidator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentSpecValidator.java @@ -39,7 +39,7 @@ public class DeploymentSpecValidator { /** Verify that each of the production zones listed in the deployment spec exist in this system */ private void validateSteps(DeploymentSpec deploymentSpec) { new DeploymentSteps(deploymentSpec, controller::system).jobs(); - deploymentSpec.zones().stream() + deploymentSpec.instances().stream().flatMap(instance -> instance.zones().stream()) .filter(zone -> zone.environment() == Environment.prod) .forEach(zone -> { if ( ! controller.zoneRegistry().hasZone(ZoneId.from(zone.environment(), @@ -51,16 +51,19 @@ public class DeploymentSpecValidator { /** Verify that no single endpoint contains regions in different clouds */ private void validateEndpoints(DeploymentSpec deploymentSpec) { - for (var endpoint : deploymentSpec.endpoints()) { - var clouds = new HashSet<CloudName>(); - for (var region : endpoint.regions()) { - for (ZoneApi zone : controller.zoneRegistry().zones().all().in(region).zones()) { - clouds.add(zone.getCloudName()); + for (var instance : deploymentSpec.instances()) { + for (var endpoint : instance.endpoints()) { + var clouds = new HashSet<CloudName>(); + for (var region : endpoint.regions()) { + for (ZoneApi zone : controller.zoneRegistry().zones().all().in(region).zones()) { + clouds.add(zone.getCloudName()); + } + } + if (clouds.size() != 1) { + throw new IllegalArgumentException("Endpoint '" + endpoint.endpointId() + "' in " + instance + + " cannot contain regions in different clouds: " + + endpoint.regions().stream().sorted().collect(Collectors.toList())); } - } - if (clouds.size() != 1) { - throw new IllegalArgumentException("Endpoint '" + endpoint.endpointId() + "' cannot contain regions in different clouds: " + - endpoint.regions().stream().sorted().collect(Collectors.toList())); } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 376048143d9..3df889d7a88 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -372,9 +372,8 @@ public class DeploymentTrigger { } else { // All jobs are complete; find the time of completion of this step. if (stepJobs.isEmpty()) { // No jobs means this is a delay step. - Duration delay = ((DeploymentSpec.Delay) step).duration(); - completedAt = completedAt.map(at -> at.plus(delay)).filter(at -> !at.isAfter(clock.instant())); - reason += " after a delay of " + delay; + completedAt = completedAt.map(at -> at.plus(step.delay())).filter(at -> !at.isAfter(clock.instant())); + reason += " after a delay of " + step.delay(); } else { completedAt = stepJobs.stream().map(job -> instance.deploymentJobs().statusOf(job).get().lastCompleted().get().at()).max(naturalOrder()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java index a16ca5cb201..9f6bbcd2a5a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java @@ -77,22 +77,25 @@ public class RotationRepository { * If a rotation is already assigned to the application, that rotation will be returned. * If no rotation is assigned, return an available rotation. The caller is responsible for assigning the rotation. * - * @param deploymentSpec The deployment spec for the application - * @param instance The instance requesting a rotation - * @param lock Lock which must be acquired by the caller + * @param deploymentSpec the deployment spec for the application + * @param instance the instance requesting a rotation + * @param lock lock which must be acquired by the caller */ public Rotation getOrAssignRotation(DeploymentSpec deploymentSpec, Instance instance, RotationLock lock) { if ( ! instance.rotations().isEmpty()) { return allRotations.get(instance.rotations().get(0).rotationId()); } - if (deploymentSpec.globalServiceId().isEmpty()) { - throw new IllegalArgumentException("global-service-id is not set in deployment spec"); + + if (deploymentSpec.requireInstance(instance.name()).globalServiceId().isEmpty()) { + throw new IllegalArgumentException("global-service-id is not set in deployment spec for instance '" + + instance.name() + "'"); } - long productionZones = deploymentSpec.zones().stream() - .filter(zone -> zone.deploysTo(Environment.prod)) - .count(); + long productionZones = deploymentSpec.requireInstance(instance.name()).zones().stream() + .filter(zone -> zone.deploysTo(Environment.prod)) + .count(); if (productionZones < 2) { - throw new IllegalArgumentException("global-service-id is set but less than 2 prod zones are defined"); + throw new IllegalArgumentException("global-service-id is set but less than 2 prod zones are defined " + + "in instance '" + instance.name() + "'"); } return findAvailableRotation(instance.id(), lock); } @@ -110,22 +113,23 @@ public class RotationRepository { * @return List of rotation assignments - either new or existing */ public List<AssignedRotation> getOrAssignRotations(DeploymentSpec deploymentSpec, Instance instance, RotationLock lock) { - if (deploymentSpec.globalServiceId().isPresent() && ! deploymentSpec.endpoints().isEmpty()) { + if (deploymentSpec.requireInstance(instance.name()).globalServiceId().isPresent() + && ! deploymentSpec.requireInstance(instance.name()).endpoints().isEmpty()) { throw new IllegalArgumentException("Cannot provision rotations with both global-service-id and 'endpoints'"); } // Support the older case of setting global-service-id - if (deploymentSpec.globalServiceId().isPresent()) { - final var regions = deploymentSpec.zones().stream() - .filter(zone -> zone.environment().isProduction()) - .flatMap(zone -> zone.region().stream()) - .collect(Collectors.toSet()); + if (deploymentSpec.requireInstance(instance.name()).globalServiceId().isPresent()) { + var regions = deploymentSpec.requireInstance(instance.name()).zones().stream() + .filter(zone -> zone.environment().isProduction()) + .flatMap(zone -> zone.region().stream()) + .collect(Collectors.toSet()); - final var rotation = getOrAssignRotation(deploymentSpec, instance, lock); + var rotation = getOrAssignRotation(deploymentSpec, instance, lock); return List.of( new AssignedRotation( - new ClusterSpec.Id(deploymentSpec.globalServiceId().get()), + new ClusterSpec.Id(deploymentSpec.requireInstance(instance.name()).globalServiceId().get()), EndpointId.default_(), rotation.id(), regions @@ -133,8 +137,8 @@ public class RotationRepository { ); } - final Map<EndpointId, AssignedRotation> existingAssignments = existingEndpointAssignments(deploymentSpec, instance); - final Map<EndpointId, AssignedRotation> updatedAssignments = assignRotationsToEndpoints(deploymentSpec, existingAssignments, lock); + Map<EndpointId, AssignedRotation> existingAssignments = existingEndpointAssignments(deploymentSpec, instance); + Map<EndpointId, AssignedRotation> updatedAssignments = assignRotationsToEndpoints(deploymentSpec, existingAssignments, lock); existingAssignments.putAll(updatedAssignments); @@ -142,11 +146,11 @@ public class RotationRepository { } private Map<EndpointId, AssignedRotation> assignRotationsToEndpoints(DeploymentSpec deploymentSpec, Map<EndpointId, AssignedRotation> existingAssignments, RotationLock lock) { - final var availableRotations = new ArrayList<>(availableRotations(lock).values()); + var availableRotations = new ArrayList<>(availableRotations(lock).values()); - final var neededRotations = deploymentSpec.endpoints().stream() - .filter(Predicate.not(endpoint -> existingAssignments.containsKey(EndpointId.of(endpoint.endpointId())))) - .collect(Collectors.toSet()); + var neededRotations = deploymentSpec.endpoints().stream() + .filter(Predicate.not(endpoint -> existingAssignments.containsKey(EndpointId.of(endpoint.endpointId())))) + .collect(Collectors.toSet()); if (neededRotations.size() > availableRotations.size()) { throw new IllegalStateException("Hosted Vespa ran out of rotations, unable to assign rotation: need " + neededRotations.size() + ", have " + availableRotations.size()); @@ -172,34 +176,26 @@ public class RotationRepository { } private Map<EndpointId, AssignedRotation> existingEndpointAssignments(DeploymentSpec deploymentSpec, Instance instance) { - // // Get the regions that has been configured for an endpoint. Empty set if the endpoint // is no longer mentioned in the configuration file. - // - final Function<EndpointId, Set<RegionName>> configuredRegionsForEndpoint = endpointId -> { - return deploymentSpec.endpoints().stream() + Function<EndpointId, Set<RegionName>> configuredRegionsForEndpoint = endpointId -> + deploymentSpec.requireInstance(instance.name()).endpoints().stream() .filter(endpoint -> endpointId.id().equals(endpoint.endpointId())) .map(Endpoint::regions) .findFirst() .orElse(Set.of()); - }; - // // Build a new AssignedRotation instance where we update set of regions from the configuration instead - // of using the one already mentioned in the assignment. This allows us to overwrite the set of regions - // when - final Function<AssignedRotation, AssignedRotation> assignedRotationWithConfiguredRegions = assignedRotation -> { - return new AssignedRotation( + // of using the one already mentioned in the assignment. This allows us to overwrite the set of regions. + Function<AssignedRotation, AssignedRotation> assignedRotationWithConfiguredRegions = assignedRotation -> + new AssignedRotation( assignedRotation.clusterId(), assignedRotation.endpointId(), assignedRotation.rotationId(), - configuredRegionsForEndpoint.apply(assignedRotation.endpointId()) - ); - }; + configuredRegionsForEndpoint.apply(assignedRotation.endpointId())); return instance.rotations().stream() - .collect( - Collectors.toMap( + .collect(Collectors.toMap( AssignedRotation::endpointId, assignedRotationWithConfiguredRegions, (a, b) -> { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index a4f3a804c55..fab1ed2ab20 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -72,7 +72,6 @@ public class ControllerTest { @Test public void testDeployment() { // Setup system - ApplicationController applications = tester.controller().applications(); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) .region("us-west-1") @@ -753,7 +752,7 @@ public class ControllerTest { tester.deployCompletely(application, applicationPackage); fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("Endpoint 'default' cannot contain regions in different clouds: [aws-us-east-1, us-west-1]", e.getMessage()); + assertEquals("Endpoint 'default' in instance 'default' cannot contain regions in different clouds: [aws-us-east-1, us-west-1]", e.getMessage()); } var applicationPackage2 = new ApplicationPackageBuilder() @@ -766,7 +765,7 @@ public class ControllerTest { tester.deployCompletely(application, applicationPackage2); fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("Endpoint 'foo' cannot contain regions in different clouds: [aws-us-east-1, us-west-1]", e.getMessage()); + assertEquals("Endpoint 'foo' in instance 'default' cannot contain regions in different clouds: [aws-us-east-1, us-west-1]", e.getMessage()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index 25e562ed046..9449f2b0854 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -47,6 +47,7 @@ public class ApplicationPackageBuilder { private final List<X509Certificate> trustedCertificates = new ArrayList<>(); private OptionalInt majorVersion = OptionalInt.empty(); + private String instances = "default"; private String upgradePolicy = null; private Environment environment = Environment.prod; private String globalServiceId = null; @@ -58,6 +59,11 @@ public class ApplicationPackageBuilder { return this; } + public ApplicationPackageBuilder instances(String instances) { + this.instances = instances; + return this; + } + public ApplicationPackageBuilder upgradePolicy(String upgradePolicy) { this.upgradePolicy = upgradePolicy; return this; @@ -90,7 +96,7 @@ public class ApplicationPackageBuilder { } public ApplicationPackageBuilder region(String regionName) { - environmentBody.append(" <region active='true'>"); + environmentBody.append(" <region active='true'>"); environmentBody.append(regionName); environmentBody.append("</region>\n"); return this; @@ -112,7 +118,7 @@ public class ApplicationPackageBuilder { public ApplicationPackageBuilder blockChange(boolean revision, boolean version, String daySpec, String hourSpec, String zoneSpec) { - blockChange.append(" <block-change"); + blockChange.append(" <block-change"); blockChange.append(" revision='").append(revision).append("'"); blockChange.append(" version='").append(version).append("'"); blockChange.append(" days='").append(daySpec).append("'"); @@ -166,14 +172,15 @@ public class ApplicationPackageBuilder { xml.append(athenzIdentityAttributes); } xml.append(">\n"); + xml.append(" <instance id='").append(instances).append("'>\n"); if (upgradePolicy != null) { - xml.append("<upgrade policy='"); + xml.append(" <upgrade policy='"); xml.append(upgradePolicy); xml.append("'/>\n"); } xml.append(notifications); xml.append(blockChange); - xml.append(" <"); + xml.append(" <"); xml.append(environment.value()); if (globalServiceId != null) { xml.append(" global-service-id='"); @@ -182,13 +189,14 @@ public class ApplicationPackageBuilder { } xml.append(">\n"); xml.append(environmentBody); - xml.append(" </"); + xml.append(" </"); xml.append(environment.value()); xml.append(">\n"); - xml.append(" <endpoints>\n"); + xml.append(" <endpoints>\n"); xml.append(endpointsBody); - xml.append(" </endpoints>\n"); - xml.append("</deployment>"); + xml.append(" </endpoints>\n"); + xml.append(" </instance>\n"); + xml.append("</deployment>\n"); return xml.toString().getBytes(UTF_8); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 7e00a2f9f64..3ff21bb2261 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -112,7 +112,18 @@ public class ApplicationApiTest extends ControllerContainerTest { private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/"; - private static final ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + private static final ApplicationPackage applicationPackageDefault = new ApplicationPackageBuilder() + .instances("default") + .environment(Environment.prod) + .globalServiceId("foo") + .region("us-central-1") + .region("us-east-3") + .region("us-west-1") + .blockChange(false, true, "mon-fri", "0-8", "UTC") + .build(); + + private static final ApplicationPackage applicationPackageInstance1 = new ApplicationPackageBuilder() + .instances("instance1") .environment(Environment.prod) .globalServiceId("foo") .region("us-central-1") @@ -220,7 +231,7 @@ public class ApplicationApiTest extends ControllerContainerTest { addUserToHostedOperatorRole(HostedAthenzIdentities.from(HOSTED_VESPA_OPERATOR)); // POST (deploy) an application to a zone - manual user deployment (includes a content hash for verification) - MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1, true); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1/deploy", POST) .data(entity) .header("X-Content-Hash", Base64.getEncoder().encodeToString(Signatures.sha256Digest(entity::data))) @@ -240,7 +251,7 @@ public class ApplicationApiTest extends ControllerContainerTest { controllerTester.jobCompletion(JobType.component) .application(id) .projectId(screwdriverProjectId) - .uploadArtifact(applicationPackage) + .uploadArtifact(applicationPackageInstance1) .submit(); // ... systemtest @@ -304,6 +315,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST (create) another application ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .instances("instance1") .environment(Environment.prod) .region("us-west-1") .build(); @@ -575,6 +587,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // Second attempt has a service under a different domain than the tenant of the application, and fails. ApplicationPackage packageWithServiceForWrongDomain = new ApplicationPackageBuilder() + .instances("instance1") .environment(Environment.prod) .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from(ATHENZ_TENANT_DOMAIN_2.getName()), AthenzService.from("service")) .region("us-west-1") @@ -587,6 +600,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // Third attempt finally has a service under the domain of the tenant, and succeeds. ApplicationPackage packageWithService = new ApplicationPackageBuilder() + .instances("instance1") .environment(Environment.prod) .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from(ATHENZ_TENANT_DOMAIN.getName()), AthenzService.from("service")) .region("us-west-1") @@ -700,6 +714,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.computeVersionStatus(); createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .instances("instance1") .globalServiceId("foo") .region("us-west-1") .region("us-east-3") @@ -708,7 +723,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // Create tenant and deploy ApplicationId id = createTenantAndApplication(); long projectId = 1; - MultiPartStreamer deployData = createApplicationDeployData(Optional.empty(), false); + MultiPartStreamer deployData = createApplicationDeployData(Optional.of(applicationPackage), false); startAndTestChange(controllerTester, id, projectId, applicationPackage, deployData, 100); // us-west-1 @@ -771,6 +786,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.computeVersionStatus(); createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .instances("instance1") .region("us-west-1") .region("us-east-3") .region("eu-west-1") @@ -847,7 +863,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId("application1")); // POST (deploy) an application to a prod zone - allowed when project ID is not specified - MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1, true); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/deploy", POST) .data(entity) .screwdriverIdentity(SCREWDRIVER_ID), @@ -879,6 +895,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // Deploy ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .instances("instance1") .region("us-east-3") .build(); ApplicationId id = createTenantAndApplication(); @@ -898,6 +915,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // New zone is added before us-east-3 applicationPackage = new ApplicationPackageBuilder() + .instances("instance1") .globalServiceId("foo") // These decides the ordering of deploymentJobs and instances in the response .region("us-west-1") @@ -1050,7 +1068,7 @@ public class ApplicationApiTest extends ControllerContainerTest { configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Failed to prepare application", ConfigServerException.ErrorCode.INVALID_APPLICATION_PACKAGE, null)); // POST (deploy) an application with an invalid application package - MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1, true); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1/deploy", POST) .data(entity) .userIdentity(USER_ID), @@ -1170,7 +1188,7 @@ public class ApplicationApiTest extends ControllerContainerTest { 200); // Deploy to an authorized zone by a user tenant is disallowed - MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackageDefault, true); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/deploy", POST) .data(entity) .userIdentity(USER_ID), @@ -1583,7 +1601,7 @@ public class ApplicationApiTest extends ControllerContainerTest { } private MultiPartStreamer createApplicationDeployData(Optional<ApplicationPackage> applicationPackage, - Optional<ApplicationVersion> applicationVersion, boolean deployDirectly) { + Optional<ApplicationVersion> applicationVersion, boolean deployDirectly) { MultiPartStreamer streamer = new MultiPartStreamer(); streamer.addJson("deployOptions", deployOptions(deployDirectly, applicationVersion)); applicationPackage.ifPresent(ap -> streamer.addBytes("applicationZip", ap.zippedContent())); |