diff options
126 files changed, 3938 insertions, 1619 deletions
diff --git a/abi-check-plugin/src/main/java/com/yahoo/abicheck/mojo/AbiCheck.java b/abi-check-plugin/src/main/java/com/yahoo/abicheck/mojo/AbiCheck.java index a09e23bbe9e..c5452ecdde3 100644 --- a/abi-check-plugin/src/main/java/com/yahoo/abicheck/mojo/AbiCheck.java +++ b/abi-check-plugin/src/main/java/com/yahoo/abicheck/mojo/AbiCheck.java @@ -192,7 +192,7 @@ public class AbiCheck extends AbstractMojo { } else { Map<String, JavaClassSignature> abiSpec = readSpec(specFile); if (!compareSignatures(abiSpec, signatures, getLog())) { - throw new MojoFailureException("ABI spec mismatch"); + throw new MojoFailureException("ABI spec mismatch. To update run 'mvn package -Dabicheck.writeSpec'"); } } } catch (IOException e) { diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/Certificates.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/Certificates.java index 447b6efb09b..a4cf54063ec 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/Certificates.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/Certificates.java @@ -43,7 +43,7 @@ public class Certificates { SHA256_WITH_ECDSA, X509CertificateBuilder.generateRandomSerialNumber()); for (var san : csr.getSubjectAlternativeNames()) { - builder = builder.addSubjectAlternativeName(san.getValue()); + builder = builder.addSubjectAlternativeName(san.decode()); } return builder.build(); } diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificateTester.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificateTester.java index 4946de93f6d..130a4ec5e66 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificateTester.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificateTester.java @@ -47,13 +47,16 @@ public class CertificateTester { return createCsr(null); } - public static Pkcs10Csr createCsr(String dnsName) { + public static Pkcs10Csr createCsr(String dnsName, String... ipAddresses) { X500Principal subject = new X500Principal("CN=subject"); KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); var builder = Pkcs10CsrBuilder.fromKeypair(subject, keyPair, SignatureAlgorithm.SHA512_WITH_ECDSA); if (dnsName != null) { builder = builder.addSubjectAlternativeName(SubjectAlternativeName.Type.DNS_NAME, dnsName); } + for (var ipAddress : ipAddresses) { + builder = builder.addSubjectAlternativeName(SubjectAlternativeName.Type.IP_ADDRESS, ipAddress); + } return builder.build(); } diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificatesTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificatesTest.java index 80940dcd02c..fa86979656d 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificatesTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/CertificatesTest.java @@ -40,13 +40,18 @@ public class CertificatesTest { public void add_san_from_csr() throws Exception { var certificates = new Certificates(new ManualClock()); var dnsName = "host.example.com"; - var csr = CertificateTester.createCsr(dnsName); + var ip = "192.0.2.42"; + var csr = CertificateTester.createCsr(dnsName, ip); var certificate = certificates.create(csr, caCertificate, keyPair.getPrivate()); assertNotNull(certificate.getSubjectAlternativeNames()); - assertEquals(1, certificate.getSubjectAlternativeNames().size()); + assertEquals(2, certificate.getSubjectAlternativeNames().size()); + + var subjectAlternativeNames = List.copyOf(certificate.getSubjectAlternativeNames()); assertEquals(List.of(SubjectAlternativeName.Type.DNS_NAME.getTag(), dnsName), - certificate.getSubjectAlternativeNames().iterator().next()); + subjectAlternativeNames.get(0)); + assertEquals(List.of(SubjectAlternativeName.Type.IP_ADDRESS.getTag(), ip), + subjectAlternativeNames.get(1)); } } diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 32c9e433157..d08cda06e5d 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -187,6 +187,35 @@ ], "fields": [] }, + "com.yahoo.config.application.api.DeploymentInstanceSpec": { + "superClass": "com.yahoo.config.application.api.DeploymentSpec$Step", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.config.provision.InstanceName, java.util.List, com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy, java.util.List, java.util.Optional, java.util.Optional, java.util.Optional, com.yahoo.config.application.api.Notifications, java.util.List)", + "public com.yahoo.config.provision.InstanceName name()", + "public java.time.Duration delay()", + "public java.util.List steps()", + "public com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy upgradePolicy()", + "public java.util.List changeBlocker()", + "public java.util.Optional globalServiceId()", + "public boolean canUpgradeAt(java.time.Instant)", + "public boolean canChangeRevisionAt(java.time.Instant)", + "public java.util.List zones()", + "public boolean deploysTo(com.yahoo.config.provision.Environment, java.util.Optional)", + "public java.util.Optional athenzDomain()", + "public java.util.Optional athenzService(com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", + "public com.yahoo.config.application.api.Notifications notifications()", + "public java.util.List endpoints()", + "public boolean includes(com.yahoo.config.provision.Environment, java.util.Optional)", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields": [] + }, "com.yahoo.config.application.api.DeploymentSpec$ChangeBlocker": { "superClass": "java.lang.Object", "interfaces": [], @@ -197,7 +226,8 @@ "public void <init>(boolean, boolean, com.yahoo.config.application.api.TimeWindow)", "public boolean blocksRevisions()", "public boolean blocksVersions()", - "public com.yahoo.config.application.api.TimeWindow window()" + "public com.yahoo.config.application.api.TimeWindow window()", + "public java.lang.String toString()" ], "fields": [] }, @@ -234,7 +264,9 @@ "methods": [ "public void <init>(java.time.Duration)", "public java.time.Duration duration()", - "public boolean deploysTo(com.yahoo.config.provision.Environment, java.util.Optional)" + "public java.time.Duration delay()", + "public boolean deploysTo(com.yahoo.config.provision.Environment, java.util.Optional)", + "public java.lang.String toString()" ], "fields": [] }, @@ -247,9 +279,11 @@ "methods": [ "public void <init>(java.util.List)", "public java.util.List zones()", + "public java.util.List steps()", "public boolean deploysTo(com.yahoo.config.provision.Environment, java.util.Optional)", "public boolean equals(java.lang.Object)", - "public int hashCode()" + "public int hashCode()", + "public java.lang.String toString()" ], "fields": [] }, @@ -264,7 +298,9 @@ "public void <init>()", "public final boolean deploysTo(com.yahoo.config.provision.Environment)", "public abstract boolean deploysTo(com.yahoo.config.provision.Environment, java.util.Optional)", - "public java.util.List zones()" + "public java.util.List zones()", + "public java.time.Duration delay()", + "public java.util.List steps()" ], "fields": [] }, @@ -293,6 +329,7 @@ "public" ], "methods": [ + "public void <init>(java.util.List, java.util.Optional, java.util.Optional, java.util.Optional, java.lang.String)", "public void <init>(java.util.Optional, com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy, java.util.Optional, java.util.List, java.util.List, java.lang.String, java.util.Optional, java.util.Optional, com.yahoo.config.application.api.Notifications, java.util.List)", "public java.util.Optional globalServiceId()", "public com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy upgradePolicy()", @@ -302,16 +339,21 @@ "public java.util.List changeBlocker()", "public java.util.List steps()", "public java.util.List zones()", + "public java.util.Optional athenzDomain()", + "public java.util.Optional athenzService(com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", + "public java.util.Optional athenzService(com.yahoo.config.provision.InstanceName, com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", "public com.yahoo.config.application.api.Notifications notifications()", "public java.util.List endpoints()", "public java.lang.String xmlForm()", "public boolean includes(com.yahoo.config.provision.Environment, java.util.Optional)", + "public java.util.Optional instance(com.yahoo.config.provision.InstanceName)", + "public com.yahoo.config.application.api.DeploymentInstanceSpec requireInstance(java.lang.String)", + "public com.yahoo.config.application.api.DeploymentInstanceSpec requireInstance(com.yahoo.config.provision.InstanceName)", + "public java.util.List instances()", "public static com.yahoo.config.application.api.DeploymentSpec fromXml(java.io.Reader)", "public static com.yahoo.config.application.api.DeploymentSpec fromXml(java.lang.String)", "public static com.yahoo.config.application.api.DeploymentSpec fromXml(java.lang.String, boolean)", "public static java.lang.String toMessageString(java.lang.Throwable)", - "public java.util.Optional athenzDomain()", - "public java.util.Optional athenzService(com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", "public boolean equals(java.lang.Object)", "public int hashCode()", "public static void main(java.lang.String[])" diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java new file mode 100644 index 00000000000..df611d66b87 --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java @@ -0,0 +1,254 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.application.api; + +import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.AthenzService; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.RegionName; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * The deployment spec for an application instance + * + * @author bratseth + */ +public class DeploymentInstanceSpec extends DeploymentSpec.Step { + + /** The name of the instance this step deploys */ + private final InstanceName name; + + private final List<DeploymentSpec.Step> steps; + private final DeploymentSpec.UpgradePolicy upgradePolicy; + private final List<DeploymentSpec.ChangeBlocker> changeBlockers; + private final Optional<String> globalServiceId; + private final Optional<AthenzDomain> athenzDomain; + private final Optional<AthenzService> athenzService; + private final Notifications notifications; + private final List<Endpoint> endpoints; + + public DeploymentInstanceSpec(InstanceName name, + List<DeploymentSpec.Step> steps, + DeploymentSpec.UpgradePolicy upgradePolicy, + List<DeploymentSpec.ChangeBlocker> changeBlockers, + Optional<String> globalServiceId, + Optional<AthenzDomain> athenzDomain, + Optional<AthenzService> athenzService, + Notifications notifications, + List<Endpoint> endpoints) { + this.name = name; + this.steps = steps; + this.upgradePolicy = upgradePolicy; + this.changeBlockers = changeBlockers; + this.globalServiceId = globalServiceId; + this.athenzDomain = athenzDomain; + this.athenzService = athenzService; + this.notifications = notifications; + this.endpoints = List.copyOf(validateEndpoints(endpoints, this.steps)); + validateZones(this.steps); + validateEndpoints(this.steps, globalServiceId, this.endpoints); + validateAthenz(); + } + + public InstanceName name() { return name; } + + /** Throw an IllegalArgumentException if any production zone is declared multiple times */ + private void validateZones(List<DeploymentSpec.Step> steps) { + Set<DeploymentSpec.DeclaredZone> zones = new HashSet<>(); + + for (DeploymentSpec.Step step : steps) + for (DeploymentSpec.DeclaredZone zone : step.zones()) + ensureUnique(zone, zones); + } + + private void ensureUnique(DeploymentSpec.DeclaredZone zone, Set<DeploymentSpec.DeclaredZone> zones) { + if ( ! zones.add(zone)) + throw new IllegalArgumentException(zone + " is listed twice in deployment.xml"); + } + + /** Validates the endpoints and makes sure default values are respected */ + private List<Endpoint> validateEndpoints(List<Endpoint> endpoints, List<DeploymentSpec.Step> steps) { + Objects.requireNonNull(endpoints, "Missing endpoints parameter"); + + var productionRegions = steps.stream() + .filter(step -> step.deploysTo(Environment.prod)) + .flatMap(step -> step.zones().stream()) + .flatMap(zone -> zone.region().stream()) + .map(RegionName::value) + .collect(Collectors.toSet()); + + var rebuiltEndpointsList = new ArrayList<Endpoint>(); + + for (var endpoint : endpoints) { + if (endpoint.regions().isEmpty()) { + var rebuiltEndpoint = endpoint.withRegions(productionRegions); + rebuiltEndpointsList.add(rebuiltEndpoint); + } else { + rebuiltEndpointsList.add(endpoint); + } + } + + return List.copyOf(rebuiltEndpointsList); + } + + /** Throw an IllegalArgumentException if an endpoint refers to a region that is not declared in 'prod' */ + private void validateEndpoints(List<DeploymentSpec.Step> steps, Optional<String> globalServiceId, List<Endpoint> endpoints) { + if (globalServiceId.isPresent() && ! endpoints.isEmpty()) { + throw new IllegalArgumentException("Providing both 'endpoints' and 'global-service-id'. Use only 'endpoints'."); + } + + var stepZones = steps.stream() + .flatMap(s -> s.zones().stream()) + .flatMap(z -> z.region().stream()) + .collect(Collectors.toSet()); + + for (var endpoint : endpoints){ + for (var endpointRegion : endpoint.regions()) { + if (! stepZones.contains(endpointRegion)) { + throw new IllegalArgumentException("Region used in endpoint that is not declared in 'prod': " + endpointRegion); + } + } + } + } + + /** + * Throw an IllegalArgumentException if Athenz configuration violates: + * domain not configured -> no zone can configure service + * domain configured -> all zones must configure service + */ + private void validateAthenz() { + // If athenz domain is not set, athenz service cannot be set on any level + if (athenzDomain.isEmpty()) { + for (DeploymentSpec.DeclaredZone zone : zones()) { + if(zone.athenzService().isPresent()) { + throw new IllegalArgumentException("Athenz service configured for zone: " + zone + ", but Athenz domain is not configured"); + } + } + // if athenz domain is not set, athenz service must be set implicitly or directly on all zones. + } else if (athenzService.isEmpty()) { + for (DeploymentSpec.DeclaredZone zone : zones()) { + if (zone.athenzService().isEmpty()) { + throw new IllegalArgumentException("Athenz domain is configured, but Athenz service not configured for zone: " + zone); + } + } + } + } + + @Override + public Duration delay() { + return Duration.ofSeconds(steps.stream().mapToLong(step -> (step.delay().getSeconds())).sum()); + } + + /** Returns the deployment steps inside this in the order they will be performed */ + @Override + public List<DeploymentSpec.Step> steps() { return steps; } + + /** Returns the upgrade policy of this, which is defaultPolicy if none is specified */ + public DeploymentSpec.UpgradePolicy upgradePolicy() { return upgradePolicy; } + + /** Returns time windows where upgrades are disallowed for these instances */ + public List<DeploymentSpec.ChangeBlocker> changeBlocker() { return changeBlockers; } + + /** Returns the ID of the service to expose through global routing, if present */ + public Optional<String> globalServiceId() { return globalServiceId; } + + /** Returns whether the instances in this step can upgrade at the given instant */ + public boolean canUpgradeAt(Instant instant) { + return changeBlockers.stream().filter(block -> block.blocksVersions()) + .noneMatch(block -> block.window().includes(instant)); + } + + /** Returns whether an application revision change for these instances can occur at the given instant */ + public boolean canChangeRevisionAt(Instant instant) { + return changeBlockers.stream().filter(block -> block.blocksRevisions()) + .noneMatch(block -> block.window().includes(instant)); + } + + /** Returns all the deployment steps which are zones in the order they are declared */ + public List<DeploymentSpec.DeclaredZone> zones() { + return steps.stream() + .flatMap(step -> step.zones().stream()) + .collect(Collectors.toList()); + } + + /** Returns whether this deployment spec specifies the given zone, either implicitly or explicitly */ + @Override + public boolean deploysTo(Environment environment, Optional<RegionName> region) { + for (DeploymentSpec.Step step : steps) + if (step.deploysTo(environment, region)) return true; + return false; + } + + /** Returns the athenz domain if configured */ + public Optional<AthenzDomain> athenzDomain() { return athenzDomain; } + + /** Returns the athenz service for environment/region if configured */ + public Optional<AthenzService> athenzService(Environment environment, RegionName region) { + AthenzService athenzService = zones().stream() + .filter(zone -> zone.deploysTo(environment, Optional.of(region))) + .findFirst() + .flatMap(DeploymentSpec.DeclaredZone::athenzService) + .orElse(this.athenzService.orElse(null)); + return Optional.ofNullable(athenzService); + } + + /** Returns the notification configuration of these instances */ + public Notifications notifications() { return notifications; } + + /** Returns the rotations configuration of these instances */ + public List<Endpoint> endpoints() { return endpoints; } + + /** Returns whether this instances deployment specifies the given zone, either implicitly or explicitly */ + public boolean includes(Environment environment, Optional<RegionName> region) { + for (DeploymentSpec.Step step : steps) + if (step.deploysTo(environment, region)) return true; + return false; + } + + DeploymentInstanceSpec withSteps(List<DeploymentSpec.Step> steps) { + return new DeploymentInstanceSpec(name, + steps, + upgradePolicy, + changeBlockers, + globalServiceId, + athenzDomain, + athenzService, + notifications, + endpoints); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DeploymentInstanceSpec other = (DeploymentInstanceSpec) o; + return globalServiceId.equals(other.globalServiceId) && + upgradePolicy == other.upgradePolicy && + changeBlockers.equals(other.changeBlockers) && + steps.equals(other.steps) && + athenzDomain.equals(other.athenzDomain) && + athenzService.equals(other.athenzService) && + notifications.equals(other.notifications) && + endpoints.equals(other.endpoints); + } + + @Override + public int hashCode() { + return Objects.hash(globalServiceId, upgradePolicy, changeBlockers, steps, athenzDomain, athenzService, notifications, endpoints); + } + + @Override + public String toString() { + return "instance '" + name + "'"; + } + +} diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index efe75d191b8..9b0454cffee 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -5,6 +5,7 @@ import com.yahoo.config.application.api.xml.DeploymentSpecXmlReader; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import java.io.BufferedReader; @@ -14,11 +15,9 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; /** @@ -46,218 +45,218 @@ public class DeploymentSpec { Optional.empty(), Notifications.none(), List.of()); - - private final Optional<String> globalServiceId; - private final UpgradePolicy upgradePolicy; - private final Optional<Integer> majorVersion; - private final List<ChangeBlocker> changeBlockers; + private final List<Step> steps; - private final String xmlForm; + + // Attributes which can be set on the root tag and which must be available outside of any particular instance + private final Optional<Integer> majorVersion; private final Optional<AthenzDomain> athenzDomain; private final Optional<AthenzService> athenzService; - private final Notifications notifications; - private final List<Endpoint> endpoints; - - public DeploymentSpec(Optional<String> globalServiceId, UpgradePolicy upgradePolicy, Optional<Integer> majorVersion, - List<ChangeBlocker> changeBlockers, List<Step> steps, String xmlForm, - Optional<AthenzDomain> athenzDomain, Optional<AthenzService> athenzService, Notifications notifications, - List<Endpoint> endpoints) { - validateTotalDelay(steps); - this.globalServiceId = globalServiceId; - this.upgradePolicy = upgradePolicy; - this.majorVersion = majorVersion; - this.changeBlockers = changeBlockers; - this.steps = List.copyOf(completeSteps(new ArrayList<>(steps))); - this.xmlForm = xmlForm; - this.athenzDomain = athenzDomain; - this.athenzService = athenzService; - this.notifications = notifications; - this.endpoints = List.copyOf(validateEndpoints(endpoints, this.steps)); - validateZones(this.steps); - validateAthenz(); - validateEndpoints(this.steps, globalServiceId, this.endpoints); - } - /** Validates the endpoints and makes sure default values are respected */ - private List<Endpoint> validateEndpoints(List<Endpoint> endpoints, List<Step> steps) { - Objects.requireNonNull(endpoints, "Missing endpoints parameter"); - - var productionRegions = steps.stream() - .filter(step -> step.deploysTo(Environment.prod)) - .flatMap(step -> step.zones().stream()) - .flatMap(zone -> zone.region().stream()) - .map(RegionName::value) - .collect(Collectors.toSet()); - - var rebuiltEndpointsList = new ArrayList<Endpoint>(); - - for (var endpoint : endpoints) { - if (endpoint.regions().isEmpty()) { - var rebuiltEndpoint = endpoint.withRegions(productionRegions); - rebuiltEndpointsList.add(rebuiltEndpoint); - } else { - rebuiltEndpointsList.add(endpoint); - } - } - - return List.copyOf(rebuiltEndpointsList); - } - - /** Throw an IllegalArgumentException if the total delay exceeds 24 hours */ - private void validateTotalDelay(List<Step> steps) { - long totalDelaySeconds = steps.stream().filter(step -> step instanceof Delay) - .mapToLong(delay -> ((Delay)delay).duration().getSeconds()) - .sum(); - if (totalDelaySeconds > Duration.ofHours(24).getSeconds()) - throw new IllegalArgumentException("The total delay specified is " + Duration.ofSeconds(totalDelaySeconds) + - " but max 24 hours is allowed"); - } - - /** Throw an IllegalArgumentException if any production zone is declared multiple times */ - private void validateZones(List<Step> steps) { - Set<DeclaredZone> zones = new HashSet<>(); - - for (Step step : steps) - for (DeclaredZone zone : step.zones()) - ensureUnique(zone, zones); - } - - /** Throw an IllegalArgumentException if an endpoint refers to a region that is not declared in 'prod' */ - private void validateEndpoints(List<Step> steps, Optional<String> globalServiceId, List<Endpoint> endpoints) { - if (globalServiceId.isPresent() && ! endpoints.isEmpty()) { - throw new IllegalArgumentException("Providing both 'endpoints' and 'global-service-id'. Use only 'endpoints'."); - } - - var stepZones = steps.stream() - .flatMap(s -> s.zones().stream()) - .flatMap(z -> z.region.stream()) - .collect(Collectors.toSet()); + private final String xmlForm; - for (var endpoint : endpoints){ - for (var endpointRegion : endpoint.regions()) { - if (! stepZones.contains(endpointRegion)) { - throw new IllegalArgumentException("Region used in endpoint that is not declared in 'prod': " + endpointRegion); - } - } + public DeploymentSpec(List<Step> steps, + Optional<Integer> majorVersion, + Optional<AthenzDomain> athenzDomain, + Optional<AthenzService> athenzService, + String xmlForm) { + if (singleInstance(steps)) { // TODO: Remove this clause after November 2019 + var singleInstance = (DeploymentInstanceSpec)steps.get(0); + this.steps = List.of(singleInstance.withSteps(completeSteps(singleInstance.steps()))); } - } - - /* - * Throw an IllegalArgumentException if Athenz configuration violates: - * domain not configured -> no zone can configure service - * domain configured -> all zones must configure service - */ - private void validateAthenz() { - // If athenz domain is not set, athenz service cannot be set on any level - if (athenzDomain.isEmpty()) { - for (DeclaredZone zone : zones()) { - if(zone.athenzService().isPresent()) { - throw new IllegalArgumentException("Athenz service configured for zone: " + zone + ", but Athenz domain is not configured"); - } - } - // if athenz domain is not set, athenz service must be set implicitly or directly on all zones. - } else if (athenzService.isEmpty()) { - for (DeclaredZone zone : zones()) { - if (zone.athenzService().isEmpty()) { - throw new IllegalArgumentException("Athenz domain is configured, but Athenz service not configured for zone: " + zone); - } - } + else { + this.steps = List.copyOf(completeSteps(steps)); } + this.majorVersion = majorVersion; + this.athenzDomain = athenzDomain; + this.athenzService = athenzService; + this.xmlForm = xmlForm; + validateTotalDelay(steps); } - private void ensureUnique(DeclaredZone zone, Set<DeclaredZone> zones) { - if ( ! zones.add(zone)) - throw new IllegalArgumentException(zone + " is listed twice in deployment.xml"); + // TODO: Remove after October 2019 + public DeploymentSpec(Optional<String> globalServiceId, UpgradePolicy upgradePolicy, Optional<Integer> majorVersion, + List<ChangeBlocker> changeBlockers, List<Step> steps, String xmlForm, + Optional<AthenzDomain> athenzDomain, Optional<AthenzService> athenzService, + Notifications notifications, + List<Endpoint> endpoints) { + this(List.of(new DeploymentInstanceSpec(InstanceName.from("default"), + steps, + upgradePolicy, + changeBlockers, + globalServiceId, + athenzDomain, + athenzService, + notifications, + endpoints)), + majorVersion, + athenzDomain, + athenzService, + xmlForm); } /** Adds missing required steps and reorders steps to a permissible order */ - private static List<Step> completeSteps(List<Step> steps) { + private static List<DeploymentSpec.Step> completeSteps(List<DeploymentSpec.Step> inputSteps) { + List<Step> steps = new ArrayList<>(inputSteps); + // Add staging if required and missing if (steps.stream().anyMatch(step -> step.deploysTo(Environment.prod)) && steps.stream().noneMatch(step -> step.deploysTo(Environment.staging))) { - steps.add(new DeclaredZone(Environment.staging)); + steps.add(new DeploymentSpec.DeclaredZone(Environment.staging)); } - + // Add test if required and missing if (steps.stream().anyMatch(step -> step.deploysTo(Environment.staging)) && steps.stream().noneMatch(step -> step.deploysTo(Environment.test))) { - steps.add(new DeclaredZone(Environment.test)); + steps.add(new DeploymentSpec.DeclaredZone(Environment.test)); } - + // Enforce order test, staging, prod - DeclaredZone testStep = remove(Environment.test, steps); + DeploymentSpec.DeclaredZone testStep = remove(Environment.test, steps); if (testStep != null) steps.add(0, testStep); - DeclaredZone stagingStep = remove(Environment.staging, steps); + DeploymentSpec.DeclaredZone stagingStep = remove(Environment.staging, steps); if (stagingStep != null) steps.add(1, stagingStep); - + return steps; } - /** + /** * Removes the first occurrence of a deployment step to the given environment and returns it. - * + * * @return the removed step, or null if it is not present */ - private static DeclaredZone remove(Environment environment, List<Step> steps) { + private static DeploymentSpec.DeclaredZone remove(Environment environment, List<DeploymentSpec.Step> steps) { for (int i = 0; i < steps.size(); i++) { - if (steps.get(i).deploysTo(environment)) - return (DeclaredZone)steps.remove(i); + if ( ! (steps.get(i) instanceof DeploymentSpec.DeclaredZone)) continue; + DeploymentSpec.DeclaredZone zoneStep = (DeploymentSpec.DeclaredZone)steps.get(i); + if (zoneStep.environment() == environment) { + steps.remove(i); + return zoneStep; + } } return null; } - /** Returns the ID of the service to expose through global routing, if present */ - public Optional<String> globalServiceId() { - return globalServiceId; + /** Throw an IllegalArgumentException if the total delay exceeds 24 hours */ + private void validateTotalDelay(List<Step> steps) { + long totalDelaySeconds = steps.stream().mapToLong(step -> (step.delay().getSeconds())).sum(); + if (totalDelaySeconds > Duration.ofHours(24).getSeconds()) + throw new IllegalArgumentException("The total delay specified is " + Duration.ofSeconds(totalDelaySeconds) + + " but max 24 hours is allowed"); + } + + // TODO: Remove after October 2019 + private DeploymentInstanceSpec singleInstance() { + if (singleInstance(steps)) return (DeploymentInstanceSpec)steps.get(0); + throw new IllegalArgumentException("This deployment spec does not support the legacy API " + + "as it has multiple instances: " + + instances().stream().map(Step::toString).collect(Collectors.joining(","))); } - /** Returns the upgrade policy of this, which is defaultPolicy if none is specified */ - public UpgradePolicy upgradePolicy() { return upgradePolicy; } + // TODO: Remove after October 2019 + public Optional<String> globalServiceId() { return singleInstance().globalServiceId(); } + + // TODO: Remove after October 2019 + public UpgradePolicy upgradePolicy() { return singleInstance().upgradePolicy(); } /** Returns the major version this application is pinned to, or empty (default) to allow all major versions */ public Optional<Integer> majorVersion() { return majorVersion; } - /** Returns whether upgrade can occur at the given instant */ - public boolean canUpgradeAt(Instant instant) { - return changeBlockers.stream().filter(block -> block.blocksVersions()) - .noneMatch(block -> block.window().includes(instant)); - } + // TODO: Remove after November 2019 + public boolean canUpgradeAt(Instant instant) { return singleInstance().canUpgradeAt(instant); } - /** Returns whether an application revision change can occur at the given instant */ - public boolean canChangeRevisionAt(Instant instant) { - return changeBlockers.stream().filter(block -> block.blocksRevisions()) - .noneMatch(block -> block.window().includes(instant)); - } + // TODO: Remove after November 2019 + public boolean canChangeRevisionAt(Instant instant) { return singleInstance().canChangeRevisionAt(instant); } - /** Returns time windows where upgrades are disallowed */ - public List<ChangeBlocker> changeBlocker() { return changeBlockers; } + // TODO: Remove after November 2019 + public List<ChangeBlocker> changeBlocker() { return singleInstance().changeBlocker(); } /** Returns the deployment steps of this in the order they will be performed */ - public List<Step> steps() { return steps; } + public List<Step> steps() { + if (singleInstance(steps)) return singleInstance().steps(); // TODO: Remove line after November 2019 + return steps; + } - /** Returns all the DeclaredZone deployment steps in the order they are declared */ + // TODO: Remove after November 2019 public List<DeclaredZone> zones() { - return steps.stream() - .flatMap(step -> step.zones().stream()) - .collect(Collectors.toList()); + return singleInstance().steps().stream() + .flatMap(step -> step.zones().stream()) + .collect(Collectors.toList()); + } + + /** Returns the Athenz domain set on the root tag, if any */ + public Optional<AthenzDomain> athenzDomain() { return athenzDomain; } + + /** Returns the Athenz service to use for the given environment and region, if any */ + // TODO: Remove after November 2019 + public Optional<AthenzService> athenzService(Environment environment, RegionName region) { + Optional<AthenzService> service = Optional.empty(); + if (singleInstance(steps)) + service = singleInstance().athenzService(environment, region); + if (service.isPresent()) + return service; + return this.athenzService; + } + + /** + * Returns the Athenz service to use for the given instance, environment and region, if any. + * This returns the default set on the deploy tag (if any) if nothing is set for this particular + * combination of instance, environment and region, and also if that combination is not specified + * at all in services. + */ + public Optional<AthenzService> athenzService(InstanceName instanceName, Environment environment, RegionName region) { + Optional<DeploymentInstanceSpec> instance = instance(instanceName); + if (instance.isEmpty()) return this.athenzService; + return instance.get().athenzService(environment, region).or(() -> this.athenzService); } - /** Returns the notification configuration */ - public Notifications notifications() { return notifications; } + // TODO: Remove after November 2019 + public Notifications notifications() { return singleInstance().notifications(); } - /** Returns the rotations configuration */ - public List<Endpoint> endpoints() { return endpoints; } + // TODO: Remove after November 2019 + public List<Endpoint> endpoints() { return singleInstance().endpoints(); } /** Returns the XML form of this spec, or null if it was not created by fromXml, nor is empty */ public String xmlForm() { return xmlForm; } - /** Returns whether this deployment spec specifies the given zone, either implicitly or explicitly */ + // TODO: Remove after November 2019 public boolean includes(Environment environment, Optional<RegionName> region) { - for (Step step : steps) - if (step.deploysTo(environment, region)) return true; - return false; + return singleInstance().deploysTo(environment, region); + } + + // TODO: Remove after November 2019 + private static boolean singleInstance(List<DeploymentSpec.Step> steps) { + return steps.size() == 1 && steps.get(0) instanceof DeploymentInstanceSpec; + } + + /** Returns the instance step containing the given instance name */ + public Optional<DeploymentInstanceSpec> instance(InstanceName name) { + for (DeploymentInstanceSpec instance : instances()) { + if (instance.name().equals(name)) + return Optional.of(instance); + } + return Optional.empty(); + } + + public DeploymentInstanceSpec requireInstance(String name) { + return requireInstance(InstanceName.from(name)); + } + + public DeploymentInstanceSpec requireInstance(InstanceName name) { + Optional<DeploymentInstanceSpec> instance = instance(name); + if (instance.isEmpty()) + throw new IllegalArgumentException("No instance '" + name + "' in deployment.xml'. Instances: " + + instances().stream().map(spec -> spec.name().toString()).collect(Collectors.joining(","))); + return instance.get(); + } + + /** Returns the steps of this which are instances */ + public List<DeploymentInstanceSpec> instances() { + return steps.stream() + .filter(step -> step instanceof DeploymentInstanceSpec).map(DeploymentInstanceSpec.class::cast) + .collect(Collectors.toList()); } /** @@ -304,40 +303,19 @@ public class DeploymentSpec { return b.toString(); } - /** Returns the athenz domain if configured */ - public Optional<AthenzDomain> athenzDomain() { - return athenzDomain; - } - - /** Returns the athenz service for environment/region if configured */ - public Optional<AthenzService> athenzService(Environment environment, RegionName region) { - AthenzService athenzService = zones().stream() - .filter(zone -> zone.deploysTo(environment, Optional.of(region))) - .findFirst() - .flatMap(DeclaredZone::athenzService) - .orElse(this.athenzService.orElse(null)); - return Optional.ofNullable(athenzService); - } - @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - DeploymentSpec that = (DeploymentSpec) o; - return globalServiceId.equals(that.globalServiceId) && - upgradePolicy == that.upgradePolicy && - majorVersion.equals(that.majorVersion) && - changeBlockers.equals(that.changeBlockers) && - steps.equals(that.steps) && - xmlForm.equals(that.xmlForm) && - athenzDomain.equals(that.athenzDomain) && - athenzService.equals(that.athenzService) && - notifications.equals(that.notifications); + DeploymentSpec other = (DeploymentSpec) o; + return majorVersion.equals(other.majorVersion) && + steps.equals(other.steps) && + xmlForm.equals(other.xmlForm); } @Override public int hashCode() { - return Objects.hash(globalServiceId, upgradePolicy, majorVersion, changeBlockers, steps, xmlForm, athenzDomain, athenzService, notifications); + return Objects.hash(majorVersion, steps, xmlForm); } /** This may be invoked by a continuous build */ @@ -365,7 +343,7 @@ public class DeploymentSpec { /** A deployment step */ public abstract static class Step { - + /** Returns whether this step deploys to the given region */ public final boolean deploysTo(Environment environment) { return deploysTo(environment, Optional.empty()); @@ -377,6 +355,12 @@ public class DeploymentSpec { /** Returns the zones deployed to in this step */ public List<DeclaredZone> zones() { return Collections.emptyList(); } + /** The delay introduced by this step (beyond the time it takes to execute the step). Default is zero. */ + public Duration delay() { return Duration.ZERO; } + + /** Returns all the steps nested in this. This default implementatiino returns an empty list. */ + public List<Step> steps() { return List.of(); } + } /** A deployment step which is to wait for some time before progressing to the next step */ @@ -387,12 +371,21 @@ public class DeploymentSpec { public Delay(Duration duration) { this.duration = duration; } - + + // TODO: Remove after October 2019 public Duration duration() { return duration; } @Override + public Duration delay() { return duration; } + + @Override public boolean deploysTo(Environment environment, Optional<RegionName> region) { return false; } + @Override + public String toString() { + return "delay " + duration; + } + } /** A deployment step which is to run deployment in a particular zone */ @@ -473,21 +466,31 @@ public class DeploymentSpec { } - /** A deployment step which is to run deployment to multiple zones in parallel */ + /** A deployment step which is to run multiple steps (zones or instances) in parallel */ public static class ParallelZones extends Step { - private final List<DeclaredZone> zones; + private final List<Step> steps; - public ParallelZones(List<DeclaredZone> zones) { - this.zones = List.copyOf(zones); + public ParallelZones(List<Step> steps) { + this.steps = List.copyOf(steps); } + /** Returns the steps inside this which are zones */ @Override - public List<DeclaredZone> zones() { return this.zones; } + public List<DeclaredZone> zones() { + return this.steps.stream() + .filter(step -> step instanceof DeclaredZone) + .map(DeclaredZone.class::cast) + .collect(Collectors.toList()); + } + + /** Returns all the steps nested in this */ + @Override + public List<Step> steps() { return steps; } @Override public boolean deploysTo(Environment environment, Optional<RegionName> region) { - return zones.stream().anyMatch(zone -> zone.deploysTo(environment, region)); + return steps().stream().anyMatch(zone -> zone.deploysTo(environment, region)); } @Override @@ -495,13 +498,19 @@ public class DeploymentSpec { if (this == o) return true; if (!(o instanceof ParallelZones)) return false; ParallelZones that = (ParallelZones) o; - return Objects.equals(zones, that.zones); + return Objects.equals(steps, that.steps); } @Override public int hashCode() { - return Objects.hash(zones); + return Objects.hash(steps); + } + + @Override + public String toString() { + return steps.size() + " parallel steps"; } + } /** Controls when this application will be upgraded to new Vespa versions */ @@ -530,6 +539,11 @@ public class DeploymentSpec { public boolean blocksRevisions() { return revision; } public boolean blocksVersions() { return version; } public TimeWindow window() { return window; } + + @Override + public String toString() { + return "change blocker revision=" + revision + " version=" + version + " window=" + window; + } } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java index 72a806bb7be..59b31985376 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application.api.xml; +import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.DeploymentSpec.DeclaredZone; import com.yahoo.config.application.api.DeploymentSpec.Delay; @@ -14,6 +15,7 @@ import com.yahoo.config.application.api.TimeWindow; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.io.IOUtils; import com.yahoo.text.XML; @@ -38,27 +40,36 @@ import java.util.stream.Collectors; */ public class DeploymentSpecXmlReader { + private static final String instanceTag = "instance"; private static final String majorVersionTag = "major-version"; private static final String testTag = "test"; private static final String stagingTag = "staging"; + private static final String upgradeTag = "upgrade"; private static final String blockChangeTag = "block-change"; private static final String prodTag = "prod"; + private static final String regionTag = "region"; + private static final String delayTag = "delay"; + private static final String parallelTag = "parallel"; private static final String endpointsTag = "endpoints"; private static final String endpointTag = "endpoint"; + private static final String notificationsTag = "notifications"; + + private static final String idAttribute = "id"; + private static final String athenzServiceAttribute = "athenz-service"; + private static final String athenzDomainAttribute = "athenz-domain"; + private static final String testerFlavorAttribute = "tester-flavor"; private final boolean validate; - /** - * Creates a validating reader - */ + /** Creates a validating reader */ public DeploymentSpecXmlReader() { this(true); } /** - * Creates a reader + * Creates a deployment spec reader * - * @param validate true to validate the input, false to accept any input which can be unabiguously parsed + * @param validate true to validate the input, false to accept any input which can be unambiguously parsed */ public DeploymentSpecXmlReader(boolean validate) { this.validate = validate; @@ -73,67 +84,137 @@ public class DeploymentSpecXmlReader { } } - /** - * Reads a deployment spec from XML - */ + /** Reads a deployment spec from XML */ public DeploymentSpec read(String xmlForm) { - List<Step> steps = new ArrayList<>(); - Optional<String> globalServiceId = Optional.empty(); Element root = XML.getDocument(xmlForm).getDocumentElement(); - if (validate) - validateTagOrder(root); - for (Element environmentTag : XML.getChildren(root)) { - if (!isEnvironmentName(environmentTag.getTagName())) continue; - - Environment environment = Environment.from(environmentTag.getTagName()); - Optional<AthenzService> athenzService = stringAttribute("athenz-service", environmentTag).map(AthenzService::from); - Optional<String> testerFlavor = stringAttribute("tester-flavor", environmentTag); - - if (environment == Environment.prod) { - for (Element stepTag : XML.getChildren(environmentTag)) { - if (stepTag.getTagName().equals("delay")) { - steps.add(new Delay(Duration.ofSeconds(longAttribute("hours", stepTag) * 60 * 60 + - longAttribute("minutes", stepTag) * 60 + - longAttribute("seconds", stepTag)))); - } - else if (stepTag.getTagName().equals("parallel")) { - List<DeclaredZone> zones = new ArrayList<>(); - for (Element regionTag : XML.getChildren(stepTag)) { - zones.add(readDeclaredZone(environment, athenzService, testerFlavor, regionTag)); - } - steps.add(new ParallelZones(zones)); - } - else { // a region: deploy step - steps.add(readDeclaredZone(environment, athenzService, testerFlavor, stepTag)); - } - } - } - else { - steps.add(new DeclaredZone(environment, Optional.empty(), false, athenzService, testerFlavor)); + + List<Step> steps = new ArrayList<>(); + if ( ! containsTag(instanceTag, root)) { // deployment spec skipping explicit instance -> "default" instance + steps.addAll(readInstanceContent("default", root, new MutableOptional<>(), root)); + } + else { + if (XML.getChildren(root).stream().anyMatch(child -> child.getTagName().equals(prodTag))) + throw new IllegalArgumentException("A deployment spec cannot have both a <prod> tag and an " + + "<instance> tag under the root: " + + "Wrap the prod tags inside the appropriate instance"); + + for (Element topLevelTag : XML.getChildren(root)) { + if (topLevelTag.getTagName().equals(instanceTag)) + steps.addAll(readInstanceContent(topLevelTag.getAttribute(idAttribute), topLevelTag, new MutableOptional<>(), root)); + else + steps.addAll(readNonInstanceSteps(topLevelTag, new MutableOptional<>(), topLevelTag)); // (No global service id here) } + } + + return new DeploymentSpec(steps, + optionalIntegerAttribute(majorVersionTag, root), + stringAttribute(athenzDomainAttribute, root).map(AthenzDomain::from), + stringAttribute(athenzServiceAttribute, root).map(AthenzService::from), + xmlForm); + } + + /** + * Reads the content of an (implicit or explicit) instance tag producing an instances step + * + * @param instanceNameString a comma-separated list of the names of the instances this is for + * @param instanceTag the element having the content of this instance + * @param parentTag the parent of instanceTag (or the same, if this instances is implicitly defined which means instanceTag is the root) + * @return the instances specified, one for each instance name element + */ + private List<DeploymentInstanceSpec> readInstanceContent(String instanceNameString, + Element instanceTag, + MutableOptional<String> globalServiceId, + Element parentTag) { + if (validate) + validateTagOrder(instanceTag); + + // Values where the parent may provide a default + DeploymentSpec.UpgradePolicy upgradePolicy = readUpgradePolicy(instanceTag, parentTag); + List<DeploymentSpec.ChangeBlocker> changeBlockers = readChangeBlockers(instanceTag, parentTag); + Optional<AthenzDomain> athenzDomain = stringAttribute(athenzDomainAttribute, instanceTag) + .or(() -> stringAttribute(athenzDomainAttribute, parentTag)) + .map(AthenzDomain::from); + Optional<AthenzService> athenzService = stringAttribute(athenzServiceAttribute, instanceTag) + .or(() -> stringAttribute(athenzServiceAttribute, parentTag)) + .map(AthenzService::from); + Notifications notifications = readNotifications(instanceTag, parentTag); + + // Values where there is no default + List<Step> steps = new ArrayList<>(); + for (Element instanceChild : XML.getChildren(instanceTag)) + steps.addAll(readNonInstanceSteps(instanceChild, globalServiceId, instanceChild)); + List<Endpoint> endpoints = readEndpoints(instanceTag); + + // Build and return instances with these values + return Arrays.stream(instanceNameString.split(",")) + .map(name -> name.trim()) + .map(name -> new DeploymentInstanceSpec(InstanceName.from(name), + steps, + upgradePolicy, + changeBlockers, + globalServiceId.asOptional(), + athenzDomain, + athenzService, + notifications, + endpoints)) + .collect(Collectors.toList()); + } + + private List<Step> readSteps(Element stepTag, MutableOptional<String> globalServiceId, Element parentTag) { + if (stepTag.getTagName().equals(instanceTag)) + return new ArrayList<>(readInstanceContent(stepTag.getAttribute(idAttribute), stepTag, globalServiceId, parentTag)); + else + return readNonInstanceSteps(stepTag, globalServiceId, parentTag); - if (environment == Environment.prod) - globalServiceId = readGlobalServiceId(environmentTag); - else if (readGlobalServiceId(environmentTag).isPresent()) - throw new IllegalArgumentException("Attribute 'global-service-id' is only valid on 'prod' tag."); + } + // Consume the given tag as 0-N steps. 0 if it is not a step, >1 if it contains multiple nested steps that should be flattened + private List<Step> readNonInstanceSteps(Element stepTag, MutableOptional<String> globalServiceId, Element parentTag) { + Optional<AthenzService> athenzService = stringAttribute(athenzServiceAttribute, stepTag) + .or(() -> stringAttribute(athenzServiceAttribute, parentTag)) + .map(AthenzService::from); + Optional<String> testerFlavor = stringAttribute(testerFlavorAttribute, stepTag) + .or(() -> stringAttribute(testerFlavorAttribute, parentTag)); + + if (prodTag.equals(stepTag.getTagName())) + globalServiceId.set(readGlobalServiceId(stepTag)); + else if (readGlobalServiceId(stepTag).isPresent()) + throw new IllegalArgumentException("Attribute 'global-service-id' is only valid on 'prod' tag."); + + switch (stepTag.getTagName()) { + case testTag: case stagingTag: + return List.of(new DeclaredZone(Environment.from(stepTag.getTagName()), Optional.empty(), false, athenzService, testerFlavor)); + case prodTag: // regions, delay and parallel may be nested within, but we can flatten them + return XML.getChildren(stepTag).stream() + .flatMap(child -> readNonInstanceSteps(child, globalServiceId, stepTag).stream()) + .collect(Collectors.toList()); + case delayTag: + return List.of(new Delay(Duration.ofSeconds(longAttribute("hours", stepTag) * 60 * 60 + + longAttribute("minutes", stepTag) * 60 + + longAttribute("seconds", stepTag)))); + case parallelTag: // regions and instances may be nested within + return List.of(new ParallelZones(XML.getChildren(stepTag).stream() + .flatMap(child -> readSteps(child, globalServiceId, stepTag).stream()) + .collect(Collectors.toList()))); + case regionTag: + return List.of(readDeclaredZone(Environment.prod, athenzService, testerFlavor, stepTag)); + default: + return List.of(); } - Optional<AthenzDomain> athenzDomain = stringAttribute("athenz-domain", root).map(AthenzDomain::from); - Optional<AthenzService> athenzService = stringAttribute("athenz-service", root).map(AthenzService::from); - return new DeploymentSpec(globalServiceId, - readUpgradePolicy(root), - optionalIntegerAttribute(majorVersionTag, root), - readChangeBlockers(root), - steps, - xmlForm, - athenzDomain, - athenzService, - readNotifications(root), - readEndpoints(root)); } - private Notifications readNotifications(Element root) { - Element notificationsElement = XML.getChild(root, "notifications"); + private boolean containsTag(String childTagName, Element parent) { + for (Element child : XML.getChildren(parent)) { + if (child.getTagName().equals(childTagName) || containsTag(childTagName, child)) + return true; + } + return false; + } + + private Notifications readNotifications(Element parent, Element fallbackParent) { + Element notificationsElement = XML.getChild(parent, notificationsTag); + if (notificationsElement == null) + notificationsElement = XML.getChild(fallbackParent, notificationsTag); if (notificationsElement == null) return Notifications.none(); @@ -158,16 +239,17 @@ public class DeploymentSpecXmlReader { return Notifications.of(emailAddresses, emailRoles); } - private List<Endpoint> readEndpoints(Element root) { - final var endpointsElement = XML.getChild(root, endpointsTag); - if (endpointsElement == null) { return Collections.emptyList(); } + private List<Endpoint> readEndpoints(Element parent) { + var endpointsElement = XML.getChild(parent, endpointsTag); + if (endpointsElement == null) + return Collections.emptyList(); - final var endpoints = new LinkedHashMap<String, Endpoint>(); + var endpoints = new LinkedHashMap<String, Endpoint>(); for (var endpointElement : XML.getChildren(endpointsElement, endpointTag)) { - final Optional<String> rotationId = stringAttribute("id", endpointElement); - final Optional<String> containerId = stringAttribute("container-id", endpointElement); - final var regions = new HashSet<String>(); + Optional<String> rotationId = stringAttribute("id", endpointElement); + Optional<String> containerId = stringAttribute("container-id", endpointElement); + var regions = new HashSet<String>(); if (containerId.isEmpty()) { throw new IllegalArgumentException("Missing 'container-id' from 'endpoint' tag."); @@ -255,10 +337,6 @@ public class DeploymentSpecXmlReader { return Optional.ofNullable(value).filter(s -> !s.equals("")); } - private boolean isEnvironmentName(String tagName) { - return tagName.equals(testTag) || tagName.equals(stagingTag) || tagName.equals(prodTag); - } - private DeclaredZone readDeclaredZone(Environment environment, Optional<AthenzService> athenzService, Optional<String> testerFlavor, Element regionTag) { return new DeclaredZone(environment, Optional.of(RegionName.from(XML.getValue(regionTag).trim())), @@ -267,44 +345,44 @@ public class DeploymentSpecXmlReader { private Optional<String> readGlobalServiceId(Element environmentTag) { String globalServiceId = environmentTag.getAttribute("global-service-id"); - if (globalServiceId == null || globalServiceId.isEmpty()) { - return Optional.empty(); - } - else { - return Optional.of(globalServiceId); - } + if (globalServiceId == null || globalServiceId.isEmpty()) return Optional.empty(); + return Optional.of(globalServiceId); } - private List<DeploymentSpec.ChangeBlocker> readChangeBlockers(Element root) { + private List<DeploymentSpec.ChangeBlocker> readChangeBlockers(Element parent, Element globalBlockersParent) { List<DeploymentSpec.ChangeBlocker> changeBlockers = new ArrayList<>(); - for (Element tag : XML.getChildren(root)) { - if (!blockChangeTag.equals(tag.getTagName())) continue; - - boolean blockVersions = trueOrMissing(tag.getAttribute("version")); - boolean blockRevisions = trueOrMissing(tag.getAttribute("revision")); - - String daySpec = tag.getAttribute("days"); - String hourSpec = tag.getAttribute("hours"); - String zoneSpec = tag.getAttribute("time-zone"); - if (zoneSpec.isEmpty()) { // Default to UTC time zone - zoneSpec = "UTC"; - } - changeBlockers.add(new DeploymentSpec.ChangeBlocker(blockRevisions, blockVersions, - TimeWindow.from(daySpec, hourSpec, zoneSpec))); + if (globalBlockersParent != parent) { + for (Element tag : XML.getChildren(globalBlockersParent, blockChangeTag)) + changeBlockers.add(readChangeBlocker(tag)); } + for (Element tag : XML.getChildren(parent, blockChangeTag)) + changeBlockers.add(readChangeBlocker(tag)); return Collections.unmodifiableList(changeBlockers); } - /** - * Returns true if the given value is "true", or if it is missing - */ + private DeploymentSpec.ChangeBlocker readChangeBlocker(Element tag) { + boolean blockVersions = trueOrMissing(tag.getAttribute("version")); + boolean blockRevisions = trueOrMissing(tag.getAttribute("revision")); + + String daySpec = tag.getAttribute("days"); + String hourSpec = tag.getAttribute("hours"); + String zoneSpec = tag.getAttribute("time-zone"); + if (zoneSpec.isEmpty()) zoneSpec = "UTC"; // default + return new DeploymentSpec.ChangeBlocker(blockRevisions, blockVersions, + TimeWindow.from(daySpec, hourSpec, zoneSpec)); + } + + /** Returns true if the given value is "true", or if it is missing */ private boolean trueOrMissing(String value) { return value == null || value.isEmpty() || value.equals("true"); } - private DeploymentSpec.UpgradePolicy readUpgradePolicy(Element root) { - Element upgradeElement = XML.getChild(root, "upgrade"); - if (upgradeElement == null) return DeploymentSpec.UpgradePolicy.defaultPolicy; + private DeploymentSpec.UpgradePolicy readUpgradePolicy(Element parent, Element fallbackParent) { + Element upgradeElement = XML.getChild(parent, upgradeTag); + if (upgradeElement == null) + upgradeElement = XML.getChild(fallbackParent, upgradeTag); + if (upgradeElement == null) + return DeploymentSpec.UpgradePolicy.defaultPolicy; String policy = upgradeElement.getAttribute("policy"); switch (policy) { @@ -324,4 +402,14 @@ public class DeploymentSpecXmlReader { "to control whether the region should receive production traffic"); } + private static class MutableOptional<T> { + + private Optional<T> value = Optional.empty(); + + public void set(Optional<T> value) { this.value = value; } + + public Optional<T> asOptional() { return value; } + + } + } diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecDeprecatedAPITest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecDeprecatedAPITest.java new file mode 100644 index 00000000000..dabdd0c4a69 --- /dev/null +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecDeprecatedAPITest.java @@ -0,0 +1,572 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.application.api; + +import com.google.common.collect.ImmutableSet; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import org.junit.Test; + +import java.io.StringReader; +import java.time.Instant; +import java.time.ZoneId; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.yahoo.config.application.api.Notifications.Role.author; +import static com.yahoo.config.application.api.Notifications.When.failing; +import static com.yahoo.config.application.api.Notifications.When.failingCommit; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author bratseth + */ +// TODO: Remove after October 2019 +public class DeploymentSpecDeprecatedAPITest { + + @Test + public void testSpec() { + String specXml = "<deployment version='1.0'>" + + " <test/>" + + "</deployment>"; + + StringReader r = new StringReader(specXml); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(specXml, spec.xmlForm()); + assertEquals(1, spec.steps().size()); + assertFalse(spec.majorVersion().isPresent()); + assertTrue(spec.steps().get(0).deploysTo(Environment.test)); + assertTrue(spec.includes(Environment.test, Optional.empty())); + assertFalse(spec.includes(Environment.test, Optional.of(RegionName.from("region1")))); + assertFalse(spec.includes(Environment.staging, Optional.empty())); + assertFalse(spec.includes(Environment.prod, Optional.empty())); + assertFalse(spec.globalServiceId().isPresent()); + } + + @Test + public void testSpecPinningMajorVersion() { + String specXml = "<deployment version='1.0' major-version='6'>" + + " <test/>" + + "</deployment>"; + + StringReader r = new StringReader(specXml); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(specXml, spec.xmlForm()); + assertEquals(1, spec.steps().size()); + assertTrue(spec.majorVersion().isPresent()); + assertEquals(6, (int)spec.majorVersion().get()); + } + + @Test + public void stagingSpec() { + StringReader r = new StringReader( + "<deployment version='1.0'>" + + " <staging/>" + + "</deployment>" + ); + + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(2, spec.steps().size()); + assertTrue(spec.steps().get(0).deploysTo(Environment.test)); + assertTrue(spec.steps().get(1).deploysTo(Environment.staging)); + assertTrue(spec.includes(Environment.test, Optional.empty())); + assertFalse(spec.includes(Environment.test, Optional.of(RegionName.from("region1")))); + assertTrue(spec.includes(Environment.staging, Optional.empty())); + assertFalse(spec.includes(Environment.prod, Optional.empty())); + assertFalse(spec.globalServiceId().isPresent()); + } + + @Test + public void minimalProductionSpec() { + StringReader r = new StringReader( + "<deployment version='1.0'>" + + " <prod>" + + " <region active='false'>us-east1</region>" + + " <region active='true'>us-west1</region>" + + " </prod>" + + "</deployment>" + ); + + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(4, spec.steps().size()); + + assertTrue(spec.steps().get(0).deploysTo(Environment.test)); + + assertTrue(spec.steps().get(1).deploysTo(Environment.staging)); + + assertTrue(spec.steps().get(2).deploysTo(Environment.prod, Optional.of(RegionName.from("us-east1")))); + assertFalse(((DeploymentSpec.DeclaredZone)spec.steps().get(2)).active()); + + assertTrue(spec.steps().get(3).deploysTo(Environment.prod, Optional.of(RegionName.from("us-west1")))); + assertTrue(((DeploymentSpec.DeclaredZone)spec.steps().get(3)).active()); + + assertTrue(spec.includes(Environment.test, Optional.empty())); + assertFalse(spec.includes(Environment.test, Optional.of(RegionName.from("region1")))); + assertTrue(spec.includes(Environment.staging, Optional.empty())); + assertTrue(spec.includes(Environment.prod, Optional.of(RegionName.from("us-east1")))); + assertTrue(spec.includes(Environment.prod, Optional.of(RegionName.from("us-west1")))); + assertFalse(spec.includes(Environment.prod, Optional.of(RegionName.from("no-such-region")))); + assertFalse(spec.globalServiceId().isPresent()); + + assertEquals(DeploymentSpec.UpgradePolicy.defaultPolicy, spec.upgradePolicy()); + } + + @Test + public void maximalProductionSpec() { + StringReader r = new StringReader( + "<deployment version='1.0'>" + + " <test/>" + + " <staging/>" + + " <prod>" + + " <region active='false'>us-east1</region>" + + " <delay hours='3' minutes='30'/>" + + " <region active='true'>us-west1</region>" + + " </prod>" + + "</deployment>" + ); + + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(5, spec.steps().size()); + assertEquals(4, spec.zones().size()); + + assertTrue(spec.steps().get(0).deploysTo(Environment.test)); + + assertTrue(spec.steps().get(1).deploysTo(Environment.staging)); + + assertTrue(spec.steps().get(2).deploysTo(Environment.prod, Optional.of(RegionName.from("us-east1")))); + assertFalse(((DeploymentSpec.DeclaredZone)spec.steps().get(2)).active()); + + assertTrue(spec.steps().get(3) instanceof DeploymentSpec.Delay); + assertEquals(3 * 60 * 60 + 30 * 60, ((DeploymentSpec.Delay)spec.steps().get(3)).duration().getSeconds()); + + assertTrue(spec.steps().get(4).deploysTo(Environment.prod, Optional.of(RegionName.from("us-west1")))); + assertTrue(((DeploymentSpec.DeclaredZone)spec.steps().get(4)).active()); + + assertTrue(spec.includes(Environment.test, Optional.empty())); + assertFalse(spec.includes(Environment.test, Optional.of(RegionName.from("region1")))); + assertTrue(spec.includes(Environment.staging, Optional.empty())); + assertTrue(spec.includes(Environment.prod, Optional.of(RegionName.from("us-east1")))); + assertTrue(spec.includes(Environment.prod, Optional.of(RegionName.from("us-west1")))); + assertFalse(spec.includes(Environment.prod, Optional.of(RegionName.from("no-such-region")))); + assertFalse(spec.globalServiceId().isPresent()); + } + + @Test + public void productionSpecWithGlobalServiceId() { + StringReader r = new StringReader( + "<deployment version='1.0'>" + + " <prod global-service-id='query'>" + + " <region active='true'>us-east-1</region>" + + " <region active='true'>us-west-1</region>" + + " </prod>" + + "</deployment>" + ); + + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(spec.globalServiceId(), Optional.of("query")); + } + + @Test(expected=IllegalArgumentException.class) + public void globalServiceIdInTest() { + StringReader r = new StringReader( + "<deployment version='1.0'>" + + " <test global-service-id='query' />" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + } + + @Test(expected=IllegalArgumentException.class) + public void globalServiceIdInStaging() { + StringReader r = new StringReader( + "<deployment version='1.0'>" + + " <staging global-service-id='query' />" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + } + + @Test + public void productionSpecWithGlobalServiceIdBeforeStaging() { + StringReader r = new StringReader( + "<deployment>" + + " <test/>" + + " <prod global-service-id='qrs'>" + + " <region active='true'>us-west-1</region>" + + " <region active='true'>us-central-1</region>" + + " <region active='true'>us-east-3</region>" + + " </prod>" + + " <staging/>" + + "</deployment>" + ); + + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals("qrs", spec.globalServiceId().get()); + } + + @Test + public void productionSpecWithUpgradePolicy() { + StringReader r = new StringReader( + "<deployment>" + + " <upgrade policy='canary'/>" + + " <prod>" + + " <region active='true'>us-west-1</region>" + + " <region active='true'>us-central-1</region>" + + " <region active='true'>us-east-3</region>" + + " </prod>" + + "</deployment>" + ); + + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals("canary", spec.upgradePolicy().toString()); + } + + @Test + public void maxDelayExceeded() { + try { + StringReader r = new StringReader( + "<deployment>" + + " <upgrade policy='canary'/>" + + " <prod>" + + " <region active='true'>us-west-1</region>" + + " <delay hours='23'/>" + + " <region active='true'>us-central-1</region>" + + " <delay minutes='59' seconds='61'/>" + + " <region active='true'>us-east-3</region>" + + " </prod>" + + "</deployment>" + ); + DeploymentSpec.fromXml(r); + fail("Expected exception due to exceeding the max total delay"); + } + catch (IllegalArgumentException e) { + // success + assertEquals("The total delay specified is PT24H1S but max 24 hours is allowed", e.getMessage()); + } + } + + @Test + public void testEmpty() { + assertFalse(DeploymentSpec.empty.globalServiceId().isPresent()); + assertEquals(DeploymentSpec.UpgradePolicy.defaultPolicy, DeploymentSpec.empty.upgradePolicy()); + assertTrue(DeploymentSpec.empty.steps().isEmpty()); + assertEquals("<deployment version='1.0'/>", DeploymentSpec.empty.xmlForm()); + } + + @Test + public void productionSpecWithParallelDeployments() { + StringReader r = new StringReader( + "<deployment>\n" + + " <prod> \n" + + " <region active='true'>us-west-1</region>\n" + + " <parallel>\n" + + " <region active='true'>us-central-1</region>\n" + + " <region active='true'>us-east-3</region>\n" + + " </parallel>\n" + + " </prod>\n" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + DeploymentSpec.ParallelZones parallelZones = ((DeploymentSpec.ParallelZones) spec.steps().get(3)); + assertEquals(2, parallelZones.zones().size()); + assertEquals(RegionName.from("us-central-1"), parallelZones.zones().get(0).region().get()); + assertEquals(RegionName.from("us-east-3"), parallelZones.zones().get(1).region().get()); + } + + @Test + public void productionSpecWithDuplicateRegions() { + StringReader r = new StringReader( + "<deployment>\n" + + " <prod>\n" + + " <region active='true'>us-west-1</region>\n" + + " <parallel>\n" + + " <region active='true'>us-west-1</region>\n" + + " <region active='true'>us-central-1</region>\n" + + " <region active='true'>us-east-3</region>\n" + + " </parallel>\n" + + " </prod>\n" + + "</deployment>" + ); + try { + DeploymentSpec.fromXml(r); + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertEquals("prod.us-west-1 is listed twice in deployment.xml", e.getMessage()); + } + } + + @Test(expected = IllegalArgumentException.class) + public void deploymentSpecWithIllegallyOrderedDeploymentSpec1() { + StringReader r = new StringReader( + "<deployment>\n" + + " <block-change days='sat' hours='10' time-zone='CET'/>\n" + + " <prod>\n" + + " <region active='true'>us-west-1</region>\n" + + " </prod>\n" + + " <block-change days='mon,tue' hours='15-16'/>\n" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + } + + @Test(expected = IllegalArgumentException.class) + public void deploymentSpecWithIllegallyOrderedDeploymentSpec2() { + StringReader r = new StringReader( + "<deployment>\n" + + " <block-change days='sat' hours='10' time-zone='CET'/>\n" + + " <test/>\n" + + " <prod>\n" + + " <region active='true'>us-west-1</region>\n" + + " </prod>\n" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + } + + @Test + public void deploymentSpecWithChangeBlocker() { + StringReader r = new StringReader( + "<deployment>\n" + + " <block-change revision='false' days='mon,tue' hours='15-16'/>\n" + + " <block-change days='sat' hours='10' time-zone='CET'/>\n" + + " <prod>\n" + + " <region active='true'>us-west-1</region>\n" + + " </prod>\n" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(2, spec.changeBlocker().size()); + assertTrue(spec.changeBlocker().get(0).blocksVersions()); + assertFalse(spec.changeBlocker().get(0).blocksRevisions()); + assertEquals(ZoneId.of("UTC"), spec.changeBlocker().get(0).window().zone()); + + assertTrue(spec.changeBlocker().get(1).blocksVersions()); + assertTrue(spec.changeBlocker().get(1).blocksRevisions()); + assertEquals(ZoneId.of("CET"), spec.changeBlocker().get(1).window().zone()); + + assertTrue(spec.canUpgradeAt(Instant.parse("2017-09-18T14:15:30.00Z"))); + assertFalse(spec.canUpgradeAt(Instant.parse("2017-09-18T15:15:30.00Z"))); + assertFalse(spec.canUpgradeAt(Instant.parse("2017-09-18T16:15:30.00Z"))); + assertTrue(spec.canUpgradeAt(Instant.parse("2017-09-18T17:15:30.00Z"))); + + assertTrue(spec.canUpgradeAt(Instant.parse("2017-09-23T09:15:30.00Z"))); + assertFalse(spec.canUpgradeAt(Instant.parse("2017-09-23T08:15:30.00Z"))); // 10 in CET + assertTrue(spec.canUpgradeAt(Instant.parse("2017-09-23T10:15:30.00Z"))); + } + + @Test + public void athenz_config_is_read_from_deployment() { + StringReader r = new StringReader( + "<deployment athenz-domain='domain' athenz-service='service'>\n" + + " <prod>\n" + + " <region active='true'>us-west-1</region>\n" + + " </prod>\n" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(spec.athenzDomain().get().value(), "domain"); + assertEquals(spec.athenzService(Environment.prod, RegionName.from("us-west-1")).get().value(), "service"); + } + + @Test + public void athenz_service_is_overridden_from_environment() { + StringReader r = new StringReader( + "<deployment athenz-domain='domain' athenz-service='service'>\n" + + " <test/>\n" + + " <prod athenz-service='prod-service'>\n" + + " <region active='true'>us-west-1</region>\n" + + " </prod>\n" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(spec.athenzDomain().get().value(), "domain"); + assertEquals(spec.athenzService(Environment.prod, RegionName.from("us-west-1")).get().value(), "prod-service"); + } + + @Test(expected = IllegalArgumentException.class) + public void it_fails_when_athenz_service_is_not_defined() { + StringReader r = new StringReader( + "<deployment athenz-domain='domain'>\n" + + " <prod>\n" + + " <region active='true'>us-west-1</region>\n" + + " </prod>\n" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + } + + @Test(expected = IllegalArgumentException.class) + public void it_fails_when_athenz_service_is_configured_but_not_athenz_domain() { + StringReader r = new StringReader( + "<deployment>\n" + + " <prod athenz-service='service'>\n" + + " <region active='true'>us-west-1</region>\n" + + " </prod>\n" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + } + + @Test + public void noNotifications() { + assertEquals(Notifications.none(), + DeploymentSpec.fromXml("<deployment />").notifications()); + } + + @Test + public void emptyNotifications() { + DeploymentSpec spec = DeploymentSpec.fromXml("<deployment>\n" + + " <notifications />" + + "</deployment>"); + assertEquals(Notifications.none(), + spec.notifications()); + } + + @Test + public void someNotifications() { + DeploymentSpec spec = DeploymentSpec.fromXml("<deployment>\n" + + " <notifications when=\"failing\">\n" + + " <email role=\"author\"/>\n" + + " <email address=\"john@dev\" when=\"failing-commit\"/>\n" + + " <email address=\"jane@dev\"/>\n" + + " </notifications>\n" + + "</deployment>"); + assertEquals(ImmutableSet.of(author), spec.notifications().emailRolesFor(failing)); + assertEquals(ImmutableSet.of(author), spec.notifications().emailRolesFor(failingCommit)); + assertEquals(ImmutableSet.of("john@dev", "jane@dev"), spec.notifications().emailAddressesFor(failingCommit)); + assertEquals(ImmutableSet.of("jane@dev"), spec.notifications().emailAddressesFor(failing)); + } + + @Test + public void customTesterFlavor() { + DeploymentSpec spec = DeploymentSpec.fromXml("<deployment>\n" + + " <test tester-flavor=\"d-1-4-20\" />\n" + + " <prod tester-flavor=\"d-2-8-50\">\n" + + " <region active=\"false\">us-north-7</region>\n" + + " </prod>\n" + + "</deployment>"); + assertEquals(Optional.of("d-1-4-20"), spec.steps().get(0).zones().get(0).testerFlavor()); + assertEquals(Optional.empty(), spec.steps().get(1).zones().get(0).testerFlavor()); + assertEquals(Optional.of("d-2-8-50"), spec.steps().get(2).zones().get(0).testerFlavor()); + } + + @Test + public void noEndpoints() { + assertEquals(Collections.emptyList(), DeploymentSpec.fromXml("<deployment />").endpoints()); + } + + @Test + public void emptyEndpoints() { + final var spec = DeploymentSpec.fromXml("<deployment><endpoints/></deployment>"); + assertEquals(Collections.emptyList(), spec.endpoints()); + } + + @Test + public void someEndpoints() { + final var spec = DeploymentSpec.fromXml("" + + "<deployment>" + + " <prod>" + + " <region active=\"true\">us-east</region>" + + " </prod>" + + " <endpoints>" + + " <endpoint id=\"foo\" container-id=\"bar\">" + + " <region>us-east</region>" + + " </endpoint>" + + " <endpoint id=\"nalle\" container-id=\"frosk\" />" + + " <endpoint container-id=\"quux\" />" + + " </endpoints>" + + "</deployment>"); + + assertEquals( + List.of("foo", "nalle", "default"), + spec.endpoints().stream().map(Endpoint::endpointId).collect(Collectors.toList()) + ); + + assertEquals( + List.of("bar", "frosk", "quux"), + spec.endpoints().stream().map(Endpoint::containerId).collect(Collectors.toList()) + ); + + assertEquals(Set.of(RegionName.from("us-east")), spec.endpoints().get(0).regions()); + } + @Test + public void invalidEndpoints() { + assertInvalid("<endpoint id='FOO' container-id='qrs'/>"); // Uppercase + assertInvalid("<endpoint id='123' container-id='qrs'/>"); // Starting with non-character + assertInvalid("<endpoint id='foo!' container-id='qrs'/>"); // Non-alphanumeric + assertInvalid("<endpoint id='foo.bar' container-id='qrs'/>"); + assertInvalid("<endpoint id='foo--bar' container-id='qrs'/>"); // Multiple consecutive dashes + assertInvalid("<endpoint id='foo-' container-id='qrs'/>"); // Trailing dash + assertInvalid("<endpoint id='foooooooooooo' container-id='qrs'/>"); // Too long + assertInvalid("<endpoint id='foo' container-id='qrs'/><endpoint id='foo' container-id='qrs'/>"); // Duplicate + } + + @Test + public void validEndpoints() { + assertEquals(List.of("default"), endpointIds("<endpoint container-id='qrs'/>")); + assertEquals(List.of("default"), endpointIds("<endpoint id='' container-id='qrs'/>")); + assertEquals(List.of("f"), endpointIds("<endpoint id='f' container-id='qrs'/>")); + assertEquals(List.of("foo"), endpointIds("<endpoint id='foo' container-id='qrs'/>")); + assertEquals(List.of("foo-bar"), endpointIds("<endpoint id='foo-bar' container-id='qrs'/>")); + assertEquals(List.of("foo", "bar"), endpointIds("<endpoint id='foo' container-id='qrs'/><endpoint id='bar' container-id='qrs'/>")); + assertEquals(List.of("fooooooooooo"), endpointIds("<endpoint id='fooooooooooo' container-id='qrs'/>")); + } + + @Test + public void endpointDefaultRegions() { + var spec = DeploymentSpec.fromXml("" + + "<deployment>" + + " <prod>" + + " <region active=\"true\">us-east</region>" + + " <region active=\"true\">us-west</region>" + + " </prod>" + + " <endpoints>" + + " <endpoint id=\"foo\" container-id=\"bar\">" + + " <region>us-east</region>" + + " </endpoint>" + + " <endpoint id=\"nalle\" container-id=\"frosk\" />" + + " <endpoint container-id=\"quux\" />" + + " </endpoints>" + + "</deployment>"); + + assertEquals(Set.of("us-east"), endpointRegions("foo", spec)); + assertEquals(Set.of("us-east", "us-west"), endpointRegions("nalle", spec)); + assertEquals(Set.of("us-east", "us-west"), endpointRegions("default", spec)); + } + + private static void assertInvalid(String endpointTag) { + try { + endpointIds(endpointTag); + fail("Expected exception for input '" + endpointTag + "'"); + } catch (IllegalArgumentException ignored) {} + } + + private static Set<String> endpointRegions(String endpointId, DeploymentSpec spec) { + return spec.endpoints().stream() + .filter(endpoint -> endpoint.endpointId().equals(endpointId)) + .flatMap(endpoint -> endpoint.regions().stream()) + .map(RegionName::value) + .collect(Collectors.toSet()); + } + + private static List<String> endpointIds(String endpointTag) { + var xml = "<deployment>" + + " <prod>" + + " <region active=\"true\">us-east</region>" + + " </prod>" + + " <endpoints>" + + endpointTag + + " </endpoints>" + + "</deployment>"; + + return DeploymentSpec.fromXml(xml).endpoints().stream() + .map(Endpoint::endpointId) + .collect(Collectors.toList()); + } + +} diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index 47eaf7a515a..c6035ac8d46 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -3,6 +3,7 @@ package com.yahoo.config.application.api; import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import org.junit.Test; @@ -14,7 +15,6 @@ import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; import static com.yahoo.config.application.api.Notifications.Role.author; import static com.yahoo.config.application.api.Notifications.When.failing; @@ -32,32 +32,36 @@ public class DeploymentSpecTest { @Test public void testSpec() { String specXml = "<deployment version='1.0'>" + - " <test/>" + + " <instance id='default'>" + + " <test/>" + + " </instance>" + "</deployment>"; StringReader r = new StringReader(specXml); DeploymentSpec spec = DeploymentSpec.fromXml(r); assertEquals(specXml, spec.xmlForm()); - assertEquals(1, spec.steps().size()); + assertEquals(1, spec.requireInstance("default").steps().size()); assertFalse(spec.majorVersion().isPresent()); - assertTrue(spec.steps().get(0).deploysTo(Environment.test)); - assertTrue(spec.includes(Environment.test, Optional.empty())); - assertFalse(spec.includes(Environment.test, Optional.of(RegionName.from("region1")))); - assertFalse(spec.includes(Environment.staging, Optional.empty())); - assertFalse(spec.includes(Environment.prod, Optional.empty())); - assertFalse(spec.globalServiceId().isPresent()); + assertTrue(spec.requireInstance("default").steps().get(0).deploysTo(Environment.test)); + assertTrue(spec.requireInstance("default").includes(Environment.test, Optional.empty())); + assertFalse(spec.requireInstance("default").includes(Environment.test, Optional.of(RegionName.from("region1")))); + assertFalse(spec.requireInstance("default").includes(Environment.staging, Optional.empty())); + assertFalse(spec.requireInstance("default").includes(Environment.prod, Optional.empty())); + assertFalse(spec.requireInstance("default").globalServiceId().isPresent()); } @Test public void testSpecPinningMajorVersion() { String specXml = "<deployment version='1.0' major-version='6'>" + - " <test/>" + + " <instance id='default'>" + + " <test/>" + + " </instance>" + "</deployment>"; StringReader r = new StringReader(specXml); DeploymentSpec spec = DeploymentSpec.fromXml(r); assertEquals(specXml, spec.xmlForm()); - assertEquals(1, spec.steps().size()); + assertEquals(1, spec.requireInstance("default").steps().size()); assertTrue(spec.majorVersion().isPresent()); assertEquals(6, (int)spec.majorVersion().get()); } @@ -66,164 +70,256 @@ public class DeploymentSpecTest { public void stagingSpec() { StringReader r = new StringReader( "<deployment version='1.0'>" + - " <staging/>" + + " <instance id='default'>" + + " <staging/>" + + " </instance>" + "</deployment>" ); DeploymentSpec spec = DeploymentSpec.fromXml(r); - assertEquals(2, spec.steps().size()); - assertTrue(spec.steps().get(0).deploysTo(Environment.test)); - assertTrue(spec.steps().get(1).deploysTo(Environment.staging)); - assertTrue(spec.includes(Environment.test, Optional.empty())); - assertFalse(spec.includes(Environment.test, Optional.of(RegionName.from("region1")))); - assertTrue(spec.includes(Environment.staging, Optional.empty())); - assertFalse(spec.includes(Environment.prod, Optional.empty())); - assertFalse(spec.globalServiceId().isPresent()); + assertEquals(2, spec.requireInstance("default").steps().size()); + assertTrue(spec.requireInstance("default").steps().get(0).deploysTo(Environment.test)); + assertTrue(spec.requireInstance("default").steps().get(1).deploysTo(Environment.staging)); + assertTrue(spec.requireInstance("default").includes(Environment.test, Optional.empty())); + assertFalse(spec.requireInstance("default").includes(Environment.test, Optional.of(RegionName.from("region1")))); + assertTrue(spec.requireInstance("default").includes(Environment.staging, Optional.empty())); + assertFalse(spec.requireInstance("default").includes(Environment.prod, Optional.empty())); + assertFalse(spec.requireInstance("default").globalServiceId().isPresent()); } @Test public void minimalProductionSpec() { StringReader r = new StringReader( - "<deployment version='1.0'>" + - " <prod>" + - " <region active='false'>us-east1</region>" + - " <region active='true'>us-west1</region>" + - " </prod>" + - "</deployment>" + "<deployment version='1.0'>" + + " <instance id='default'>" + + " <prod>" + + " <region active='false'>us-east1</region>" + + " <region active='true'>us-west1</region>" + + " </prod>" + + " </instance>" + + "</deployment>" ); DeploymentSpec spec = DeploymentSpec.fromXml(r); - assertEquals(4, spec.steps().size()); + assertEquals(4, spec.requireInstance("default").steps().size()); + + assertTrue(spec.requireInstance("default").steps().get(0).deploysTo(Environment.test)); - assertTrue(spec.steps().get(0).deploysTo(Environment.test)); + assertTrue(spec.requireInstance("default").steps().get(1).deploysTo(Environment.staging)); - assertTrue(spec.steps().get(1).deploysTo(Environment.staging)); + assertTrue(spec.requireInstance("default").steps().get(2).deploysTo(Environment.prod, Optional.of(RegionName.from("us-east1")))); + assertFalse(((DeploymentSpec.DeclaredZone)spec.requireInstance("default").steps().get(2)).active()); - assertTrue(spec.steps().get(2).deploysTo(Environment.prod, Optional.of(RegionName.from("us-east1")))); - assertFalse(((DeploymentSpec.DeclaredZone)spec.steps().get(2)).active()); + assertTrue(spec.requireInstance("default").steps().get(3).deploysTo(Environment.prod, Optional.of(RegionName.from("us-west1")))); + assertTrue(((DeploymentSpec.DeclaredZone)spec.requireInstance("default").steps().get(3)).active()); - assertTrue(spec.steps().get(3).deploysTo(Environment.prod, Optional.of(RegionName.from("us-west1")))); - assertTrue(((DeploymentSpec.DeclaredZone)spec.steps().get(3)).active()); + assertTrue(spec.requireInstance("default").includes(Environment.test, Optional.empty())); + assertFalse(spec.requireInstance("default").includes(Environment.test, Optional.of(RegionName.from("region1")))); + assertTrue(spec.requireInstance("default").includes(Environment.staging, Optional.empty())); + assertTrue(spec.requireInstance("default").includes(Environment.prod, Optional.of(RegionName.from("us-east1")))); + assertTrue(spec.requireInstance("default").includes(Environment.prod, Optional.of(RegionName.from("us-west1")))); + assertFalse(spec.requireInstance("default").includes(Environment.prod, Optional.of(RegionName.from("no-such-region")))); + assertFalse(spec.requireInstance("default").globalServiceId().isPresent()); - assertTrue(spec.includes(Environment.test, Optional.empty())); - assertFalse(spec.includes(Environment.test, Optional.of(RegionName.from("region1")))); - assertTrue(spec.includes(Environment.staging, Optional.empty())); - assertTrue(spec.includes(Environment.prod, Optional.of(RegionName.from("us-east1")))); - assertTrue(spec.includes(Environment.prod, Optional.of(RegionName.from("us-west1")))); - assertFalse(spec.includes(Environment.prod, Optional.of(RegionName.from("no-such-region")))); - assertFalse(spec.globalServiceId().isPresent()); - - assertEquals(DeploymentSpec.UpgradePolicy.defaultPolicy, spec.upgradePolicy()); + assertEquals(DeploymentSpec.UpgradePolicy.defaultPolicy, spec.requireInstance("default").upgradePolicy()); } @Test public void maximalProductionSpec() { StringReader r = new StringReader( - "<deployment version='1.0'>" + - " <test/>" + - " <staging/>" + - " <prod>" + - " <region active='false'>us-east1</region>" + - " <delay hours='3' minutes='30'/>" + - " <region active='true'>us-west1</region>" + - " </prod>" + - "</deployment>" + "<deployment version='1.0'>" + + " <instance id='default'>" + // The block checked by assertCorrectFirstInstance + " <test/>" + + " <staging/>" + + " <prod>" + + " <region active='false'>us-east1</region>" + + " <delay hours='3' minutes='30'/>" + + " <region active='true'>us-west1</region>" + + " </prod>" + + " </instance>" + + "</deployment>" ); DeploymentSpec spec = DeploymentSpec.fromXml(r); - assertEquals(5, spec.steps().size()); - assertEquals(4, spec.zones().size()); + assertCorrectFirstInstance(spec.requireInstance("default")); + } - assertTrue(spec.steps().get(0).deploysTo(Environment.test)); + @Test + public void maximalProductionSpecMultipleInstances() { + StringReader r = new StringReader( + "<deployment version='1.0'>" + + " <instance id='instance1'>" + // The block checked by assertCorrectFirstInstance + " <test/>" + + " <staging/>" + + " <prod>" + + " <region active='false'>us-east1</region>" + + " <delay hours='3' minutes='30'/>" + + " <region active='true'>us-west1</region>" + + " </prod>" + + " </instance>" + + " <instance id='instance2'>" + + " <prod>" + + " <region active='true'>us-central1</region>" + + " </prod>" + + " </instance>" + + "</deployment>" + ); - assertTrue(spec.steps().get(1).deploysTo(Environment.staging)); + DeploymentSpec spec = DeploymentSpec.fromXml(r); - assertTrue(spec.steps().get(2).deploysTo(Environment.prod, Optional.of(RegionName.from("us-east1")))); - assertFalse(((DeploymentSpec.DeclaredZone)spec.steps().get(2)).active()); + assertCorrectFirstInstance(spec.requireInstance("instance1")); - assertTrue(spec.steps().get(3) instanceof DeploymentSpec.Delay); - assertEquals(3 * 60 * 60 + 30 * 60, ((DeploymentSpec.Delay)spec.steps().get(3)).duration().getSeconds()); + DeploymentInstanceSpec instance2 = spec.requireInstance("instance2"); + assertEquals(1, instance2.steps().size()); + assertEquals(1, instance2.zones().size()); - assertTrue(spec.steps().get(4).deploysTo(Environment.prod, Optional.of(RegionName.from("us-west1")))); - assertTrue(((DeploymentSpec.DeclaredZone)spec.steps().get(4)).active()); + assertTrue(instance2.steps().get(0).deploysTo(Environment.prod, Optional.of(RegionName.from("us-central1")))); + } - assertTrue(spec.includes(Environment.test, Optional.empty())); - assertFalse(spec.includes(Environment.test, Optional.of(RegionName.from("region1")))); - assertTrue(spec.includes(Environment.staging, Optional.empty())); - assertTrue(spec.includes(Environment.prod, Optional.of(RegionName.from("us-east1")))); - assertTrue(spec.includes(Environment.prod, Optional.of(RegionName.from("us-west1")))); - assertFalse(spec.includes(Environment.prod, Optional.of(RegionName.from("no-such-region")))); - assertFalse(spec.globalServiceId().isPresent()); + @Test + public void testMultipleInstancesShortForm() { + StringReader r = new StringReader( + "<deployment version='1.0'>" + + " <instance id='instance1, instance2'>" + // The block checked by assertCorrectFirstInstance + " <test/>" + + " <staging/>" + + " <prod>" + + " <region active='false'>us-east1</region>" + + " <delay hours='3' minutes='30'/>" + + " <region active='true'>us-west1</region>" + + " </prod>" + + " </instance>" + + "</deployment>" + ); + + DeploymentSpec spec = DeploymentSpec.fromXml(r); + + assertCorrectFirstInstance(spec.requireInstance("instance1")); + assertCorrectFirstInstance(spec.requireInstance("instance2")); + } + + private void assertCorrectFirstInstance(DeploymentInstanceSpec instance) { + assertEquals(5, instance.steps().size()); + assertEquals(4, instance.zones().size()); + + assertTrue(instance.steps().get(0).deploysTo(Environment.test)); + + assertTrue(instance.steps().get(1).deploysTo(Environment.staging)); + + assertTrue(instance.steps().get(2).deploysTo(Environment.prod, Optional.of(RegionName.from("us-east1")))); + assertFalse(((DeploymentSpec.DeclaredZone)instance.steps().get(2)).active()); + + assertTrue(instance.steps().get(3) instanceof DeploymentSpec.Delay); + assertEquals(3 * 60 * 60 + 30 * 60, instance.steps().get(3).delay().getSeconds()); + + assertTrue(instance.steps().get(4).deploysTo(Environment.prod, Optional.of(RegionName.from("us-west1")))); + assertTrue(((DeploymentSpec.DeclaredZone)instance.steps().get(4)).active()); + + assertTrue(instance.includes(Environment.test, Optional.empty())); + assertFalse(instance.includes(Environment.test, Optional.of(RegionName.from("region1")))); + assertTrue(instance.includes(Environment.staging, Optional.empty())); + assertTrue(instance.includes(Environment.prod, Optional.of(RegionName.from("us-east1")))); + assertTrue(instance.includes(Environment.prod, Optional.of(RegionName.from("us-west1")))); + assertFalse(instance.includes(Environment.prod, Optional.of(RegionName.from("no-such-region")))); + assertFalse(instance.globalServiceId().isPresent()); } @Test public void productionSpecWithGlobalServiceId() { StringReader r = new StringReader( "<deployment version='1.0'>" + - " <prod global-service-id='query'>" + - " <region active='true'>us-east-1</region>" + - " <region active='true'>us-west-1</region>" + - " </prod>" + + " <instance id='default'>" + + " <prod global-service-id='query'>" + + " <region active='true'>us-east-1</region>" + + " <region active='true'>us-west-1</region>" + + " </prod>" + + " </instance>" + "</deployment>" ); DeploymentSpec spec = DeploymentSpec.fromXml(r); - assertEquals(spec.globalServiceId(), Optional.of("query")); + assertEquals(spec.requireInstance("default").globalServiceId(), Optional.of("query")); } @Test(expected=IllegalArgumentException.class) public void globalServiceIdInTest() { StringReader r = new StringReader( "<deployment version='1.0'>" + - " <test global-service-id='query' />" + + " <instance id='default'>" + + " <test global-service-id='query' />" + + " </instance>" + "</deployment>" ); - DeploymentSpec spec = DeploymentSpec.fromXml(r); + DeploymentSpec.fromXml(r); } @Test(expected=IllegalArgumentException.class) public void globalServiceIdInStaging() { StringReader r = new StringReader( "<deployment version='1.0'>" + - " <staging global-service-id='query' />" + + " <instance id='default'>" + + " <staging global-service-id='query' />" + + " </instance>" + "</deployment>" ); - DeploymentSpec spec = DeploymentSpec.fromXml(r); + DeploymentSpec.fromXml(r); } @Test public void productionSpecWithGlobalServiceIdBeforeStaging() { StringReader r = new StringReader( "<deployment>" + - " <test/>" + - " <prod global-service-id='qrs'>" + - " <region active='true'>us-west-1</region>" + - " <region active='true'>us-central-1</region>" + - " <region active='true'>us-east-3</region>" + - " </prod>" + - " <staging/>" + + " <instance id='default'>" + + " <test/>" + + " <prod global-service-id='qrs'>" + + " <region active='true'>us-west-1</region>" + + " <region active='true'>us-central-1</region>" + + " <region active='true'>us-east-3</region>" + + " </prod>" + + " <staging/>" + + " </instance>" + "</deployment>" ); DeploymentSpec spec = DeploymentSpec.fromXml(r); - assertEquals("qrs", spec.globalServiceId().get()); + assertEquals("qrs", spec.requireInstance("default").globalServiceId().get()); } @Test public void productionSpecWithUpgradePolicy() { StringReader r = new StringReader( "<deployment>" + - " <upgrade policy='canary'/>" + - " <prod>" + - " <region active='true'>us-west-1</region>" + - " <region active='true'>us-central-1</region>" + - " <region active='true'>us-east-3</region>" + - " </prod>" + + " <instance id='default'>" + + " <upgrade policy='canary'/>" + + " <prod>" + + " <region active='true'>us-west-1</region>" + + " <region active='true'>us-central-1</region>" + + " <region active='true'>us-east-3</region>" + + " </prod>" + + " </instance>" + "</deployment>" ); DeploymentSpec spec = DeploymentSpec.fromXml(r); - assertEquals("canary", spec.upgradePolicy().toString()); + assertEquals("canary", spec.requireInstance("default").upgradePolicy().toString()); + } + + @Test + public void upgradePolicyDefault() { + StringReader r = new StringReader( + "<deployment version='1.0'>" + + " <upgrade policy='canary'/>" + + " <instance id='instance1'>" + + " <upgrade policy='conservative'/>" + + " </instance>" + + " <instance id='instance2'>" + + " </instance>" + + "</deployment>" + ); + + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals("conservative", spec.requireInstance("instance1").upgradePolicy().toString()); + assertEquals("canary", spec.requireInstance("instance2").upgradePolicy().toString()); } @Test @@ -231,14 +327,16 @@ public class DeploymentSpecTest { try { StringReader r = new StringReader( "<deployment>" + - " <upgrade policy='canary'/>" + - " <prod>" + - " <region active='true'>us-west-1</region>" + - " <delay hours='23'/>" + - " <region active='true'>us-central-1</region>" + - " <delay minutes='59' seconds='61'/>" + - " <region active='true'>us-east-3</region>" + - " </prod>" + + " <instance id='default'>" + + " <upgrade policy='canary'/>" + + " <prod>" + + " <region active='true'>us-west-1</region>" + + " <delay hours='23'/>" + + " <region active='true'>us-central-1</region>" + + " <delay minutes='59' seconds='61'/>" + + " <region active='true'>us-east-3</region>" + + " </prod>" + + " </instance>" + "</deployment>" ); DeploymentSpec.fromXml(r); @@ -252,7 +350,7 @@ public class DeploymentSpecTest { @Test public void testEmpty() { - assertFalse(DeploymentSpec.empty.globalServiceId().isPresent()); + assertFalse(DeploymentSpec.empty.requireInstance("default").globalServiceId().isPresent()); assertEquals(DeploymentSpec.UpgradePolicy.defaultPolicy, DeploymentSpec.empty.upgradePolicy()); assertTrue(DeploymentSpec.empty.steps().isEmpty()); assertEquals("<deployment version='1.0'/>", DeploymentSpec.empty.xmlForm()); @@ -261,36 +359,139 @@ public class DeploymentSpecTest { @Test public void productionSpecWithParallelDeployments() { StringReader r = new StringReader( - "<deployment>\n" + - " <prod> \n" + - " <region active='true'>us-west-1</region>\n" + - " <parallel>\n" + - " <region active='true'>us-central-1</region>\n" + - " <region active='true'>us-east-3</region>\n" + - " </parallel>\n" + - " </prod>\n" + - "</deployment>" + "<deployment>" + + " <instance id='default'>" + + " <prod>" + + " <region active='true'>us-west-1</region>" + + " <parallel>" + + " <region active='true'>us-central-1</region>" + + " <region active='true'>us-east-3</region>" + + " </parallel>" + + " </prod>" + + " </instance>" + + "</deployment>" ); DeploymentSpec spec = DeploymentSpec.fromXml(r); - DeploymentSpec.ParallelZones parallelZones = ((DeploymentSpec.ParallelZones) spec.steps().get(3)); + DeploymentSpec.ParallelZones parallelZones = ((DeploymentSpec.ParallelZones) spec.requireInstance("default").steps().get(3)); assertEquals(2, parallelZones.zones().size()); assertEquals(RegionName.from("us-central-1"), parallelZones.zones().get(0).region().get()); assertEquals(RegionName.from("us-east-3"), parallelZones.zones().get(1).region().get()); } @Test + public void testTestAndStagingOutsideAndInsideInstance() { + StringReader r = new StringReader( + "<deployment>" + + " <test/>" + + " <staging/>" + + " <instance id='instance0'>" + + " <prod>" + + " <region active='true'>us-west-1</region>" + + " </prod>" + + " </instance>" + + " <instance id='instance1'>" + + " <test/>" + + " <staging/>" + + " <prod>" + + " <region active='true'>us-west-1</region>" + + " </prod>" + + " </instance>" + + "</deployment>" + ); + + DeploymentSpec spec = DeploymentSpec.fromXml(r); + List<DeploymentSpec.Step> steps = spec.steps(); + assertEquals(4, steps.size()); + assertEquals("test", steps.get(0).toString()); + assertEquals("staging", steps.get(1).toString()); + assertEquals("instance 'instance0'", steps.get(2).toString()); + assertEquals("instance 'instance1'", steps.get(3).toString()); + + List<DeploymentSpec.Step> instance0Steps = ((DeploymentInstanceSpec)steps.get(2)).steps(); + assertEquals(1, instance0Steps.size()); + assertEquals("prod.us-west-1", instance0Steps.get(0).toString()); + + List<DeploymentSpec.Step> instance1Steps = ((DeploymentInstanceSpec)steps.get(3)).steps(); + assertEquals(3, instance1Steps.size()); + assertEquals("test", instance1Steps.get(0).toString()); + assertEquals("staging", instance1Steps.get(1).toString()); + assertEquals("prod.us-west-1", instance1Steps.get(2).toString()); + } + + @Test + public void testParallelInstances() { + StringReader r = new StringReader( + "<deployment>" + + " <parallel>" + + " <instance id='instance0'>" + + " <prod>" + + " <region active='true'>us-west-1</region>" + + " </prod>" + + " </instance>" + + " <instance id='instance1'>" + + " <prod>" + + " <region active='true'>us-east-3</region>" + + " </prod>" + + " </instance>" + + " </parallel>" + + "</deployment>" + ); + + DeploymentSpec spec = DeploymentSpec.fromXml(r); + List<DeploymentSpec.Step> steps = spec.steps(); + assertEquals(3, steps.size()); + assertEquals("test", steps.get(0).toString()); + assertEquals("staging", steps.get(1).toString()); + assertEquals("2 parallel steps", steps.get(2).toString()); + + List<DeploymentSpec.Step> parallelSteps = steps.get(2).steps(); + assertEquals("instance 'instance0'", parallelSteps.get(0).toString()); + assertEquals("instance 'instance1'", parallelSteps.get(1).toString()); + } + + @Test + public void testInstancesWithDelay() { + StringReader r = new StringReader( + "<deployment>" + + " <instance id='instance0'>" + + " <prod>" + + " <region active='true'>us-west-1</region>" + + " </prod>" + + " </instance>" + + " <delay hours='12'/>" + + " <instance id='instance1'>" + + " <prod>" + + " <region active='true'>us-east-3</region>" + + " </prod>" + + " </instance>" + + "</deployment>" + ); + + DeploymentSpec spec = DeploymentSpec.fromXml(r); + List<DeploymentSpec.Step> steps = spec.steps(); + assertEquals(5, steps.size()); + assertEquals("test", steps.get(0).toString()); + assertEquals("staging", steps.get(1).toString()); + assertEquals("instance 'instance0'", steps.get(2).toString()); + assertEquals("delay PT12H", steps.get(3).toString()); + assertEquals("instance 'instance1'", steps.get(4).toString()); + } + + @Test public void productionSpecWithDuplicateRegions() { StringReader r = new StringReader( - "<deployment>\n" + - " <prod>\n" + - " <region active='true'>us-west-1</region>\n" + - " <parallel>\n" + - " <region active='true'>us-west-1</region>\n" + - " <region active='true'>us-central-1</region>\n" + - " <region active='true'>us-east-3</region>\n" + - " </parallel>\n" + - " </prod>\n" + - "</deployment>" + "<deployment>" + + " <instance id='default'>" + + " <prod>" + + " <region active='true'>us-west-1</region>" + + " <parallel>" + + " <region active='true'>us-west-1</region>" + + " <region active='true'>us-central-1</region>" + + " <region active='true'>us-east-3</region>" + + " </parallel>" + + " </prod>" + + " </instance>" + + "</deployment>" ); try { DeploymentSpec.fromXml(r); @@ -303,197 +504,349 @@ public class DeploymentSpecTest { @Test(expected = IllegalArgumentException.class) public void deploymentSpecWithIllegallyOrderedDeploymentSpec1() { StringReader r = new StringReader( - "<deployment>\n" + - " <block-change days='sat' hours='10' time-zone='CET'/>\n" + - " <prod>\n" + - " <region active='true'>us-west-1</region>\n" + - " </prod>\n" + - " <block-change days='mon,tue' hours='15-16'/>\n" + + "<deployment>" + + " <instance id='default'>" + + " <block-change days='sat' hours='10' time-zone='CET'/>" + + " <prod>" + + " <region active='true'>us-west-1</region>" + + " </prod>" + + " <block-change days='mon,tue' hours='15-16'/>" + + " </instance>" + "</deployment>" ); - DeploymentSpec spec = DeploymentSpec.fromXml(r); + DeploymentSpec.fromXml(r); } @Test(expected = IllegalArgumentException.class) public void deploymentSpecWithIllegallyOrderedDeploymentSpec2() { StringReader r = new StringReader( "<deployment>\n" + - " <block-change days='sat' hours='10' time-zone='CET'/>\n" + - " <test/>\n" + - " <prod>\n" + - " <region active='true'>us-west-1</region>\n" + - " </prod>\n" + + " <instance id='default'>" + + " <block-change days='sat' hours='10' time-zone='CET'/>" + + " <test/>" + + " <prod>" + + " <region active='true'>us-west-1</region>" + + " </prod>" + + " </instance>" + "</deployment>" ); - DeploymentSpec spec = DeploymentSpec.fromXml(r); + DeploymentSpec.fromXml(r); } @Test public void deploymentSpecWithChangeBlocker() { StringReader r = new StringReader( - "<deployment>\n" + - " <block-change revision='false' days='mon,tue' hours='15-16'/>\n" + - " <block-change days='sat' hours='10' time-zone='CET'/>\n" + - " <prod>\n" + - " <region active='true'>us-west-1</region>\n" + - " </prod>\n" + + "<deployment>" + + " <instance id='default'>" + + " <block-change revision='false' days='mon,tue' hours='15-16'/>" + + " <block-change days='sat' hours='10' time-zone='CET'/>" + + " <prod>" + + " <region active='true'>us-west-1</region>" + + " </prod>" + + " </instance>" + "</deployment>" ); DeploymentSpec spec = DeploymentSpec.fromXml(r); - assertEquals(2, spec.changeBlocker().size()); - assertTrue(spec.changeBlocker().get(0).blocksVersions()); - assertFalse(spec.changeBlocker().get(0).blocksRevisions()); - assertEquals(ZoneId.of("UTC"), spec.changeBlocker().get(0).window().zone()); + assertEquals(2, spec.requireInstance("default").changeBlocker().size()); + assertTrue(spec.requireInstance("default").changeBlocker().get(0).blocksVersions()); + assertFalse(spec.requireInstance("default").changeBlocker().get(0).blocksRevisions()); + assertEquals(ZoneId.of("UTC"), spec.requireInstance("default").changeBlocker().get(0).window().zone()); - assertTrue(spec.changeBlocker().get(1).blocksVersions()); - assertTrue(spec.changeBlocker().get(1).blocksRevisions()); - assertEquals(ZoneId.of("CET"), spec.changeBlocker().get(1).window().zone()); + assertTrue(spec.requireInstance("default").changeBlocker().get(1).blocksVersions()); + assertTrue(spec.requireInstance("default").changeBlocker().get(1).blocksRevisions()); + assertEquals(ZoneId.of("CET"), spec.requireInstance("default").changeBlocker().get(1).window().zone()); - assertTrue(spec.canUpgradeAt(Instant.parse("2017-09-18T14:15:30.00Z"))); - assertFalse(spec.canUpgradeAt(Instant.parse("2017-09-18T15:15:30.00Z"))); - assertFalse(spec.canUpgradeAt(Instant.parse("2017-09-18T16:15:30.00Z"))); - assertTrue(spec.canUpgradeAt(Instant.parse("2017-09-18T17:15:30.00Z"))); + assertTrue(spec.requireInstance("default").canUpgradeAt(Instant.parse("2017-09-18T14:15:30.00Z"))); + assertFalse(spec.requireInstance("default").canUpgradeAt(Instant.parse("2017-09-18T15:15:30.00Z"))); + assertFalse(spec.requireInstance("default").canUpgradeAt(Instant.parse("2017-09-18T16:15:30.00Z"))); + assertTrue(spec.requireInstance("default").canUpgradeAt(Instant.parse("2017-09-18T17:15:30.00Z"))); - assertTrue(spec.canUpgradeAt(Instant.parse("2017-09-23T09:15:30.00Z"))); - assertFalse(spec.canUpgradeAt(Instant.parse("2017-09-23T08:15:30.00Z"))); // 10 in CET - assertTrue(spec.canUpgradeAt(Instant.parse("2017-09-23T10:15:30.00Z"))); + assertTrue(spec.requireInstance("default").canUpgradeAt(Instant.parse("2017-09-23T09:15:30.00Z"))); + assertFalse(spec.requireInstance("default").canUpgradeAt(Instant.parse("2017-09-23T08:15:30.00Z"))); // 10 in CET + assertTrue(spec.requireInstance("default").canUpgradeAt(Instant.parse("2017-09-23T10:15:30.00Z"))); + } + + @Test + public void testChangeBlockerInheritance() { + StringReader r = new StringReader( + "<deployment version='1.0'>" + + " <block-change revision='false' days='mon,tue' hours='15-16'/>" + + " <instance id='instance1'>" + + " <block-change days='sat' hours='10' time-zone='CET'/>" + + " </instance>" + + " <instance id='instance2'>" + + " </instance>" + + "</deployment>" + ); + + DeploymentSpec spec = DeploymentSpec.fromXml(r); + + String inheritedChangeBlocker = "change blocker revision=false version=true window=time window for hour(s) [15, 16] on [monday, tuesday] in UTC"; + + assertEquals(2, spec.requireInstance("instance1").changeBlocker().size()); + assertEquals(inheritedChangeBlocker, spec.requireInstance("instance1").changeBlocker().get(0).toString()); + assertEquals("change blocker revision=true version=true window=time window for hour(s) [10] on [saturday] in CET", + spec.requireInstance("instance1").changeBlocker().get(1).toString()); + + assertEquals(1, spec.requireInstance("instance2").changeBlocker().size()); + assertEquals(inheritedChangeBlocker, spec.requireInstance("instance2").changeBlocker().get(0).toString()); } @Test public void athenz_config_is_read_from_deployment() { StringReader r = new StringReader( - "<deployment athenz-domain='domain' athenz-service='service'>\n" + - " <prod>\n" + - " <region active='true'>us-west-1</region>\n" + - " </prod>\n" + + "<deployment athenz-domain='domain' athenz-service='service'>" + + " <instance id='instance1'>" + + " <prod>" + + " <region active='true'>us-west-1</region>" + + " </prod>" + + " </instance>" + "</deployment>" ); DeploymentSpec spec = DeploymentSpec.fromXml(r); - assertEquals(spec.athenzDomain().get().value(), "domain"); - assertEquals(spec.athenzService(Environment.prod, RegionName.from("us-west-1")).get().value(), "service"); + assertEquals("domain", spec.athenzDomain().get().value()); + assertEquals("service", spec.athenzService(InstanceName.from("instance1"), + Environment.prod, + RegionName.from("us-west-1")).get().value()); + assertEquals("service", spec.athenzService(InstanceName.from("non-existent"), + Environment.prod, + RegionName.from("us-west-1")).get().value()); + assertEquals("domain", spec.requireInstance("instance1").athenzDomain().get().value()); + assertEquals("service", spec.requireInstance("instance1").athenzService(Environment.prod, + RegionName.from("us-west-1")).get().value()); + } + + @Test + public void athenz_config_is_read_from_instance() { + StringReader r = new StringReader( + "<deployment>" + + " <instance id='default' athenz-domain='domain' athenz-service='service'>" + + " <prod>" + + " <region active='true'>us-west-1</region>" + + " </prod>" + + " </instance>" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(spec.requireInstance("default").athenzDomain().get().value(), "domain"); + assertEquals(spec.requireInstance("default").athenzService(Environment.prod, RegionName.from("us-west-1")).get().value(), "service"); + assertEquals(Optional.empty(), spec.athenzService(InstanceName.from("non-existent"), + Environment.prod, + RegionName.from("us-west-1"))); } @Test public void athenz_service_is_overridden_from_environment() { StringReader r = new StringReader( - "<deployment athenz-domain='domain' athenz-service='service'>\n" + - " <test/>\n" + - " <prod athenz-service='prod-service'>\n" + - " <region active='true'>us-west-1</region>\n" + - " </prod>\n" + + "<deployment athenz-domain='domain' athenz-service='service'>" + + " <instance id='default' athenz-domain='domain' athenz-service='service'>" + + " <test/>" + + " <prod athenz-service='prod-service'>" + + " <region active='true'>us-west-1</region>" + + " </prod>" + + " </instance>" + "</deployment>" ); DeploymentSpec spec = DeploymentSpec.fromXml(r); - assertEquals(spec.athenzDomain().get().value(), "domain"); - assertEquals(spec.athenzService(Environment.prod, RegionName.from("us-west-1")).get().value(), "prod-service"); + assertEquals(spec.requireInstance("default").athenzDomain().get().value(), "domain"); + assertEquals(spec.athenzService(InstanceName.from("default"), + Environment.prod, + RegionName.from("us-west-1")).get().value(), + "prod-service"); + assertEquals(spec.requireInstance("default").athenzService(Environment.prod, + RegionName.from("us-west-1")).get().value(), + "prod-service"); + assertEquals(spec.athenzService(InstanceName.from("non-existent"), + Environment.prod, + RegionName.from("us-west-1")).get().value(), + "service"); } @Test(expected = IllegalArgumentException.class) public void it_fails_when_athenz_service_is_not_defined() { StringReader r = new StringReader( - "<deployment athenz-domain='domain'>\n" + - " <prod>\n" + - " <region active='true'>us-west-1</region>\n" + - " </prod>\n" + + "<deployment>" + + " <instance id='default' athenz-domain='domain'>" + + " <prod>" + + " <region active='true'>us-west-1</region>" + + " </prod>" + + " </instance>" + "</deployment>" ); - DeploymentSpec spec = DeploymentSpec.fromXml(r); + DeploymentSpec.fromXml(r); } @Test(expected = IllegalArgumentException.class) public void it_fails_when_athenz_service_is_configured_but_not_athenz_domain() { StringReader r = new StringReader( - "<deployment>\n" + - " <prod athenz-service='service'>\n" + - " <region active='true'>us-west-1</region>\n" + - " </prod>\n" + + "<deployment>" + + " <instance id='default'>" + + " <prod athenz-service='service'>" + + " <region active='true'>us-west-1</region>" + + " </prod>" + + " </instance>" + "</deployment>" ); - DeploymentSpec spec = DeploymentSpec.fromXml(r); + DeploymentSpec.fromXml(r); } @Test public void noNotifications() { assertEquals(Notifications.none(), - DeploymentSpec.fromXml("<deployment />").notifications()); + DeploymentSpec.fromXml("<deployment>" + + " <instance id='default'/>" + + "</deployment>").requireInstance("default").notifications()); } @Test public void emptyNotifications() { - DeploymentSpec spec = DeploymentSpec.fromXml("<deployment>\n" + - " <notifications />" + + DeploymentSpec spec = DeploymentSpec.fromXml("<deployment>" + + " <instance id='default'>" + + " <notifications/>" + + " </instance>" + "</deployment>"); - assertEquals(Notifications.none(), - spec.notifications()); + assertEquals(Notifications.none(), spec.requireInstance("default").notifications()); } @Test public void someNotifications() { DeploymentSpec spec = DeploymentSpec.fromXml("<deployment>\n" + - " <notifications when=\"failing\">\n" + - " <email role=\"author\"/>\n" + - " <email address=\"john@dev\" when=\"failing-commit\"/>\n" + - " <email address=\"jane@dev\"/>\n" + - " </notifications>\n" + + " <instance id='default'>" + + " <notifications when=\"failing\">" + + " <email role=\"author\"/>" + + " <email address=\"john@dev\" when=\"failing-commit\"/>" + + " <email address=\"jane@dev\"/>" + + " </notifications>" + + " </instance>" + "</deployment>"); - assertEquals(ImmutableSet.of(author), spec.notifications().emailRolesFor(failing)); - assertEquals(ImmutableSet.of(author), spec.notifications().emailRolesFor(failingCommit)); - assertEquals(ImmutableSet.of("john@dev", "jane@dev"), spec.notifications().emailAddressesFor(failingCommit)); - assertEquals(ImmutableSet.of("jane@dev"), spec.notifications().emailAddressesFor(failing)); + assertEquals(ImmutableSet.of(author), spec.requireInstance("default").notifications().emailRolesFor(failing)); + assertEquals(ImmutableSet.of(author), spec.requireInstance("default").notifications().emailRolesFor(failingCommit)); + assertEquals(ImmutableSet.of("john@dev", "jane@dev"), spec.requireInstance("default").notifications().emailAddressesFor(failingCommit)); + assertEquals(ImmutableSet.of("jane@dev"), spec.requireInstance("default").notifications().emailAddressesFor(failing)); + } + + @Test + public void notificationsWithMultipleInstances() { + StringReader r = new StringReader( + "<deployment version='1.0'>" + + " <instance id='instance1'>" + + " <notifications when=\"failing\">" + + " <email role=\"author\"/>" + + " <email address=\"john@operator\"/>" + + " </notifications>" + + " </instance>" + + " <instance id='instance2'>" + + " <notifications when=\"failing-commit\">" + + " <email role=\"author\"/>" + + " <email address=\"mary@dev\"/>" + + " </notifications>" + + " </instance>" + + "</deployment>" + ); + + DeploymentSpec spec = DeploymentSpec.fromXml(r); + DeploymentInstanceSpec instance1 = spec.requireInstance("instance1"); + assertEquals(Set.of(author), instance1.notifications().emailRolesFor(failing)); + assertEquals(Set.of("john@operator"), instance1.notifications().emailAddressesFor(failing)); + + DeploymentInstanceSpec instance2 = spec.requireInstance("instance2"); + assertEquals(Set.of(author), instance2.notifications().emailRolesFor(failingCommit)); + assertEquals(Set.of("mary@dev"), instance2.notifications().emailAddressesFor(failingCommit)); + } + + @Test + public void notificationsDefault() { + StringReader r = new StringReader( + "<deployment version='1.0'>" + + " <notifications when=\"failing-commit\">" + + " <email role=\"author\"/>" + + " <email address=\"mary@dev\"/>" + + " </notifications>" + + " <instance id='instance1'>" + + " <notifications when=\"failing\">" + + " <email role=\"author\"/>" + + " <email address=\"john@operator\"/>" + + " </notifications>" + + " </instance>" + + " <instance id='instance2'>" + + " </instance>" + + "</deployment>" + ); + + DeploymentSpec spec = DeploymentSpec.fromXml(r); + DeploymentInstanceSpec instance1 = spec.requireInstance("instance1"); + assertEquals(Set.of(author), instance1.notifications().emailRolesFor(failing)); + assertEquals(Set.of("john@operator"), instance1.notifications().emailAddressesFor(failing)); + + DeploymentInstanceSpec instance2 = spec.requireInstance("instance2"); + assertEquals(Set.of(author), instance2.notifications().emailRolesFor(failingCommit)); + assertEquals(Set.of("mary@dev"), instance2.notifications().emailAddressesFor(failingCommit)); } @Test public void customTesterFlavor() { - DeploymentSpec spec = DeploymentSpec.fromXml("<deployment>\n" + - " <test tester-flavor=\"d-1-4-20\" />\n" + - " <prod tester-flavor=\"d-2-8-50\">\n" + - " <region active=\"false\">us-north-7</region>\n" + - " </prod>\n" + + DeploymentSpec spec = DeploymentSpec.fromXml("<deployment>" + + " <instance id='default'>" + + " <test tester-flavor=\"d-1-4-20\" />" + + " <prod tester-flavor=\"d-2-8-50\">" + + " <region active=\"false\">us-north-7</region>" + + " </prod>" + + " </instance>" + "</deployment>"); - assertEquals(Optional.of("d-1-4-20"), spec.steps().get(0).zones().get(0).testerFlavor()); - assertEquals(Optional.empty(), spec.steps().get(1).zones().get(0).testerFlavor()); - assertEquals(Optional.of("d-2-8-50"), spec.steps().get(2).zones().get(0).testerFlavor()); + assertEquals(Optional.of("d-1-4-20"), spec.requireInstance("default").steps().get(0).zones().get(0).testerFlavor()); + assertEquals(Optional.empty(), spec.requireInstance("default").steps().get(1).zones().get(0).testerFlavor()); + assertEquals(Optional.of("d-2-8-50"), spec.requireInstance("default").steps().get(2).zones().get(0).testerFlavor()); } @Test public void noEndpoints() { - assertEquals(Collections.emptyList(), DeploymentSpec.fromXml("<deployment />").endpoints()); + assertEquals(Collections.emptyList(), + DeploymentSpec.fromXml("<deployment>" + + " <instance id='default'/>" + + "</deployment>").requireInstance("default").endpoints()); } @Test public void emptyEndpoints() { - final var spec = DeploymentSpec.fromXml("<deployment><endpoints/></deployment>"); - assertEquals(Collections.emptyList(), spec.endpoints()); + var spec = DeploymentSpec.fromXml("<deployment>" + + " <instance id='default'>" + + " <endpoints/>" + + " </instance>" + + "</deployment>"); + assertEquals(Collections.emptyList(), spec.requireInstance("default").endpoints()); } @Test public void someEndpoints() { - final var spec = DeploymentSpec.fromXml("" + - "<deployment>" + - " <prod>" + - " <region active=\"true\">us-east</region>" + - " </prod>" + - " <endpoints>" + - " <endpoint id=\"foo\" container-id=\"bar\">" + - " <region>us-east</region>" + - " </endpoint>" + - " <endpoint id=\"nalle\" container-id=\"frosk\" />" + - " <endpoint container-id=\"quux\" />" + - " </endpoints>" + - "</deployment>"); + var spec = DeploymentSpec.fromXml("" + + "<deployment>" + + " <instance id='default'>" + + " <prod>" + + " <region active=\"true\">us-east</region>" + + " </prod>" + + " <endpoints>" + + " <endpoint id=\"foo\" container-id=\"bar\">" + + " <region>us-east</region>" + + " </endpoint>" + + " <endpoint id=\"nalle\" container-id=\"frosk\" />" + + " <endpoint container-id=\"quux\" />" + + " </endpoints>" + + " </instance>" + + "</deployment>"); assertEquals( List.of("foo", "nalle", "default"), - spec.endpoints().stream().map(Endpoint::endpointId).collect(Collectors.toList()) + spec.requireInstance("default").endpoints().stream().map(Endpoint::endpointId).collect(Collectors.toList()) ); assertEquals( List.of("bar", "frosk", "quux"), - spec.endpoints().stream().map(Endpoint::containerId).collect(Collectors.toList()) + spec.requireInstance("default").endpoints().stream().map(Endpoint::containerId).collect(Collectors.toList()) ); - assertEquals(Set.of(RegionName.from("us-east")), spec.endpoints().get(0).regions()); + assertEquals(Set.of(RegionName.from("us-east")), spec.requireInstance("default").endpoints().get(0).regions()); } + @Test public void invalidEndpoints() { assertInvalid("<endpoint id='FOO' container-id='qrs'/>"); // Uppercase @@ -520,19 +873,21 @@ public class DeploymentSpecTest { @Test public void endpointDefaultRegions() { var spec = DeploymentSpec.fromXml("" + - "<deployment>" + - " <prod>" + - " <region active=\"true\">us-east</region>" + - " <region active=\"true\">us-west</region>" + - " </prod>" + - " <endpoints>" + - " <endpoint id=\"foo\" container-id=\"bar\">" + - " <region>us-east</region>" + - " </endpoint>" + - " <endpoint id=\"nalle\" container-id=\"frosk\" />" + - " <endpoint container-id=\"quux\" />" + - " </endpoints>" + - "</deployment>"); + "<deployment>" + + " <instance id='default'>" + + " <prod>" + + " <region active=\"true\">us-east</region>" + + " <region active=\"true\">us-west</region>" + + " </prod>" + + " <endpoints>" + + " <endpoint id=\"foo\" container-id=\"bar\">" + + " <region>us-east</region>" + + " </endpoint>" + + " <endpoint id=\"nalle\" container-id=\"frosk\" />" + + " <endpoint container-id=\"quux\" />" + + " </endpoints>" + + " </instance>" + + "</deployment>"); assertEquals(Set.of("us-east"), endpointRegions("foo", spec)); assertEquals(Set.of("us-east", "us-west"), endpointRegions("nalle", spec)); @@ -547,7 +902,7 @@ public class DeploymentSpecTest { } private static Set<String> endpointRegions(String endpointId, DeploymentSpec spec) { - return spec.endpoints().stream() + return spec.requireInstance("default").endpoints().stream() .filter(endpoint -> endpoint.endpointId().equals(endpointId)) .flatMap(endpoint -> endpoint.regions().stream()) .map(RegionName::value) @@ -556,15 +911,17 @@ public class DeploymentSpecTest { private static List<String> endpointIds(String endpointTag) { var xml = "<deployment>" + - " <prod>" + - " <region active=\"true\">us-east</region>" + - " </prod>" + - " <endpoints>" + + " <instance id='default'>" + + " <prod>" + + " <region active=\"true\">us-east</region>" + + " </prod>" + + " <endpoints>" + endpointTag + - " </endpoints>" + + " </endpoints>" + + " </instance>" + "</deployment>"; - return DeploymentSpec.fromXml(xml).endpoints().stream() + return DeploymentSpec.fromXml(xml).requireInstance("default").endpoints().stream() .map(Endpoint::endpointId) .collect(Collectors.toList()); } diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java new file mode 100644 index 00000000000..ad5c6375aa6 --- /dev/null +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java @@ -0,0 +1,526 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.application.api; + +import com.google.common.collect.ImmutableSet; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import org.junit.Test; + +import java.io.StringReader; +import java.time.Instant; +import java.time.ZoneId; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.yahoo.config.application.api.Notifications.Role.author; +import static com.yahoo.config.application.api.Notifications.When.failing; +import static com.yahoo.config.application.api.Notifications.When.failingCommit; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author bratseth + */ +public class DeploymentSpecWithoutInstanceTest { + + @Test + public void testSpec() { + String specXml = "<deployment version='1.0'>" + + " <test/>" + + "</deployment>"; + + StringReader r = new StringReader(specXml); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(specXml, spec.xmlForm()); + assertEquals(1, spec.steps().size()); + assertFalse(spec.majorVersion().isPresent()); + assertTrue(spec.steps().get(0).deploysTo(Environment.test)); + assertTrue(spec.requireInstance("default").includes(Environment.test, Optional.empty())); + assertFalse(spec.requireInstance("default").includes(Environment.test, Optional.of(RegionName.from("region1")))); + assertFalse(spec.requireInstance("default").includes(Environment.staging, Optional.empty())); + assertFalse(spec.requireInstance("default").includes(Environment.prod, Optional.empty())); + assertFalse(spec.requireInstance("default").globalServiceId().isPresent()); + } + + @Test + public void testSpecPinningMajorVersion() { + String specXml = "<deployment version='1.0' major-version='6'>" + + " <test/>" + + "</deployment>"; + + StringReader r = new StringReader(specXml); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(specXml, spec.xmlForm()); + assertEquals(1, spec.steps().size()); + assertTrue(spec.majorVersion().isPresent()); + assertEquals(6, (int)spec.majorVersion().get()); + } + + @Test + public void stagingSpec() { + StringReader r = new StringReader( + "<deployment version='1.0'>" + + " <staging/>" + + "</deployment>" + ); + + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(2, spec.steps().size()); + assertTrue(spec.requireInstance("default").steps().get(0).deploysTo(Environment.test)); + assertTrue(spec.requireInstance("default").steps().get(1).deploysTo(Environment.staging)); + assertTrue(spec.requireInstance("default").includes(Environment.test, Optional.empty())); + assertFalse(spec.requireInstance("default").includes(Environment.test, Optional.of(RegionName.from("region1")))); + assertTrue(spec.requireInstance("default").includes(Environment.staging, Optional.empty())); + assertFalse(spec.requireInstance("default").includes(Environment.prod, Optional.empty())); + assertFalse(spec.requireInstance("default").globalServiceId().isPresent()); + } + + @Test + public void minimalProductionSpec() { + StringReader r = new StringReader( + "<deployment version='1.0'>" + + " <prod>" + + " <region active='false'>us-east1</region>" + + " <region active='true'>us-west1</region>" + + " </prod>" + + "</deployment>" + ); + + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(4, spec.requireInstance("default").steps().size()); + + assertTrue(spec.requireInstance("default").steps().get(0).deploysTo(Environment.test)); + + assertTrue(spec.requireInstance("default").steps().get(1).deploysTo(Environment.staging)); + + assertTrue(spec.requireInstance("default").steps().get(2).deploysTo(Environment.prod, Optional.of(RegionName.from("us-east1")))); + assertFalse(((DeploymentSpec.DeclaredZone)spec.requireInstance("default").steps().get(2)).active()); + + assertTrue(spec.requireInstance("default").steps().get(3).deploysTo(Environment.prod, Optional.of(RegionName.from("us-west1")))); + assertTrue(((DeploymentSpec.DeclaredZone)spec.requireInstance("default").steps().get(3)).active()); + + assertTrue(spec.requireInstance("default").includes(Environment.test, Optional.empty())); + assertFalse(spec.requireInstance("default").includes(Environment.test, Optional.of(RegionName.from("region1")))); + assertTrue(spec.requireInstance("default").includes(Environment.staging, Optional.empty())); + assertTrue(spec.requireInstance("default").includes(Environment.prod, Optional.of(RegionName.from("us-east1")))); + assertTrue(spec.requireInstance("default").includes(Environment.prod, Optional.of(RegionName.from("us-west1")))); + assertFalse(spec.requireInstance("default").includes(Environment.prod, Optional.of(RegionName.from("no-such-region")))); + assertFalse(spec.requireInstance("default").globalServiceId().isPresent()); + + assertEquals(DeploymentSpec.UpgradePolicy.defaultPolicy, spec.requireInstance("default").upgradePolicy()); + } + + @Test + public void maximalProductionSpec() { + StringReader r = new StringReader( + "<deployment version='1.0'>" + + " <test/>" + + " <staging/>" + + " <prod>" + + " <region active='false'>us-east1</region>" + + " <delay hours='3' minutes='30'/>" + + " <region active='true'>us-west1</region>" + + " </prod>" + + "</deployment>" + ); + + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(5, spec.requireInstance("default").steps().size()); + assertEquals(4, spec.requireInstance("default").zones().size()); + + assertTrue(spec.requireInstance("default").steps().get(0).deploysTo(Environment.test)); + + assertTrue(spec.requireInstance("default").steps().get(1).deploysTo(Environment.staging)); + + assertTrue(spec.requireInstance("default").steps().get(2).deploysTo(Environment.prod, Optional.of(RegionName.from("us-east1")))); + assertFalse(((DeploymentSpec.DeclaredZone)spec.requireInstance("default").steps().get(2)).active()); + + assertTrue(spec.requireInstance("default").steps().get(3) instanceof DeploymentSpec.Delay); + assertEquals(3 * 60 * 60 + 30 * 60, spec.requireInstance("default").steps().get(3).delay().getSeconds()); + + assertTrue(spec.requireInstance("default").steps().get(4).deploysTo(Environment.prod, Optional.of(RegionName.from("us-west1")))); + assertTrue(((DeploymentSpec.DeclaredZone)spec.requireInstance("default").steps().get(4)).active()); + + assertTrue(spec.requireInstance("default").includes(Environment.test, Optional.empty())); + assertFalse(spec.requireInstance("default").includes(Environment.test, Optional.of(RegionName.from("region1")))); + assertTrue(spec.requireInstance("default").includes(Environment.staging, Optional.empty())); + assertTrue(spec.requireInstance("default").includes(Environment.prod, Optional.of(RegionName.from("us-east1")))); + assertTrue(spec.requireInstance("default").includes(Environment.prod, Optional.of(RegionName.from("us-west1")))); + assertFalse(spec.requireInstance("default").includes(Environment.prod, Optional.of(RegionName.from("no-such-region")))); + assertFalse(spec.requireInstance("default").globalServiceId().isPresent()); + } + + @Test + public void productionSpecWithGlobalServiceId() { + StringReader r = new StringReader( + "<deployment version='1.0'>" + + " <prod global-service-id='query'>" + + " <region active='true'>us-east-1</region>" + + " <region active='true'>us-west-1</region>" + + " </prod>" + + "</deployment>" + ); + + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(spec.requireInstance("default").globalServiceId(), Optional.of("query")); + } + + @Test(expected=IllegalArgumentException.class) + public void globalServiceIdInTest() { + StringReader r = new StringReader( + "<deployment version='1.0'>" + + " <test global-service-id='query' />" + + "</deployment>" + ); + DeploymentSpec.fromXml(r); + } + + @Test(expected=IllegalArgumentException.class) + public void globalServiceIdInStaging() { + StringReader r = new StringReader( + "<deployment version='1.0'>" + + " <staging global-service-id='query' />" + + "</deployment>" + ); + DeploymentSpec.fromXml(r); + } + + @Test + public void productionSpecWithGlobalServiceIdBeforeStaging() { + StringReader r = new StringReader( + "<deployment>" + + " <test/>" + + " <prod global-service-id='qrs'>" + + " <region active='true'>us-west-1</region>" + + " <region active='true'>us-central-1</region>" + + " <region active='true'>us-east-3</region>" + + " </prod>" + + " <staging/>" + + "</deployment>" + ); + + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals("qrs", spec.requireInstance("default").globalServiceId().get()); + } + + @Test + public void productionSpecWithUpgradePolicy() { + StringReader r = new StringReader( + "<deployment>" + + " <upgrade policy='canary'/>" + + " <prod>" + + " <region active='true'>us-west-1</region>" + + " <region active='true'>us-central-1</region>" + + " <region active='true'>us-east-3</region>" + + " </prod>" + + "</deployment>" + ); + + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals("canary", spec.requireInstance("default").upgradePolicy().toString()); + } + + @Test + public void maxDelayExceeded() { + try { + StringReader r = new StringReader( + "<deployment>" + + " <upgrade policy='canary'/>" + + " <prod>" + + " <region active='true'>us-west-1</region>" + + " <delay hours='23'/>" + + " <region active='true'>us-central-1</region>" + + " <delay minutes='59' seconds='61'/>" + + " <region active='true'>us-east-3</region>" + + " </prod>" + + "</deployment>" + ); + DeploymentSpec.fromXml(r); + fail("Expected exception due to exceeding the max total delay"); + } + catch (IllegalArgumentException e) { + // success + assertEquals("The total delay specified is PT24H1S but max 24 hours is allowed", e.getMessage()); + } + } + + @Test + public void testEmpty() { + assertFalse(DeploymentSpec.empty.requireInstance("default").globalServiceId().isPresent()); + assertEquals(DeploymentSpec.UpgradePolicy.defaultPolicy, DeploymentSpec.empty.upgradePolicy()); + assertTrue(DeploymentSpec.empty.steps().isEmpty()); + assertEquals("<deployment version='1.0'/>", DeploymentSpec.empty.xmlForm()); + } + + @Test + public void productionSpecWithParallelDeployments() { + StringReader r = new StringReader( + "<deployment>\n" + + " <prod> \n" + + " <region active='true'>us-west-1</region>\n" + + " <parallel>\n" + + " <region active='true'>us-central-1</region>\n" + + " <region active='true'>us-east-3</region>\n" + + " </parallel>\n" + + " </prod>\n" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + DeploymentSpec.ParallelZones parallelZones = ((DeploymentSpec.ParallelZones) spec.requireInstance("default").steps().get(3)); + assertEquals(2, parallelZones.zones().size()); + assertEquals(RegionName.from("us-central-1"), parallelZones.zones().get(0).region().get()); + assertEquals(RegionName.from("us-east-3"), parallelZones.zones().get(1).region().get()); + } + + @Test + public void productionSpecWithDuplicateRegions() { + StringReader r = new StringReader( + "<deployment>\n" + + " <prod>\n" + + " <region active='true'>us-west-1</region>\n" + + " <parallel>\n" + + " <region active='true'>us-west-1</region>\n" + + " <region active='true'>us-central-1</region>\n" + + " <region active='true'>us-east-3</region>\n" + + " </parallel>\n" + + " </prod>\n" + + "</deployment>" + ); + try { + DeploymentSpec.fromXml(r); + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertEquals("prod.us-west-1 is listed twice in deployment.xml", e.getMessage()); + } + } + + @Test(expected = IllegalArgumentException.class) + public void deploymentSpecWithIllegallyOrderedDeploymentSpec1() { + StringReader r = new StringReader( + "<deployment>\n" + + " <block-change days='sat' hours='10' time-zone='CET'/>\n" + + " <prod>\n" + + " <region active='true'>us-west-1</region>\n" + + " </prod>\n" + + " <block-change days='mon,tue' hours='15-16'/>\n" + + "</deployment>" + ); + DeploymentSpec.fromXml(r); + } + + @Test(expected = IllegalArgumentException.class) + public void deploymentSpecWithIllegallyOrderedDeploymentSpec2() { + StringReader r = new StringReader( + "<deployment>\n" + + " <block-change days='sat' hours='10' time-zone='CET'/>\n" + + " <test/>\n" + + " <prod>\n" + + " <region active='true'>us-west-1</region>\n" + + " </prod>\n" + + "</deployment>" + ); + DeploymentSpec.fromXml(r); + } + + @Test + public void deploymentSpecWithChangeBlocker() { + StringReader r = new StringReader( + "<deployment>\n" + + " <block-change revision='false' days='mon,tue' hours='15-16'/>\n" + + " <block-change days='sat' hours='10' time-zone='CET'/>\n" + + " <prod>\n" + + " <region active='true'>us-west-1</region>\n" + + " </prod>\n" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(2, spec.requireInstance("default").changeBlocker().size()); + assertTrue(spec.requireInstance("default").changeBlocker().get(0).blocksVersions()); + assertFalse(spec.requireInstance("default").changeBlocker().get(0).blocksRevisions()); + assertEquals(ZoneId.of("UTC"), spec.requireInstance("default").changeBlocker().get(0).window().zone()); + + assertTrue(spec.requireInstance("default").changeBlocker().get(1).blocksVersions()); + assertTrue(spec.requireInstance("default").changeBlocker().get(1).blocksRevisions()); + assertEquals(ZoneId.of("CET"), spec.requireInstance("default").changeBlocker().get(1).window().zone()); + + assertTrue(spec.requireInstance("default").canUpgradeAt(Instant.parse("2017-09-18T14:15:30.00Z"))); + assertFalse(spec.requireInstance("default").canUpgradeAt(Instant.parse("2017-09-18T15:15:30.00Z"))); + assertFalse(spec.requireInstance("default").canUpgradeAt(Instant.parse("2017-09-18T16:15:30.00Z"))); + assertTrue(spec.requireInstance("default").canUpgradeAt(Instant.parse("2017-09-18T17:15:30.00Z"))); + + assertTrue(spec.requireInstance("default").canUpgradeAt(Instant.parse("2017-09-23T09:15:30.00Z"))); + assertFalse(spec.requireInstance("default").canUpgradeAt(Instant.parse("2017-09-23T08:15:30.00Z"))); // 10 in CET + assertTrue(spec.requireInstance("default").canUpgradeAt(Instant.parse("2017-09-23T10:15:30.00Z"))); + } + + @Test + public void athenz_config_is_read_from_deployment() { + StringReader r = new StringReader( + "<deployment athenz-domain='domain' athenz-service='service'>\n" + + " <prod>\n" + + " <region active='true'>us-west-1</region>\n" + + " </prod>\n" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(spec.requireInstance("default").athenzDomain().get().value(), "domain"); + assertEquals(spec.requireInstance("default").athenzService(Environment.prod, RegionName.from("us-west-1")).get().value(), "service"); + } + + @Test + public void athenz_service_is_overridden_from_environment() { + StringReader r = new StringReader( + "<deployment athenz-domain='domain' athenz-service='service'>\n" + + " <test/>\n" + + " <prod athenz-service='prod-service'>\n" + + " <region active='true'>us-west-1</region>\n" + + " </prod>\n" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(spec.requireInstance("default").athenzDomain().get().value(), "domain"); + assertEquals(spec.requireInstance("default").athenzService(Environment.prod, RegionName.from("us-west-1")).get().value(), "prod-service"); + } + + @Test(expected = IllegalArgumentException.class) + public void it_fails_when_athenz_service_is_not_defined() { + StringReader r = new StringReader( + "<deployment athenz-domain='domain'>\n" + + " <prod>\n" + + " <region active='true'>us-west-1</region>\n" + + " </prod>\n" + + "</deployment>" + ); + DeploymentSpec.fromXml(r); + } + + @Test(expected = IllegalArgumentException.class) + public void it_fails_when_athenz_service_is_configured_but_not_athenz_domain() { + StringReader r = new StringReader( + "<deployment>\n" + + " <prod athenz-service='service'>\n" + + " <region active='true'>us-west-1</region>\n" + + " </prod>\n" + + "</deployment>" + ); + DeploymentSpec.fromXml(r); + } + + @Test + public void noNotifications() { + assertEquals(Notifications.none(), + DeploymentSpec.fromXml("<deployment />").requireInstance("default").notifications()); + } + + @Test + public void emptyNotifications() { + DeploymentSpec spec = DeploymentSpec.fromXml("<deployment>\n" + + " <notifications />" + + "</deployment>"); + assertEquals(Notifications.none(), spec.requireInstance("default").notifications()); + } + + @Test + public void someNotifications() { + DeploymentSpec spec = DeploymentSpec.fromXml("<deployment>\n" + + " <notifications when=\"failing\">\n" + + " <email role=\"author\"/>\n" + + " <email address=\"john@dev\" when=\"failing-commit\"/>\n" + + " <email address=\"jane@dev\"/>\n" + + " </notifications>\n" + + "</deployment>"); + assertEquals(ImmutableSet.of(author), spec.requireInstance("default").notifications().emailRolesFor(failing)); + assertEquals(ImmutableSet.of(author), spec.requireInstance("default").notifications().emailRolesFor(failingCommit)); + assertEquals(ImmutableSet.of("john@dev", "jane@dev"), spec.requireInstance("default").notifications().emailAddressesFor(failingCommit)); + assertEquals(ImmutableSet.of("jane@dev"), spec.requireInstance("default").notifications().emailAddressesFor(failing)); + } + + @Test + public void customTesterFlavor() { + DeploymentSpec spec = DeploymentSpec.fromXml("<deployment>\n" + + " <test tester-flavor=\"d-1-4-20\" />\n" + + " <prod tester-flavor=\"d-2-8-50\">\n" + + " <region active=\"false\">us-north-7</region>\n" + + " </prod>\n" + + "</deployment>"); + assertEquals(Optional.of("d-1-4-20"), spec.requireInstance("default").steps().get(0).zones().get(0).testerFlavor()); + assertEquals(Optional.empty(), spec.requireInstance("default").steps().get(1).zones().get(0).testerFlavor()); + assertEquals(Optional.of("d-2-8-50"), spec.requireInstance("default").steps().get(2).zones().get(0).testerFlavor()); + } + + @Test + public void noEndpoints() { + assertEquals(Collections.emptyList(), DeploymentSpec.fromXml("<deployment />").requireInstance("default").endpoints()); + } + + @Test + public void emptyEndpoints() { + var spec = DeploymentSpec.fromXml("<deployment><endpoints/></deployment>"); + assertEquals(Collections.emptyList(), spec.requireInstance("default").endpoints()); + } + + @Test + public void someEndpoints() { + var spec = DeploymentSpec.fromXml("" + + "<deployment>" + + " <prod>" + + " <region active=\"true\">us-east</region>" + + " </prod>" + + " <endpoints>" + + " <endpoint id=\"foo\" container-id=\"bar\">" + + " <region>us-east</region>" + + " </endpoint>" + + " <endpoint id=\"nalle\" container-id=\"frosk\" />" + + " <endpoint container-id=\"quux\" />" + + " </endpoints>" + + "</deployment>"); + + assertEquals( + List.of("foo", "nalle", "default"), + spec.requireInstance("default").endpoints().stream().map(Endpoint::endpointId).collect(Collectors.toList()) + ); + + assertEquals( + List.of("bar", "frosk", "quux"), + spec.requireInstance("default").endpoints().stream().map(Endpoint::containerId).collect(Collectors.toList()) + ); + + assertEquals(Set.of(RegionName.from("us-east")), spec.requireInstance("default").endpoints().get(0).regions()); + } + + @Test + public void endpointDefaultRegions() { + var spec = DeploymentSpec.fromXml("" + + "<deployment>" + + " <prod>" + + " <region active=\"true\">us-east</region>" + + " <region active=\"true\">us-west</region>" + + " </prod>" + + " <endpoints>" + + " <endpoint id=\"foo\" container-id=\"bar\">" + + " <region>us-east</region>" + + " </endpoint>" + + " <endpoint id=\"nalle\" container-id=\"frosk\" />" + + " <endpoint container-id=\"quux\" />" + + " </endpoints>" + + "</deployment>"); + + assertEquals(Set.of("us-east"), endpointRegions("foo", spec)); + assertEquals(Set.of("us-east", "us-west"), endpointRegions("nalle", spec)); + assertEquals(Set.of("us-east", "us-west"), endpointRegions("default", spec)); + } + + private static Set<String> endpointRegions(String endpointId, DeploymentSpec spec) { + return spec.requireInstance("default").endpoints().stream() + .filter(endpoint -> endpoint.endpointId().equals(endpointId)) + .flatMap(endpoint -> endpoint.regions().stream()) + .map(RegionName::value) + .collect(Collectors.toSet()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidator.java deleted file mode 100644 index 7757a8d4748..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidator.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation; - -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.container.ContainerCluster; -import com.yahoo.vespa.model.container.ContainerModel; - -import java.io.Reader; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Validates that deployment file (deployment.xml) has valid values (for now - * only global-service-id is validated) - * - * @author hmusum - */ -public class DeploymentFileValidator extends Validator { - - @Override - public void validate(VespaModel model, DeployState deployState) { - Optional<Reader> deployment = deployState.getApplicationPackage().getDeployment(); - - if (deployment.isPresent()) { - Reader deploymentReader = deployment.get(); - DeploymentSpec deploymentSpec = DeploymentSpec.fromXml(deploymentReader); - final Optional<String> globalServiceId = deploymentSpec.globalServiceId(); - if (globalServiceId.isPresent()) { - Set<ContainerCluster> containerClusters = model.getRoot().configModelRepo().getModels(ContainerModel.class).stream(). - map(ContainerModel::getCluster).filter(cc -> cc.getName().equals(globalServiceId.get())).collect(Collectors.toSet()); - if (containerClusters.size() != 1) { - throw new IllegalArgumentException("global-service-id '" + globalServiceId.get() + "' specified in deployment.xml does not match any container cluster id"); - } - } - } - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java new file mode 100644 index 00000000000..ac38336a405 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java @@ -0,0 +1,40 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.application.api.DeploymentInstanceSpec; +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.ContainerModel; + +import java.io.Reader; +import java.util.List; +import java.util.Optional; + +/** + * Validates that deployment spec (deployment.xml) has valid values (for now + * only global-service-id is validated) + * + * @author hmusum + * @author bratseth + */ +public class DeploymentSpecValidator extends Validator { + + @Override + public void validate(VespaModel model, DeployState deployState) { + Optional<Reader> deployment = deployState.getApplicationPackage().getDeployment(); + if ( deployment.isEmpty()) return; + + Reader deploymentReader = deployment.get(); + DeploymentSpec deploymentSpec = DeploymentSpec.fromXml(deploymentReader); + List<ContainerModel> containers = model.getRoot().configModelRepo().getModels(ContainerModel.class); + for (DeploymentInstanceSpec instance : deploymentSpec.instances()) { + instance.globalServiceId().ifPresent(globalServiceId -> { + if ( containers.stream().noneMatch(container -> container.getCluster().getName().equals(globalServiceId))) + throw new IllegalArgumentException("The global-service-id in " + instance + ", '" + globalServiceId + + "' specified in deployment.xml does not match any container cluster id"); + }); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 042c7cc867c..7d0d068f9d6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -53,7 +53,7 @@ public class Validation { new StreamingValidator().validate(model, deployState); new RankSetupValidator(validationParameters.ignoreValidationErrors()).validate(model, deployState); new NoPrefixForIndexes().validate(model, deployState); - new DeploymentFileValidator().validate(model, deployState); + new DeploymentSpecValidator().validate(model, deployState); new RankingConstantsValidator().validate(model, deployState); new SecretStoreValidator().validate(model, deployState); new TlsSecretsValidator().validate(model, deployState); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index f4c7f49a9a0..1c0645aef2b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -7,6 +7,7 @@ import com.yahoo.component.Version; import com.yahoo.config.application.Xml; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.api.ConfigServerSpec; @@ -197,7 +198,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addClientProviders(deployState, spec, cluster); addServerProviders(deployState, spec, cluster); - addAthensCopperArgos(cluster, context); // Must be added after nodes. } @@ -228,14 +228,17 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } private void addRotationProperties(ApplicationContainerCluster cluster, Zone zone, Set<Rotation> rotations, Set<ContainerEndpoint> endpoints, DeploymentSpec spec) { + Optional<String> globalServiceId = spec.instance(app.getApplicationId().instance()).flatMap(instance -> instance.globalServiceId()); cluster.getContainers().forEach(container -> { - setRotations(container, rotations, endpoints, spec.globalServiceId(), cluster.getName()); + setRotations(container, rotations, endpoints, globalServiceId, cluster.getName()); container.setProp("activeRotation", Boolean.toString(zoneHasActiveRotation(zone, spec))); }); } private boolean zoneHasActiveRotation(Zone zone, DeploymentSpec spec) { - return spec.zones().stream() + Optional<DeploymentInstanceSpec> instance = spec.instance(app.getApplicationId().instance()); + if (instance.isEmpty()) return false; + return instance.get().zones().stream() .anyMatch(declaredZone -> declaredZone.deploysTo(zone.environment(), Optional.of(zone.region())) && declaredZone.active()); } @@ -893,8 +896,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Zone zone, DeploymentSpec spec) { spec.athenzDomain().ifPresent(domain -> { - AthenzService service = spec.athenzService(zone.environment(), zone.region()) - .orElseThrow(() -> new RuntimeException("Missing Athenz service configuration")); + AthenzService service = spec.athenzService(app.getApplicationId().instance(), zone.environment(), zone.region()) + .orElseThrow(() -> new RuntimeException("Missing Athenz service configuration in instance '" + app.getApplicationId().instance() + "'")); String zoneDnsSuffix = zone.environment().value() + "-" + zone.region().value() + "." + athenzDnsSuffix; IdentityProvider identityProvider = new IdentityProvider(domain, service, getLoadBalancerName(loadBalancerName, configServerSpecs), ztsUrl, zoneDnsSuffix, zone); cluster.addComponent(identityProvider); diff --git a/config-model/src/main/resources/schema/deployment.rnc b/config-model/src/main/resources/schema/deployment.rnc index 7b15a1c062d..1e1d9ad3aa9 100644 --- a/config-model/src/main/resources/schema/deployment.rnc +++ b/config-model/src/main/resources/schema/deployment.rnc @@ -7,13 +7,37 @@ start = element deployment { attribute major-version { text }? & attribute athenz-domain { xsd:string }? & attribute athenz-service { xsd:string }? & - Upgrade? & - BlockChange* & - Notifications? & - Endpoints? & - Test? & - Staging? & - Prod* + Step +} + +Step = + StepExceptInstance & + Instance* + +StepExceptInstance = + Delay* & + ParallelInstances* & + Upgrade? & + BlockChange* & + Notifications? & + Endpoints? & + Test? & + Staging? & + Prod* + +Instance = element instance { + attribute id { xsd:string } & + attribute athenz-domain { xsd:string }? & + attribute athenz-service { xsd:string }? & + StepExceptInstance +} + +ParallelRegions = element parallel { + Region* +} + +ParallelInstances = element parallel { + Instance* } Upgrade = element upgrade { @@ -57,7 +81,7 @@ Prod = element prod { attribute tester-flavor { xsd:string }? & Region* & Delay* & - Parallel* + ParallelRegions* } Region = element region { @@ -71,10 +95,6 @@ Delay = element delay { attribute seconds { xsd:long }? } -Parallel = element parallel { - Region* -} - EndpointRegion = element region { text } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidatorTest.java index 5fc3f815b09..c6d56455d44 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidatorTest.java @@ -18,7 +18,7 @@ import static org.junit.Assert.fail; /** * @author hmusum */ -public class DeploymentFileValidatorTest { +public class DeploymentSpecValidatorTest { @Test public void testDeploymentWithNonExistentGlobalId() throws IOException, SAXException { @@ -58,7 +58,7 @@ public class DeploymentFileValidatorTest { try { final DeployState deployState = builder.build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new DeploymentFileValidator().validate(model, deployState); + new DeploymentSpecValidator().validate(model, deployState); fail("Did not get expected exception"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), containsString("specified in deployment.xml does not match any container cluster id")); diff --git a/config-model/src/test/schema-test-files/deployment-with-instances.xml b/config-model/src/test/schema-test-files/deployment-with-instances.xml new file mode 100644 index 00000000000..e23404df093 --- /dev/null +++ b/config-model/src/test/schema-test-files/deployment-with-instances.xml @@ -0,0 +1,54 @@ +<!-- Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<deployment version='1.0' major-version='6' athenz-domain='vespa' athenz-service='service'> + <upgrade policy='canary'/> + + <test/> + <staging/> + + <block-change revision='true' version='false' days="mon,tue" hours="14,15"/> + + <instance id="one,two"> + <block-change days="mon,tue" hours="14,15" time-zone="CET"/> + <prod global-service-id='qrs' athenz-service='other-service'> + <region active='true'>us-west-1</region> + <delay hours='3'/> + <region active='true'>us-central-1</region> + <delay hours='3' minutes='7' seconds='13'/> + <region active='true'>us-east-3</region> + <parallel> + <region active='true'>us-north-1</region> + <region active='true'>us-south-1</region> + </parallel> + <parallel> + <region active='true'>us-north-2</region> + <region active='true'>us-south-2</region> + </parallel> + </prod> + <endpoints> + <endpoint id="foo" container-id="bar"> + <region>us-east</region> + </endpoint> + <endpoint container-id="bar" /> + </endpoints> + </instance> + + <delay hours='2'/> + + <parallel> + <instance id="three"> + <test/> + <staging/> + </instance> + <instance id="four" athenz-service='four-service' athenz-domain='my-domain'> + <upgrade policy='conservative'/> + <block-change days="mon,tue,wed" hours="14,15"/> + <prod> + <region active='true'>us-central-1</region> + </prod> + <endpoints> + <endpoint container-id="barz" /> + </endpoints> + </instance> + </parallel> + +</deployment> diff --git a/config-model/src/test/sh/test-schema.sh b/config-model/src/test/sh/test-schema.sh index e037cdf3c28..4b34d975c0d 100755 --- a/config-model/src/test/sh/test-schema.sh +++ b/config-model/src/test/sh/test-schema.sh @@ -25,6 +25,10 @@ cmd="java -jar $jar target/generated-sources/trang/resources/schema/deployment.r echo $cmd $cmd +cmd="java -jar $jar target/generated-sources/trang/resources/schema/deployment.rng src/test/schema-test-files/deployment-with-instances.xml" +echo $cmd +$cmd + cmd="java -jar $jar target/generated-sources/trang/resources/schema/validation-overrides.rng src/test/schema-test-files/validation-overrides.xml" echo $cmd $cmd diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Environment.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Environment.java index b9573b21199..012f246a227 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Environment.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Environment.java @@ -5,7 +5,6 @@ package com.yahoo.config.provision; * Environments in hosted Vespa. * * @author bratseth - * @since 5.11 */ public enum Environment { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 54c96c0461d..96df067843d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -9,6 +9,7 @@ import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ConfigDefinitionRepo; @@ -121,7 +122,8 @@ public class SessionPreparer { preparation.writeTlsZK(); var globalServiceId = context.getApplicationPackage().getDeployment() .map(DeploymentSpec::fromXml) - .flatMap(DeploymentSpec::globalServiceId); + .map(spec -> spec.requireInstance(context.getApplicationPackage().getApplicationId().instance())) + .flatMap(DeploymentInstanceSpec::globalServiceId); preparation.writeContainerEndpointsZK(globalServiceId); preparation.distribute(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index 8b8be1a27d7..f2c6aac2bda 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -81,7 +81,7 @@ public class ZKApplicationPackageTest { assertThat(readInfo.getHosts().iterator().next().flavor(), is(TEST_FLAVOR)); assertEquals("6.0.1", readInfo.getHosts().iterator().next().version().get().toString()); assertTrue(zkApp.getDeployment().isPresent()); - assertThat(DeploymentSpec.fromXml(zkApp.getDeployment().get()).globalServiceId().get(), is("mydisc")); + assertEquals("mydisc", DeploymentSpec.fromXml(zkApp.getDeployment().get()).requireInstance("default").globalServiceId().get()); } private void feed(ConfigCurator zk, File dirToFeed) throws IOException { diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java index a37255436ca..eceffb379aa 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java @@ -31,16 +31,12 @@ import com.yahoo.vespa.defaults.Defaults; */ public final class VipStatusHandler extends ThreadedHttpRequestHandler { - private static final Logger log = Logger.getLogger(VipStatusHandler.class.getName()); - private static final String NUM_REQUESTS_METRIC = "jdisc.http.requests.status"; private final boolean accessDisk; private final File statusFile; private final VipStatus vipStatus; - private volatile boolean previouslyInRotation = true; - // belongs in the response, but that's not a static class static final String OK_MESSAGE = "<title>OK</title>\n"; static final byte[] VIP_OK = Utf8.toBytes(OK_MESSAGE); @@ -162,6 +158,7 @@ public final class VipStatusHandler extends ThreadedHttpRequestHandler { * out of capacity. This is the default behavior. */ @Inject + @SuppressWarnings("unused") // injected public VipStatusHandler(VipStatusConfig vipConfig, Metric metric, VipStatus vipStatus) { // One thread should be enough for status handling - otherwise something else is completely wrong, // in which case this will eventually start returning a 503 (due to work rejection) as the bounded diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index facf894272d..edaa5b4b824 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -5884,6 +5884,62 @@ ], "fields": [] }, + "com.yahoo.search.query.profile.SubstituteString$Component": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "abstract" + ], + "methods": [ + "public void <init>()", + "protected abstract java.lang.String getValue(java.util.Map, com.yahoo.processing.request.Properties)" + ], + "fields": [] + }, + "com.yahoo.search.query.profile.SubstituteString$PropertyComponent": { + "superClass": "com.yahoo.search.query.profile.SubstituteString$Component", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(java.lang.String)", + "public java.lang.String getValue(java.util.Map, com.yahoo.processing.request.Properties)", + "public java.lang.String toString()" + ], + "fields": [] + }, + "com.yahoo.search.query.profile.SubstituteString$RelativePropertyComponent": { + "superClass": "com.yahoo.search.query.profile.SubstituteString$Component", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(java.lang.String)", + "public java.lang.String getValue(java.util.Map, com.yahoo.processing.request.Properties)", + "public java.lang.String fieldName()", + "public java.lang.String toString()" + ], + "fields": [] + }, + "com.yahoo.search.query.profile.SubstituteString$StringComponent": { + "superClass": "com.yahoo.search.query.profile.SubstituteString$Component", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(java.lang.String)", + "public java.lang.String getValue(java.util.Map, com.yahoo.processing.request.Properties)", + "public java.lang.String toString()" + ], + "fields": [] + }, "com.yahoo.search.query.profile.SubstituteString": { "superClass": "java.lang.Object", "interfaces": [], @@ -5892,7 +5948,11 @@ ], "methods": [ "public static com.yahoo.search.query.profile.SubstituteString create(java.lang.String)", + "public void <init>(java.util.List, java.lang.String)", + "public boolean hasRelative()", "public java.lang.String substitute(java.util.Map, com.yahoo.processing.request.Properties)", + "public java.util.List components()", + "public java.lang.String stringValue()", "public int hashCode()", "public boolean equals(java.lang.Object)", "public java.lang.String toString()" @@ -6004,8 +6064,9 @@ ], "methods": [ "public void <init>()", + "public java.lang.Object valueFor(com.yahoo.search.query.profile.DimensionBinding)", "public void add(java.lang.Object, com.yahoo.search.query.profile.DimensionBinding)", - "public com.yahoo.search.query.profile.compiled.DimensionalValue build()" + "public com.yahoo.search.query.profile.compiled.DimensionalValue build(java.util.Map)" ], "fields": [] }, diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 1f621eb926c..0c8e564578b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -7,7 +7,6 @@ import com.yahoo.component.chain.dependencies.After; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.handler.VipStatus; import com.yahoo.jdisc.Metric; -import com.yahoo.net.HostName; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.fastsearch.ClusterParams; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; @@ -27,8 +26,6 @@ import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher; import org.apache.commons.lang.StringUtils; -import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -371,6 +368,10 @@ public class ClusterSearcher extends Searcher { } @Override - public void deconstruct() { } + public void deconstruct() { + if (server != null) { + server.shutDown(); + } + } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java index b0b3a7800e9..9a4913b3840 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -173,4 +173,9 @@ public class FastSearcher extends VespaBackEndSearcher { return getLogger().isLoggable(Level.FINE); } + @Override + public void shutDown() { + super.shutDown(); + dispatcher.shutDown(); + } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index 8f4b49ac71e..bc3ac6cdef1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -392,4 +392,6 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { return getLogger().isLoggable(Level.FINE); } + public void shutDown() { } + } diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java index 22c7f59872c..a016f7d695c 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java @@ -9,7 +9,9 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; @@ -26,9 +28,9 @@ public class ClusterMonitor<T> { private static Logger log = Logger.getLogger(ClusterMonitor.class.getName()); - private NodeManager<T> nodeManager; + private final NodeManager<T> nodeManager; - private MonitorThread monitorThread; + private final MonitorThread monitorThread; private volatile boolean shutdown = false; @@ -119,28 +121,35 @@ public class ClusterMonitor<T> { } public void run() { - log.fine("Starting cluster monitor thread"); + log.info("Starting cluster monitor thread " + getName()); // Pings must happen in a separate thread from this to handle timeouts // By using a cached thread pool we ensured that 1) a single thread will be used // for all pings when there are no problems (important because it ensures that // any thread local connections are reused) 2) a new thread will be started to execute // new pings when a ping is not responding - Executor pingExecutor=Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory("search.ping")); + ExecutorService pingExecutor=Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory("search.ping")); while (!isInterrupted()) { try { Thread.sleep(configuration.getCheckInterval()); log.finest("Activating ping"); ping(pingExecutor); } - catch (Exception e) { + catch (Throwable e) { if (shutdown && e instanceof InterruptedException) { break; + } else if ( ! (e instanceof Exception) ) { + log.log(Level.WARNING,"Error in monitor thread, will quit", e); + break; } else { - log.log(Level.WARNING,"Error in monitor thread",e); + log.log(Level.WARNING,"Exception in monitor thread", e); } } } - log.fine("Stopped cluster monitor thread"); + pingExecutor.shutdown(); + try { + pingExecutor.awaitTermination(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { } + log.info("Stopped cluster monitor thread " + getName()); } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 7369b33e82d..ddd319b7bcb 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -195,6 +195,10 @@ public class Dispatcher extends AbstractComponent { return Optional.empty(); } + public void shutDown() { + searchCluster.shutDown(); + } + private void emitDispatchMetric(Optional<SearchInvoker> invoker) { if (invoker.isEmpty()) { metric.add(FDISPATCH_METRIC, 1, metricContext); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java index b47f2fefa5b..09ad715b471 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java @@ -16,17 +16,15 @@ public class Node { private final int key; private int pathIndex; private final String hostname; - private final int fs4port; - final int group; + private final int group; private final AtomicBoolean statusIsKnown = new AtomicBoolean(false); private final AtomicBoolean working = new AtomicBoolean(true); private final AtomicLong activeDocuments = new AtomicLong(0); - public Node(int key, String hostname, int fs4port, int group) { + public Node(int key, String hostname, int group) { this.key = key; this.hostname = hostname; - this.fs4port = fs4port; this.group = group; } @@ -41,14 +39,15 @@ public class Node { public String hostname() { return hostname; } - public int fs4port() { return fs4port; } - /** Returns the id of this group this node belongs to */ public int group() { return group; } public void setWorking(boolean working) { this.statusIsKnown.lazySet(true); this.working.lazySet(working); + if ( ! working ) { + activeDocuments.set(0); + } } /** Returns whether this node is currently responding to requests, or null if status is not known */ @@ -57,17 +56,17 @@ public class Node { } /** Updates the active documents on this node */ - public void setActiveDocuments(long activeDocuments) { + void setActiveDocuments(long activeDocuments) { this.activeDocuments.set(activeDocuments); } /** Returns the active documents on this node. If unknown, 0 is returned. */ - public long getActiveDocuments() { - return this.activeDocuments.get(); + long getActiveDocuments() { + return activeDocuments.get(); } @Override - public int hashCode() { return Objects.hash(hostname, fs4port); } + public int hashCode() { return Objects.hash(hostname, key, pathIndex, group); } @Override public boolean equals(Object o) { @@ -75,11 +74,15 @@ public class Node { if ( ! (o instanceof Node)) return false; Node other = (Node)o; if ( ! Objects.equals(this.hostname, other.hostname)) return false; - if ( ! Objects.equals(this.fs4port, other.fs4port)) return false; + if ( ! Objects.equals(this.key, other.key)) return false; + if ( ! Objects.equals(this.pathIndex, other.pathIndex)) return false; + if ( ! Objects.equals(this.group, other.group)) return false; + return true; } @Override - public String toString() { return "search node " + hostname + ":" + fs4port + " in group " + group; } + public String toString() { return "search node key = " + key + " hostname = "+ hostname + " path = " + pathIndex + " in group " + group + + " statusIsKnown = " + statusIsKnown.get() + " working = " + working.get() + " activeDocs = " + activeDocuments.get(); } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index a55a970e8ff..3595a24ca92 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -46,6 +46,7 @@ public class SearchCluster implements NodeManager<Node> { private final ClusterMonitor<Node> clusterMonitor; private final VipStatus vipStatus; private PingFactory pingFactory; + private long nextLogTime = 0; /** * A search node on this local machine having the entire corpus, which we therefore @@ -73,7 +74,7 @@ public class SearchCluster implements NodeManager<Node> { } this.groups = groupsBuilder.build(); LinkedHashMap<Integer, Group> groupIntroductionOrder = new LinkedHashMap<>(); - nodes.forEach(node -> groupIntroductionOrder.put(node.group(), groups.get(node.group))); + nodes.forEach(node -> groupIntroductionOrder.put(node.group(), groups.get(node.group()))); this.orderedGroups = ImmutableList.<Group>builder().addAll(groupIntroductionOrder.values()).build(); // Index nodes by host @@ -91,6 +92,10 @@ public class SearchCluster implements NodeManager<Node> { this.clusterMonitor = new ClusterMonitor<>(this); } + public void shutDown() { + clusterMonitor.shutdown(); + } + public void startClusterMonitoring(PingFactory pingFactory) { this.pingFactory = pingFactory; @@ -141,7 +146,7 @@ public class SearchCluster implements NodeManager<Node> { } for (DispatchConfig.Node node : dispatchConfig.node()) { if (filter.test(node)) { - nodesBuilder.add(new Node(node.key(), node.host(), node.fs4port(), node.group())); + nodesBuilder.add(new Node(node.key(), node.host(), node.group())); } } return nodesBuilder.build(); @@ -409,14 +414,21 @@ public class SearchCluster implements NodeManager<Node> { private void trackGroupCoverageChanges(int index, Group group, boolean fullCoverage, long averageDocuments) { boolean changed = group.isFullCoverageStatusChanged(fullCoverage); - if (changed) { + if (changed || (!fullCoverage && System.currentTimeMillis() > nextLogTime)) { + nextLogTime = System.currentTimeMillis() + 30 * 1000; int requiredNodes = groupSize() - dispatchConfig.maxNodesDownPerGroup(); if (fullCoverage) { log.info(() -> String.format("Group %d is now good again (%d/%d active docs, coverage %d/%d)", index, group.getActiveDocuments(), averageDocuments, group.workingNodes(), groupSize())); } else { - log.warning(() -> String.format("Coverage of group %d is only %d/%d (requires %d)", - index, group.workingNodes(), groupSize(), requiredNodes)); + StringBuilder missing = new StringBuilder(); + for (var node : group.nodes()) { + if (node.isWorking() != Boolean.TRUE) { + missing.append('\n').append(node.toString()); + } + } + log.warning(() -> String.format("Coverage of group %d is only %d/%d (requires %d) (%d/%d active docs) Failed nodes are:%s", + index, group.workingNodes(), groupSize(), requiredNodes, group.getActiveDocuments(), averageDocuments, missing.toString())); } } } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java index 0059b761734..1fc1e19e3ee 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java @@ -21,7 +21,7 @@ public class DimensionBinding { private DimensionValues values; /** The binding from those dimensions to values, and possibly other values */ - private Map<String,String> context; + private Map<String, String> context; public static final DimensionBinding nullBinding = new DimensionBinding(Collections.unmodifiableList(Collections.emptyList()), DimensionValues.empty, null); @@ -195,11 +195,11 @@ public class DimensionBinding { @Override public String toString() { if (isInvalid()) return "Invalid DimensionBinding"; - if (dimensions==null) return "DimensionBinding []"; - StringBuilder b=new StringBuilder("DimensionBinding ["); - for (int i=0; i<dimensions.size(); i++) { + if (dimensions == null) return "DimensionBinding []"; + StringBuilder b = new StringBuilder("DimensionBinding ["); + for (int i = 0; i < dimensions.size(); i++) { b.append(dimensions.get(i)).append("=").append(values.get(i)); - if (i<dimensions.size()-1) + if (i < dimensions.size()-1) b.append(", "); } b.append("]"); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java index acca2d403be..f5f6b2d2550 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java @@ -102,7 +102,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable */ public List<QueryProfile> inherited() { if (isFrozen()) return inherited; // Frozen profiles always have an unmodifiable, non-null list - if (inherited==null) return Collections.emptyList(); + if (inherited == null) return Collections.emptyList(); return Collections.unmodifiableList(inherited); } @@ -474,17 +474,17 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable /** Returns this value, or its corresponding substitution string if it contains substitutions */ protected Object convertToSubstitutionString(Object value) { - if (value==null) return value; - if (value.getClass()!=String.class) return value; - SubstituteString substituteString=SubstituteString.create((String)value); - if (substituteString==null) return value; + if (value == null) return value; + if (value.getClass() != String.class) return value; + SubstituteString substituteString = SubstituteString.create((String)value); + if (substituteString == null) return value; return substituteString; } /** Returns the field description of this field, or null if it is not typed */ protected FieldDescription getFieldDescription(CompoundName name, DimensionBinding binding) { - FieldDescriptionQueryProfileVisitor visitor=new FieldDescriptionQueryProfileVisitor(name.asList()); - accept(visitor, binding,null); + FieldDescriptionQueryProfileVisitor visitor = new FieldDescriptionQueryProfileVisitor(name.asList()); + accept(visitor, binding, null); return visitor.result(); } @@ -493,23 +493,23 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable * false if it is declared unoverridable (in instance or type), and null if this profile has no * opinion on the matter because the value is not set in this. */ - Boolean isLocalOverridable(String localName,DimensionBinding binding) { - if (localLookup(localName, binding)==null) return null; // Not set - Boolean isLocalInstanceOverridable=isLocalInstanceOverridable(localName); - if (isLocalInstanceOverridable!=null) + Boolean isLocalOverridable(String localName, DimensionBinding binding) { + if (localLookup(localName, binding) == null) return null; // Not set + Boolean isLocalInstanceOverridable = isLocalInstanceOverridable(localName); + if (isLocalInstanceOverridable != null) return isLocalInstanceOverridable.booleanValue(); - if (type!=null) return type.isOverridable(localName); + if (type != null) return type.isOverridable(localName); return true; } protected Boolean isLocalInstanceOverridable(String localName) { - if (overridable==null) return null; + if (overridable == null) return null; return overridable.get(localName); } - protected Object lookup(CompoundName name,boolean allowQueryProfileResult, DimensionBinding dimensionBinding) { - SingleValueQueryProfileVisitor visitor=new SingleValueQueryProfileVisitor(name.asList(),allowQueryProfileResult); - accept(visitor,dimensionBinding,null); + protected Object lookup(CompoundName name, boolean allowQueryProfileResult, DimensionBinding dimensionBinding) { + SingleValueQueryProfileVisitor visitor = new SingleValueQueryProfileVisitor(name.asList(), allowQueryProfileResult); + accept(visitor, dimensionBinding, null); return visitor.getResult(); } @@ -518,7 +518,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable } void acceptAndEnter(String key, QueryProfileVisitor visitor,DimensionBinding dimensionBinding, QueryProfile owner) { - boolean allowContent=visitor.enter(key); + boolean allowContent = visitor.enter(key); accept(allowContent, visitor, dimensionBinding, owner); if (allowContent) visitor.leave(key); @@ -548,25 +548,25 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable } protected void visitVariants(boolean allowContent,QueryProfileVisitor visitor,DimensionBinding dimensionBinding) { - if (getVariants()!=null) + if (getVariants() != null) getVariants().accept(allowContent, getType(), visitor, dimensionBinding); } protected void visitInherited(boolean allowContent,QueryProfileVisitor visitor,DimensionBinding dimensionBinding, QueryProfile owner) { - if (inherited==null) return; + if (inherited == null) return; for (QueryProfile inheritedProfile : inherited) { - inheritedProfile.accept(allowContent,visitor,dimensionBinding.createFor(inheritedProfile.getDimensions()), owner); + inheritedProfile.accept(allowContent, visitor, dimensionBinding.createFor(inheritedProfile.getDimensions()), owner); if (visitor.isDone()) return; } } private void visitContent(QueryProfileVisitor visitor,DimensionBinding dimensionBinding) { - String contentKey=visitor.getLocalKey(); + String contentKey = visitor.getLocalKey(); // Visit this' content - if (contentKey!=null) { // Get only the content of the current key - if (type!=null) - contentKey=type.unalias(contentKey); + if (contentKey != null) { // Get only the content of the current key + if (type != null) + contentKey = type.unalias(contentKey); visitor.acceptValue(contentKey, getContent(contentKey), dimensionBinding, this); } else { // get all content in this @@ -590,11 +590,11 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable /** Sets the value of a node in <i>this</i> profile - the local name given must not be nested (contain dots) */ protected QueryProfile setLocalNode(String localName, Object value,QueryProfileType parentType, DimensionBinding dimensionBinding, QueryProfileRegistry registry) { - if (parentType!=null && type==null && !isFrozen()) - type=parentType; + if (parentType != null && type == null && ! isFrozen()) + type = parentType; - value=checkAndConvertAssignment(localName, value, registry); - localPut(localName,value,dimensionBinding); + value = checkAndConvertAssignment(localName, value, registry); + localPut(localName, value, dimensionBinding); return this; } @@ -605,20 +605,20 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable */ static Object combineValues(Object newValue, Object existingValue) { if (newValue instanceof QueryProfile) { - QueryProfile newProfile=(QueryProfile)newValue; - if ( existingValue==null || ! (existingValue instanceof QueryProfile)) { + QueryProfile newProfile = (QueryProfile)newValue; + if ( existingValue == null || ! (existingValue instanceof QueryProfile)) { if (!isModifiable(newProfile)) - newProfile=new BackedOverridableQueryProfile(newProfile); // Make the query profile reference overridable - newProfile.value=existingValue; + newProfile = new BackedOverridableQueryProfile(newProfile); // Make the query profile reference overridable + newProfile.value = existingValue; return newProfile; } // if both are profiles: - return combineProfiles(newProfile,(QueryProfile)existingValue); + return combineProfiles(newProfile, (QueryProfile)existingValue); } else { if (existingValue instanceof QueryProfile) { // we need to set a non-leaf value on a query profile - QueryProfile existingProfile=(QueryProfile)existingValue; + QueryProfile existingProfile = (QueryProfile)existingValue; if (isModifiable(existingProfile)) { existingProfile.setValue(newValue); return null; @@ -636,16 +636,16 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable } private static QueryProfile combineProfiles(QueryProfile newProfile,QueryProfile existingProfile) { - QueryProfile returnValue=null; + QueryProfile returnValue = null; QueryProfile existingModifiable; // Ensure the existing profile is modifiable - if (existingProfile.getClass()==QueryProfile.class) { + if (existingProfile.getClass() == QueryProfile.class) { existingModifiable = new BackedOverridableQueryProfile(existingProfile); - returnValue=existingModifiable; + returnValue = existingModifiable; } else { // is an overridable wrapper - existingModifiable=existingProfile; // May be used as-is + existingModifiable = existingProfile; // May be used as-is } // Make the existing profile inherit the new one @@ -655,7 +655,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable existingModifiable.addInherited(newProfile); // Remove content from the existing which the new one does not allow overrides of - if (existingModifiable.content!=null) { + if (existingModifiable.content != null) { for (String key : existingModifiable.content.unmodifiableMap().keySet()) { if ( ! newProfile.isLocalOverridable(key, null)) { existingModifiable.content.remove(key); @@ -681,10 +681,10 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable * @throws IllegalArgumentException if the assignment is illegal */ protected Object checkAndConvertAssignment(String localName, Object value, QueryProfileRegistry registry) { - if (type==null) return value; // no type checking + if (type == null) return value; // no type checking - FieldDescription fieldDescription=type.getField(localName); - if (fieldDescription==null) { + FieldDescription fieldDescription = type.getField(localName); + if (fieldDescription == null) { if (type.isStrict()) throw new IllegalArgumentException("'" + localName + "' is not declared in " + type + ", and the type is strict"); return value; @@ -710,8 +710,8 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable /** Do a variant-aware content lookup in this */ protected Object localLookup(String name, DimensionBinding dimensionBinding) { Object node = null; - if ( variants != null && !dimensionBinding.isNull()) - node = variants.get(name,type,true,dimensionBinding); + if ( variants != null && ! dimensionBinding.isNull()) + node = variants.get(name,type,true, dimensionBinding); if (node == null) node = content == null ? null : content.get(name); return node; @@ -801,7 +801,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable } /** Sets a value directly in this query profile (unless frozen) */ - private void localPut(String localName,Object value, DimensionBinding dimensionBinding) { + private void localPut(String localName, Object value, DimensionBinding dimensionBinding) { ensureNotFrozen(); if (type != null) @@ -813,17 +813,17 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable if (dimensionBinding.isNull()) { Object combinedValue; if (value instanceof QueryProfile) - combinedValue = combineValues(value,content==null ? null : content.get(localName)); + combinedValue = combineValues(value, content == null ? null : content.get(localName)); else combinedValue = combineValues(value, localLookup(localName, dimensionBinding)); if (combinedValue!=null) - content.put(localName,combinedValue); + content.put(localName, combinedValue); } else { if (variants == null) variants = new QueryProfileVariants(dimensionBinding.getDimensions(), this); - variants.set(localName,dimensionBinding.getValues(),value); + variants.set(localName, dimensionBinding.getValues(), value); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java index accef7ba154..fd2852fda60 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java @@ -33,31 +33,36 @@ public class QueryProfileCompiler { } public static CompiledQueryProfile compile(QueryProfile in, CompiledQueryProfileRegistry registry) { - DimensionalMap.Builder<CompoundName, Object> values = new DimensionalMap.Builder<>(); - DimensionalMap.Builder<CompoundName, QueryProfileType> types = new DimensionalMap.Builder<>(); - DimensionalMap.Builder<CompoundName, Object> references = new DimensionalMap.Builder<>(); - DimensionalMap.Builder<CompoundName, Object> unoverridables = new DimensionalMap.Builder<>(); - - // Resolve values for each existing variant and combine into a single data structure - Set<DimensionBindingForPath> variants = collectVariants(CompoundName.empty, in, DimensionBinding.nullBinding); - variants.add(new DimensionBindingForPath(DimensionBinding.nullBinding, CompoundName.empty)); // if this contains no variants - log.fine(() -> "Compiling " + in.toString() + " having " + variants.size() + " variants"); - for (DimensionBindingForPath variant : variants) { - log.finer(() -> " Compiling variant " + variant); - for (Map.Entry<String, Object> entry : in.listValues(variant.path(), variant.binding().getContext(), null).entrySet()) { - values.put(variant.path().append(entry.getKey()), variant.binding(), entry.getValue()); + try { + DimensionalMap.Builder<CompoundName, Object> values = new DimensionalMap.Builder<>(); + DimensionalMap.Builder<CompoundName, QueryProfileType> types = new DimensionalMap.Builder<>(); + DimensionalMap.Builder<CompoundName, Object> references = new DimensionalMap.Builder<>(); + DimensionalMap.Builder<CompoundName, Object> unoverridables = new DimensionalMap.Builder<>(); + + // Resolve values for each existing variant and combine into a single data structure + Set<DimensionBindingForPath> variants = collectVariants(CompoundName.empty, in, DimensionBinding.nullBinding); + variants.add(new DimensionBindingForPath(DimensionBinding.nullBinding, CompoundName.empty)); // if this contains no variants + log.fine(() -> "Compiling " + in.toString() + " having " + variants.size() + " variants"); + for (DimensionBindingForPath variant : variants) { + log.finer(() -> " Compiling variant " + variant); + for (Map.Entry<String, Object> entry : in.listValues(variant.path(), variant.binding().getContext(), null).entrySet()) { + values.put(variant.path().append(entry.getKey()), variant.binding(), entry.getValue()); + } + for (Map.Entry<CompoundName, QueryProfileType> entry : in.listTypes(variant.path(), variant.binding().getContext()).entrySet()) + types.put(variant.path().append(entry.getKey()), variant.binding(), entry.getValue()); + for (CompoundName reference : in.listReferences(variant.path(), variant.binding().getContext())) + references.put(variant.path().append(reference), variant.binding(), Boolean.TRUE); // Used as a set; value is ignored + for (CompoundName name : in.listUnoverridable(variant.path(), variant.binding().getContext())) + unoverridables.put(variant.path().append(name), variant.binding(), Boolean.TRUE); // Used as a set; value is ignored } - for (Map.Entry<CompoundName, QueryProfileType> entry : in.listTypes(variant.path(), variant.binding().getContext()).entrySet()) - types.put(variant.path().append(entry.getKey()), variant.binding(), entry.getValue()); - for (CompoundName reference : in.listReferences(variant.path(), variant.binding().getContext())) - references.put(variant.path().append(reference), variant.binding(), Boolean.TRUE); // Used as a set; value is ignored - for (CompoundName name : in.listUnoverridable(variant.path(), variant.binding().getContext())) - unoverridables.put(variant.path().append(name), variant.binding(), Boolean.TRUE); // Used as a set; value is ignored - } - return new CompiledQueryProfile(in.getId(), in.getType(), - values.build(), types.build(), references.build(), unoverridables.build(), - registry); + return new CompiledQueryProfile(in.getId(), in.getType(), + values.build(), types.build(), references.build(), unoverridables.build(), + registry); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid " + in, e); + } } /** diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/SubstituteString.java b/container-search/src/main/java/com/yahoo/search/query/profile/SubstituteString.java index 3252f0f4662..446bb250856 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/SubstituteString.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/SubstituteString.java @@ -2,6 +2,7 @@ package com.yahoo.search.query.profile; import com.yahoo.processing.request.Properties; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import java.util.ArrayList; import java.util.List; @@ -22,6 +23,7 @@ public class SubstituteString { private final List<Component> components; private final String stringValue; + private final boolean hasRelative; /** * Returns a new SubstituteString if the given string contains substitutions, null otherwise. @@ -35,34 +37,48 @@ public class SubstituteString { int end = value.indexOf("}", start + 2); if (end < 0) throw new IllegalArgumentException("Unterminated value substitution '" + value.substring(start) + "'"); - String propertyName = value.substring(start+2,end); - if (propertyName.indexOf("%{") >= 0) + String propertyName = value.substring(start + 2, end); + if (propertyName.contains("%{")) throw new IllegalArgumentException("Unterminated value substitution '" + value.substring(start) + "'"); components.add(new StringComponent(value.substring(lastEnd, start))); - components.add(new PropertyComponent(propertyName)); - lastEnd = end+1; + if (propertyName.startsWith(".")) + components.add(new RelativePropertyComponent(propertyName.substring(1))); + else + components.add(new PropertyComponent(propertyName)); + lastEnd = end + 1; start = value.indexOf("%{", lastEnd); } components.add(new StringComponent(value.substring(lastEnd))); return new SubstituteString(components, value); } - private SubstituteString(List<Component> components, String stringValue) { + public SubstituteString(List<Component> components, String stringValue) { this.components = components; this.stringValue = stringValue; + this.hasRelative = components.stream().anyMatch(component -> component instanceof RelativePropertyComponent); } + /** Returns whether this has at least one relative component */ + public boolean hasRelative() { return hasRelative; } + /** - * Perform the substitution in this, by looking up in the given query profile, + * Perform the substitution in this, by looking up in the given properties, * and returns the resulting string + * + * @param context the content which is used to resolve profile variants when looking up substitution values + * @param substitution the properties in which values to be substituted are looked up */ public String substitute(Map<String, String> context, Properties substitution) { StringBuilder b = new StringBuilder(); for (Component component : components) - b.append(component.getValue(context,substitution)); + b.append(component.getValue(context, substitution)); return b.toString(); } + public List<Component> components() { return components; } + + public String stringValue() { return stringValue; } + @Override public int hashCode() { return stringValue.hashCode(); @@ -81,13 +97,13 @@ public class SubstituteString { return stringValue; } - private abstract static class Component { + public abstract static class Component { protected abstract String getValue(Map<String, String> context, Properties substitution); } - private final static class StringComponent extends Component { + public final static class StringComponent extends Component { private final String value; @@ -107,7 +123,7 @@ public class SubstituteString { } - private final static class PropertyComponent extends Component { + public final static class PropertyComponent extends Component { private final String propertyName; @@ -116,7 +132,7 @@ public class SubstituteString { } @Override - public String getValue(Map<String,String> context, Properties substitution) { + public String getValue(Map<String, String> context, Properties substitution) { Object value = substitution.get(propertyName, context, substitution); if (value == null) return ""; return String.valueOf(value); @@ -129,4 +145,30 @@ public class SubstituteString { } + /** + * A component where the value should be looked up in the profile containing the substitution field + * rather than globally + */ + public final static class RelativePropertyComponent extends Component { + + private final String fieldName; + + public RelativePropertyComponent(String fieldName) { + this.fieldName = fieldName; + } + + @Override + public String getValue(Map<String, String> context, Properties substitution) { + throw new IllegalStateException("Should be resolved during compilation"); + } + + public String fieldName() { return fieldName; } + + @Override + public String toString() { + return "%{" + fieldName + "}"; + } + + } + } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java index d94d601f103..2774bd4ebf2 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java @@ -34,7 +34,7 @@ public class Binding implements Comparable<Binding> { private final int hashCode; @SuppressWarnings("unchecked") - public static final Binding nullBinding= new Binding(Integer.MAX_VALUE, Collections.<String,String>emptyMap()); + public static final Binding nullBinding = new Binding(Integer.MAX_VALUE, Collections.<String,String>emptyMap()); public static Binding createFrom(DimensionBinding dimensionBinding) { if (dimensionBinding.getDimensions().size() > maxDimensions) diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java index d6e93701ca1..ea85a2be242 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java @@ -45,7 +45,8 @@ public class CompiledQueryProfile extends AbstractComponent implements Cloneable /** * Creates a new query profile from an id. */ - public CompiledQueryProfile(ComponentId id, QueryProfileType type, + public CompiledQueryProfile(ComponentId id, + QueryProfileType type, DimensionalMap<CompoundName, Object> entries, DimensionalMap<CompoundName, QueryProfileType> types, DimensionalMap<CompoundName, Object> references, diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalMap.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalMap.java index b4a1c66e4e0..2e8f5dcf91c 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalMap.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalMap.java @@ -58,7 +58,7 @@ public class DimensionalMap<KEY, VALUE> { public DimensionalMap<KEY, VALUE> build() { Map<KEY, DimensionalValue<VALUE>> map = new HashMap<>(); for (Map.Entry<KEY, DimensionalValue.Builder<VALUE>> entry : entries.entrySet()) { - map.put(entry.getKey(), entry.getValue().build()); + map.put(entry.getKey(), entry.getValue().build(entries)); } return new DimensionalMap<>(map); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java index 506472c97d1..50d0a2de46f 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java @@ -1,7 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.compiled; +import com.yahoo.processing.request.CompoundName; import com.yahoo.search.query.profile.DimensionBinding; +import com.yahoo.search.query.profile.SubstituteString; import java.util.ArrayList; import java.util.Collections; @@ -58,6 +60,15 @@ public class DimensionalValue<VALUE> { /** The minimal set of variants needed to capture all values at this key */ private Map<VALUE, Value.Builder<VALUE>> buildableVariants = new HashMap<>(); + /** Returns the value for the given binding, or null if none */ + public VALUE valueFor(DimensionBinding variantBinding) { + for (var entry : buildableVariants.entrySet()) { + if (entry.getValue().variants.contains(variantBinding)) + return entry.getKey(); + } + return null; + } + public void add(VALUE value, DimensionBinding variantBinding) { // Note: We know we can index by the value because its possible types are constrained // to what query profiles allow: String, primitives and query profiles @@ -69,10 +80,10 @@ public class DimensionalValue<VALUE> { variant.addVariant(variantBinding); } - public DimensionalValue<VALUE> build() { + public DimensionalValue<VALUE> build(Map<?, DimensionalValue.Builder<VALUE>> entries) { List<Value> variants = new ArrayList<>(); for (Value.Builder buildableVariant : buildableVariants.values()) { - variants.addAll(buildableVariant.build()); + variants.addAll(buildableVariant.build(entries)); } return new DimensionalValue(variants); } @@ -139,14 +150,17 @@ public class DimensionalValue<VALUE> { } /** Build a separate value object for each dimension combination which has this value */ - public List<Value<VALUE>> build() { + public List<Value<VALUE>> build(Map<CompoundName, DimensionalValue.Builder<VALUE>> entries) { // Shortcut for efficiency of the normal case - if (variants.size()==1) - return Collections.singletonList(new Value<>(value, Binding.createFrom(variants.iterator().next()))); + if (variants.size() == 1) { + return Collections.singletonList(new Value<>(substituteIfRelative(value, variants.iterator().next(), entries), + Binding.createFrom(variants.iterator().next()))); + } List<Value<VALUE>> values = new ArrayList<>(variants.size()); - for (DimensionBinding variant : variants) - values.add(new Value<>(value, Binding.createFrom(variant))); + for (DimensionBinding variant : variants) { + values.add(new Value<>(substituteIfRelative(value, variant, entries), Binding.createFrom(variant))); + } return values; } @@ -154,6 +168,46 @@ public class DimensionalValue<VALUE> { return value; } + @SuppressWarnings("unchecked") + private VALUE substituteIfRelative(VALUE value, + DimensionBinding variant, + Map<CompoundName, DimensionalValue.Builder<VALUE>> entries) { + if (value instanceof SubstituteString) { + SubstituteString substitute = (SubstituteString)value; + if (substitute.hasRelative()) { + List<SubstituteString.Component> resolvedComponents = new ArrayList<>(substitute.components().size()); + for (SubstituteString.Component component : substitute.components()) { + if (component instanceof SubstituteString.RelativePropertyComponent) { + SubstituteString.RelativePropertyComponent relativeComponent = (SubstituteString.RelativePropertyComponent)component; + var substituteValues = lookupByLocalName(relativeComponent.fieldName(), entries); + if (substituteValues == null) + throw new IllegalArgumentException("Could not resolve local substitution '" + + relativeComponent.fieldName() + "' in variant " + + variant); + String resolved = substituteValues.valueFor(variant).toString(); + resolvedComponents.add(new SubstituteString.StringComponent(resolved)); + } + else { + resolvedComponents.add(component); + } + } + return (VALUE)new SubstituteString(resolvedComponents, substitute.stringValue()); + } + } + return value; + } + + private DimensionalValue.Builder<VALUE> lookupByLocalName(String localName, + Map<CompoundName, DimensionalValue.Builder<VALUE>> entries) { + for (var entry : entries.entrySet()) { + if (entry.getKey().last().equals(localName)) + return entry.getValue(); + } + return null; + } + } + } + } diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index eb4d65693bb..4011611b049 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -81,7 +81,7 @@ public class FastSearcherTestCase { @Test public void testSinglePassGroupingIsForcedWithSingleNodeGroups() { FastSearcher fastSearcher = new FastSearcher("container.0", - MockDispatcher.create(Collections.singletonList(new Node(0, "host0", 123, 0))), + MockDispatcher.create(Collections.singletonList(new Node(0, "host0", 0))), new SummaryParameters(null), new ClusterParams("testhittype"), documentdbInfoConfig); @@ -102,7 +102,7 @@ public class FastSearcherTestCase { @Test public void testSinglePassGroupingIsNotForcedWithSingleNodeGroups() { - MockDispatcher dispatcher = MockDispatcher.create(ImmutableList.of(new Node(0, "host0", 123, 0), new Node(2, "host1", 123, 0))); + MockDispatcher dispatcher = MockDispatcher.create(ImmutableList.of(new Node(0, "host0", 0), new Node(2, "host1", 0))); FastSearcher fastSearcher = new FastSearcher("container.0", dispatcher, diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java index 0aa91442712..4fbbd9dd936 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java @@ -39,7 +39,6 @@ class MockDispatcher extends Dispatcher { for (Node node : nodes) { DispatchConfig.Node.Builder dispatchConfigNodeBuilder = new DispatchConfig.Node.Builder(); dispatchConfigNodeBuilder.host(node.hostname()); - dispatchConfigNodeBuilder.fs4port(node.fs4port()); dispatchConfigNodeBuilder.port(0); // Mandatory, but currently not used here dispatchConfigNodeBuilder.group(node.group()); dispatchConfigNodeBuilder.key(key++); // not used diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java index 310f536f961..3d544f5c114 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java @@ -70,7 +70,7 @@ public class DispatcherTest { SearchCluster cl = new MockSearchCluster("1", 0, 0) { @Override public Optional<Node> localCorpusDispatchTarget() { - return Optional.of(new Node(1, "test", 123, 1)); + return Optional.of(new Node(1, "test", 1)); } }; MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (n, a) -> true); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java index 1ebf7940f25..0496194f8ed 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java @@ -28,7 +28,7 @@ import static org.junit.Assert.assertThat; public class LoadBalancerTest { @Test public void requireThatLoadBalancerServesSingleNodeSetups() { - Node n1 = new Node(0, "test-node1", 0, 0); + Node n1 = new Node(0, "test-node1", 0); SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1), 1, null); LoadBalancer lb = new LoadBalancer(cluster, true); @@ -41,8 +41,8 @@ public class LoadBalancerTest { @Test public void requireThatLoadBalancerServesMultiGroupSetups() { - Node n1 = new Node(0, "test-node1", 0, 0); - Node n2 = new Node(1, "test-node2", 1, 1); + Node n1 = new Node(0, "test-node1", 0); + Node n2 = new Node(1, "test-node2", 1); SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null); LoadBalancer lb = new LoadBalancer(cluster, true); @@ -55,10 +55,10 @@ public class LoadBalancerTest { @Test public void requireThatLoadBalancerServesClusteredGroups() { - Node n1 = new Node(0, "test-node1", 0, 0); - Node n2 = new Node(1, "test-node2", 1, 0); - Node n3 = new Node(0, "test-node3", 0, 1); - Node n4 = new Node(1, "test-node4", 1, 1); + Node n1 = new Node(0, "test-node1", 0); + Node n2 = new Node(1, "test-node2", 0); + Node n3 = new Node(0, "test-node3", 1); + Node n4 = new Node(1, "test-node4", 1); SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2, n3, n4), 2, null); LoadBalancer lb = new LoadBalancer(cluster, true); @@ -68,8 +68,8 @@ public class LoadBalancerTest { @Test public void requireThatLoadBalancerReturnsDifferentGroups() { - Node n1 = new Node(0, "test-node1", 0, 0); - Node n2 = new Node(1, "test-node2", 1, 1); + Node n1 = new Node(0, "test-node1", 0); + Node n2 = new Node(1, "test-node2", 1); SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null); LoadBalancer lb = new LoadBalancer(cluster, true); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java index 2fe434d6f3f..c5fbda7c2f5 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java @@ -19,7 +19,7 @@ class MockInvoker extends SearchInvoker { private List<Hit> hits; protected MockInvoker(int key, Coverage coverage) { - super(Optional.of(new Node(key, "?", 0, 0))); + super(Optional.of(new Node(key, "?", 0))); this.coverage = coverage; } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index e3ff54102d4..0bcc30d9b10 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -38,7 +38,7 @@ public class MockSearchCluster extends SearchCluster { for (int group = 0; group < groups; group++) { List<Node> nodes = new ArrayList<>(); for (int node = 0; node < nodesPerGroup; node++) { - Node n = new Node(dk, "host" + dk, -1, group); + Node n = new Node(dk, "host" + dk, group); n.setWorking(true); nodes.add(n); hostBuilder.put(n.hostname(), n); @@ -124,8 +124,9 @@ public class MockSearchCluster extends SearchCluster { builder.minWaitAfterCoverageFactor(0); builder.maxWaitAfterCoverageFactor(0.5); } + int port = 10000; for (Node n : nodes) { - builder.node(new DispatchConfig.Node.Builder().key(n.key()).host(n.hostname()).port(n.fs4port()).group(n.group())); + builder.node(new DispatchConfig.Node.Builder().key(n.key()).host(n.hostname()).port(port++).group(n.group())); } return new DispatchConfig(builder); } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java index d629bd36bb1..c07bf119782 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java @@ -36,7 +36,7 @@ public class RpcSearchInvokerTest { var mockClient = parameterCollectorClient(compressionTypeHolder, payloadHolder, lengthHolder); var mockPool = new RpcResourcePool(ImmutableMap.of(7, mockClient.createConnection("foo", 123))); @SuppressWarnings("resource") - var invoker = new RpcSearchInvoker(mockSearcher(), new Node(7, "seven", 77, 1), mockPool); + var invoker = new RpcSearchInvoker(mockSearcher(), new Node(7, "seven", 1), mockPool); Query q = new Query("search/?query=test&hits=10&offset=3"); invoker.sendSearchRequest(q); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java index f29d6ddf324..f42185e955f 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java @@ -63,7 +63,7 @@ public class SearchClusterTest { for (String name : nodeNames) { int key = nodes.size() % nodesPergroup; int group = nodes.size() / nodesPergroup; - nodes.add(new Node(key, name, 13333, group)); + nodes.add(new Node(key, name, group)); numDocsPerNode.add(new AtomicInteger(1)); pingCounts.add(new AtomicInteger(0)); } @@ -132,7 +132,7 @@ public class SearchClusterTest { @Override public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) { - int index = node.group*numPerGroup + node.key(); + int index = node.group() * numPerGroup + node.key(); return new Pinger(activeDocs.get(index), pingCounts.get(index)); } } diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileSubstitutionTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileSubstitutionTestCase.java index ca1447b475a..b3b83b9c07e 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileSubstitutionTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileSubstitutionTestCase.java @@ -17,26 +17,56 @@ import static org.junit.Assert.fail; public class QueryProfileSubstitutionTestCase { @Test + public void testSubstitutionOnly() { + QueryProfile p = new QueryProfile("test"); + p.set("message","%{world}", null); + p.set("world", "world", null); + assertEquals("world", p.compile(null).get("message")); + } + + @Test public void testSingleSubstitution() { QueryProfile p = new QueryProfile("test"); p.set("message","Hello %{world}!", null); p.set("world", "world", null); - assertEquals("Hello world!",p.compile(null).get("message")); + assertEquals("Hello world!", p.compile(null).get("message")); - QueryProfile p2=new QueryProfile("test2"); + QueryProfile p2 = new QueryProfile("test2"); p2.addInherited(p); p2.set("world", "universe", null); assertEquals("Hello universe!", p2.compile(null).get("message")); } @Test + public void testRelativeSubstitution() { + QueryProfile p = new QueryProfile("test"); + p.set("message","Hello %{.world}!", null); + p.set("world", "world", null); + assertEquals("Hello world!", p.compile(null).get("message")); + } + + @Test + public void testRelativeSubstitutionNotFound() { + try { + QueryProfile p = new QueryProfile("test"); + p.set("message", "Hello %{.world}!", null); + assertEquals("Hello world!", p.compile(null).get("message")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("Invalid query profile 'test': Could not resolve local substitution 'world' in variant DimensionBinding []", + Exceptions.toMessageString(e)); + } + } + + @Test public void testMultipleSubstitutions() { - QueryProfile p=new QueryProfile("test"); + QueryProfile p = new QueryProfile("test"); p.set("message","%{greeting} %{entity}%{exclamation}", null); p.set("greeting","Hola", null); p.set("entity","local group", null); p.set("exclamation","?", null); - assertEquals("Hola local group?",p.compile(null).get("message")); + assertEquals("Hola local group?", p.compile(null).get("message")); QueryProfile p2 = new QueryProfile("test2"); p2.addInherited(p); diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java index d9bf4a1db97..3da4558d67c 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java @@ -978,44 +978,47 @@ public class QueryProfileVariantsTestCase { @Test public void testQueryProfileReferencesWithSubstitution() { - QueryProfile main=new QueryProfile("main"); + QueryProfile main = new QueryProfile("main"); main.setDimensions(new String[] {"x1"}); - QueryProfile referencedMain=new QueryProfile("referencedMain"); + QueryProfile referencedMain = new QueryProfile("referencedMain"); referencedMain.set("r1","%{prefix}mainReferenced-r1", null); // In both referencedMain.set("r2","%{prefix}mainReferenced-r2", null); // Only in this - QueryProfile referencedVariant=new QueryProfile("referencedVariant"); + QueryProfile referencedVariant = new QueryProfile("referencedVariant"); referencedVariant.set("r1","%{prefix}variantReferenced-r1", null); // In both referencedVariant.set("r3","%{prefix}variantReferenced-r3", null); // Only in this + referencedVariant.set("inthis", "local value", null); + referencedVariant.set("r4","This has %{.inthis}", null); // Relative - main.set("a",referencedMain, null); - main.set("a",referencedVariant,new String[] {"x1"}, null); - main.set("prefix","mainPrefix:", null); - main.set("prefix","variantPrefix:",new String[] {"x1"}, null); + main.set("a", referencedMain, null); + main.set("a", referencedVariant,new String[] {"x1"}, null); + main.set("prefix", "mainPrefix:", null); + main.set("prefix", "variantPrefix:", new String[] {"x1"}, null); - Properties properties=new QueryProfileProperties(main.compile(null)); + Properties properties = new QueryProfileProperties(main.compile(null)); // No context - Map<String,Object> listed=properties.listProperties(); - assertEquals(3,listed.size()); - assertEquals("mainPrefix:mainReferenced-r1",listed.get("a.r1")); - assertEquals("mainPrefix:mainReferenced-r2",listed.get("a.r2")); + Map<String,Object> listed = properties.listProperties(); + assertEquals(3, listed.size()); + assertEquals("mainPrefix:mainReferenced-r1", listed.get("a.r1")); + assertEquals("mainPrefix:mainReferenced-r2", listed.get("a.r2")); // Context x=x1 - listed=properties.listProperties(toMap(main,new String[] {"x1"})); - assertEquals(4,listed.size()); - assertEquals("variantPrefix:variantReferenced-r1",listed.get("a.r1")); - assertEquals("variantPrefix:mainReferenced-r2",listed.get("a.r2")); - assertEquals("variantPrefix:variantReferenced-r3",listed.get("a.r3")); + listed = properties.listProperties(toMap(main, new String[] {"x1"})); + assertEquals(6, listed.size()); + assertEquals("variantPrefix:variantReferenced-r1", listed.get("a.r1")); + assertEquals("variantPrefix:mainReferenced-r2", listed.get("a.r2")); + assertEquals("variantPrefix:variantReferenced-r3", listed.get("a.r3")); + assertEquals("This has local value", listed.get("a.r4")); } @Test public void testNewsCase1() { - QueryProfile shortcuts=new QueryProfile("shortcuts"); - shortcuts.setDimensions(new String[] {"custid_1","custid_2","custid_3","custid_4","custid_5","custid_6"}); - shortcuts.set("testout","outside", null); - shortcuts.set("test.out","dotoutside", null); - shortcuts.set("testin","inside",new String[] {"yahoo","ca","sc"}, null); - shortcuts.set("test.in","dotinside",new String[] {"yahoo","ca","sc"}, null); + QueryProfile shortcuts = new QueryProfile("shortcuts"); + shortcuts.setDimensions(new String[] {"custid_1", "custid_2", "custid_3", "custid_4", "custid_5", "custid_6"}); + shortcuts.set("testout", "outside", null); + shortcuts.set("test.out", "dotoutside", null); + shortcuts.set("testin", "inside", new String[] {"yahoo","ca","sc"}, null); + shortcuts.set("test.in", "dotinside", new String[] {"yahoo","ca","sc"}, null); QueryProfile profile=new QueryProfile("default"); profile.setDimensions(new String[] {"custid_1","custid_2","custid_3","custid_4","custid_5","custid_6"}); @@ -1024,10 +1027,10 @@ public class QueryProfileVariantsTestCase { profile.freeze(); Query query = new Query(HttpRequest.createTestRequest("?query=test&custid_1=yahoo&custid_2=ca&custid_3=sc", Method.GET), profile.compile(null)); - assertEquals("outside",query.properties().get("testout")); - assertEquals("dotoutside",query.properties().get("test.out")); - assertEquals("inside",query.properties().get("testin")); - assertEquals("dotinside",query.properties().get("test.in")); + assertEquals("outside", query.properties().get("testout")); + assertEquals("dotoutside", query.properties().get("test.out")); + assertEquals("inside", query.properties().get("testin")); + assertEquals("dotinside", query.properties().get("test.in")); } @Test diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/AwsLimitsFetcher.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/AwsLimitsFetcher.java deleted file mode 100644 index 4e76f67e7cf..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/AwsLimitsFetcher.java +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.aws; - -/** - * @author freva - */ -public interface AwsLimitsFetcher { - - /** Returns the AWS EC2 instance limits in the given AWS region */ - Ec2InstanceCounts getEc2InstanceLimits(String awsRegion); - - /** Returns the current usage of AWS EC2 instances in the given AWS region */ - Ec2InstanceCounts getEc2InstanceUsage(String awsRegion); -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/Ec2InstanceCounts.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/Ec2InstanceCounts.java deleted file mode 100644 index 044789f14e4..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/Ec2InstanceCounts.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.aws; - -import java.util.Map; -import java.util.Objects; - -/** - * @author freva - */ -public class Ec2InstanceCounts { - private final int totalCount; - private final Map<String, Integer> instanceCounts; - - public Ec2InstanceCounts(int totalCount, Map<String, Integer> instanceCounts) { - this.totalCount = totalCount; - this.instanceCounts = Map.copyOf(instanceCounts); - } - - public int getTotalCount() { - return totalCount; - } - - /** Returns map of counts by instance type, e.g. 'r5.2xlarge' */ - public Map<String, Integer> getInstanceCounts() { - return instanceCounts; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Ec2InstanceCounts that = (Ec2InstanceCounts) o; - return totalCount == that.totalCount && - instanceCounts.equals(that.instanceCounts); - } - - @Override - public int hashCode() { - return Objects.hash(totalCount, instanceCounts); - } - - @Override - public String toString() { - return "Ec2InstanceLimits{" + - "totalLimit=" + totalCount + - ", instanceCounts=" + instanceCounts + - '}'; - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java index a378bcb63bd..5ee6df9f034 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.resource; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import java.time.Instant; @@ -17,14 +18,16 @@ public class ResourceSnapshot { private final ApplicationId applicationId; private final ResourceAllocation resourceAllocation; private final Instant timestamp; + private final ZoneId zoneId; - public ResourceSnapshot(ApplicationId applicationId, double cpuCores, double memoryGb, double diskGb, Instant timestamp) { + public ResourceSnapshot(ApplicationId applicationId, double cpuCores, double memoryGb, double diskGb, Instant timestamp, ZoneId zoneId) { this.applicationId = applicationId; this.resourceAllocation = new ResourceAllocation(cpuCores, memoryGb, diskGb); this.timestamp = timestamp; + this.zoneId = zoneId; } - public static ResourceSnapshot from(List<Node> nodes, Instant timestamp) { + public static ResourceSnapshot from(List<Node> nodes, Instant timestamp, ZoneId zoneId) { Set<ApplicationId> applicationIds = nodes.stream() .filter(node -> node.owner().isPresent()) .map(node -> node.owner().get()) @@ -37,7 +40,8 @@ public class ResourceSnapshot { nodes.stream().mapToDouble(Node::vcpu).sum(), nodes.stream().mapToDouble(Node::memoryGb).sum(), nodes.stream().mapToDouble(Node::diskGb).sum(), - timestamp + timestamp, + zoneId ); } @@ -61,4 +65,8 @@ public class ResourceSnapshot { return timestamp; } + public ZoneId getZoneId() { + return zoneId; + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index bfe7fc1ee2e..8592460a24f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -214,7 +214,7 @@ public class ApplicationController { public ApplicationStore applicationStore() { return applicationStore; } /** Returns all content clusters in all current deployments of the given application. */ - public Map<ZoneId, List<String>> listClusters(ApplicationId id, Iterable<ZoneId> zones) { + public Map<ZoneId, List<String>> contentClustersByZone(ApplicationId id, Iterable<ZoneId> zones) { ImmutableMap.Builder<ZoneId, List<String>> clusters = ImmutableMap.builder(); for (ZoneId zone : zones) clusters.put(zone, ImmutableList.copyOf(configServer.getContentClusters(new DeploymentId(id, zone)))); @@ -385,10 +385,9 @@ public class ApplicationController { } if (zone.environment().isProduction()) // Assign and register endpoints - application = withRotation(application, instance); - - endpoints = registerEndpointsInDns(application.get().deploymentSpec(), application.get().require(instanceId.instance()), zone); + application = withRotation(applicationPackage.deploymentSpec(), application, instance); + endpoints = registerEndpointsInDns(applicationPackage.deploymentSpec(), application.get().require(instanceId.instance()), zone); if (controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) { // Provisions a new certificate if missing @@ -518,9 +517,9 @@ public class ApplicationController { } /** Makes sure the application has a global rotation, if eligible. */ - private LockedApplication withRotation(LockedApplication application, InstanceName instanceName) { + private LockedApplication withRotation(DeploymentSpec deploymentSpec, LockedApplication application, InstanceName instanceName) { try (RotationLock rotationLock = rotationRepository.lock()) { - var rotations = rotationRepository.getOrAssignRotations(application.get().deploymentSpec(), + var rotations = rotationRepository.getOrAssignRotations(deploymentSpec, application.get().require(instanceName), rotationLock); application = application.with(instanceName, instance -> instance.with(rotations)); @@ -536,7 +535,7 @@ public class ApplicationController { */ private Set<ContainerEndpoint> registerEndpointsInDns(DeploymentSpec deploymentSpec, Instance instance, ZoneId zone) { var containerEndpoints = new HashSet<ContainerEndpoint>(); - var registerLegacyNames = deploymentSpec.globalServiceId().isPresent(); + boolean registerLegacyNames = deploymentSpec.instance(instance.name()).flatMap(i -> i.globalServiceId()).isPresent(); for (var assignedRotation : instance.rotations()) { var names = new ArrayList<String>(); var endpoints = instance.endpointsIn(controller.system(), assignedRotation.endpointId()) @@ -628,8 +627,8 @@ public class ApplicationController { private LockedApplication withoutDeletedDeployments(LockedApplication application, InstanceName instance) { DeploymentSpec deploymentSpec = application.get().deploymentSpec(); List<Deployment> deploymentsToRemove = application.get().require(instance).productionDeployments().values().stream() - .filter(deployment -> ! deploymentSpec.includes(deployment.zone().environment(), - Optional.of(deployment.zone().region()))) + .filter(deployment -> ! deploymentSpec.requireInstance(instance).includes(deployment.zone().environment(), + Optional.of(deployment.zone().region()))) .collect(Collectors.toList()); if (deploymentsToRemove.isEmpty()) return application; @@ -653,7 +652,7 @@ public class ApplicationController { private Instance withoutUnreferencedDeploymentJobs(DeploymentSpec deploymentSpec, Instance instance) { for (JobType job : JobList.from(instance).production().mapToList(JobStatus::type)) { ZoneId zone = job.zone(controller.system()); - if (deploymentSpec.includes(zone.environment(), Optional.of(zone.region()))) + if (deploymentSpec.requireInstance(instance.name()).includes(zone.environment(), Optional.of(zone.region()))) continue; instance = instance.withoutDeploymentJob(job); } @@ -911,9 +910,9 @@ public class ApplicationController { * 2. If the principal is given, verify that the principal is tenant admin or admin of the tenant domain * 3. If the principal is not given, verify that the Athenz domain of the tenant equals Athenz domain given in deployment.xml * - * @param tenantName Tenant where application should be deployed - * @param applicationPackage Application package - * @param deployer Principal initiating the deployment, possibly empty + * @param tenantName tenant where application should be deployed + * @param applicationPackage application package + * @param deployer principal initiating the deployment, possibly empty */ public void verifyApplicationIdentityConfiguration(TenantName tenantName, ApplicationPackage applicationPackage, Optional<Principal> deployer) { verifyAllowedLaunchAthenzService(applicationPackage.deploymentSpec()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java index f885b7a146e..627cde28fd0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java @@ -13,7 +13,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationV import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; -import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; @@ -87,19 +86,12 @@ public class Instance { Deployment previousDeployment = deployments.getOrDefault(zone, new Deployment(zone, applicationVersion, version, instant)); Deployment newDeployment = new Deployment(zone, applicationVersion, version, instant, - previousDeployment.clusterUtils(), previousDeployment.clusterInfo(), previousDeployment.metrics().with(warnings), previousDeployment.activity()); return with(newDeployment); } - public Instance withClusterUtilization(ZoneId zone, Map<ClusterSpec.Id, ClusterUtilization> clusterUtilization) { - Deployment deployment = deployments.get(zone); - if (deployment == null) return this; // No longer deployed in this zone. - return with(deployment.withClusterUtils(clusterUtilization)); - } - public Instance withClusterInfo(ZoneId zone, Map<ClusterSpec.Id, ClusterInfo> clusterInfo) { Deployment deployment = deployments.get(zone); if (deployment == null) return this; // No longer deployed in this zone. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java index 03d084cd9e3..361dcf9dbf9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; -import com.google.common.collect.ImmutableMap; import com.yahoo.component.Version; import com.yahoo.config.provision.ClusterSpec.Id; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; @@ -9,7 +8,6 @@ import com.yahoo.config.provision.zone.ZoneId; import java.time.Instant; import java.util.Collections; -import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -25,26 +23,24 @@ public class Deployment { private final ApplicationVersion applicationVersion; private final Version version; private final Instant deployTime; - private final Map<Id, ClusterUtilization> clusterUtilization; private final Map<Id, ClusterInfo> clusterInfo; private final DeploymentMetrics metrics; private final DeploymentActivity activity; public Deployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, Instant deployTime) { - this(zone, applicationVersion, version, deployTime, Collections.emptyMap(), Collections.emptyMap(), + this(zone, applicationVersion, version, deployTime, Collections.emptyMap(), DeploymentMetrics.none, DeploymentActivity.none); } public Deployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, Instant deployTime, - Map<Id, ClusterUtilization> clusterUtilization, Map<Id, ClusterInfo> clusterInfo, + Map<Id, ClusterInfo> clusterInfo, DeploymentMetrics metrics, DeploymentActivity activity) { this.zone = Objects.requireNonNull(zone, "zone cannot be null"); this.applicationVersion = Objects.requireNonNull(applicationVersion, "applicationVersion cannot be null"); this.version = Objects.requireNonNull(version, "version cannot be null"); this.deployTime = Objects.requireNonNull(deployTime, "deployTime cannot be null"); - this.clusterUtilization = ImmutableMap.copyOf(Objects.requireNonNull(clusterUtilization, "clusterUtilization cannot be null")); - this.clusterInfo = ImmutableMap.copyOf(Objects.requireNonNull(clusterInfo, "clusterInfo cannot be null")); + this.clusterInfo = Map.copyOf(Objects.requireNonNull(clusterInfo, "clusterInfo cannot be null")); this.metrics = Objects.requireNonNull(metrics, "deploymentMetrics cannot be null"); this.activity = Objects.requireNonNull(activity, "activity cannot be null"); } @@ -74,52 +70,26 @@ public class Deployment { return clusterInfo; } - /** Returns utilization of the clusters allocated to this */ - // TODO(mpolden): No longer updated. Remove this and associated serialization - public Map<Id, ClusterUtilization> clusterUtils() { - return clusterUtilization; - } - public Deployment recordActivityAt(Instant instant) { - return new Deployment(zone, applicationVersion, version, deployTime, clusterUtilization, clusterInfo, metrics, + return new Deployment(zone, applicationVersion, version, deployTime, clusterInfo, metrics, activity.recordAt(instant, metrics)); } - public Deployment withClusterUtils(Map<Id, ClusterUtilization> clusterUtilization) { - return new Deployment(zone, applicationVersion, version, deployTime, clusterUtilization, clusterInfo, metrics, + public Deployment withClusterUtils() { + return new Deployment(zone, applicationVersion, version, deployTime, clusterInfo, metrics, activity); } public Deployment withClusterInfo(Map<Id, ClusterInfo> newClusterInfo) { - return new Deployment(zone, applicationVersion, version, deployTime, clusterUtilization, newClusterInfo, metrics, + return new Deployment(zone, applicationVersion, version, deployTime, newClusterInfo, metrics, activity); } public Deployment withMetrics(DeploymentMetrics metrics) { - return new Deployment(zone, applicationVersion, version, deployTime, clusterUtilization, clusterInfo, metrics, + return new Deployment(zone, applicationVersion, version, deployTime, clusterInfo, metrics, activity); } - /** - * Calculate cost for this deployment. - * - * This is based on cluster utilization and cluster info. - */ - public DeploymentCost calculateCost() { - - Map<String, ClusterCost> costClusters = new HashMap<>(); - for (Id clusterId : clusterUtilization.keySet()) { - - // Only include cluster cost if we have both cluster utilization and cluster info - if (clusterInfo.containsKey(clusterId)) { - costClusters.put(clusterId.value(), new ClusterCost(clusterInfo.get(clusterId), - clusterUtilization.get(clusterId))); - } - } - - return new DeploymentCost(costClusters); - } - @Override public String toString() { return "deployment to " + zone + " of " + applicationVersion + " on version " + version + " at " + deployTime; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentCost.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentCost.java index 371e1c41e32..393c14b35d3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentCost.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentCost.java @@ -17,7 +17,7 @@ public class DeploymentCost { private final Map<String, ClusterCost> clusters; - DeploymentCost(Map<String, ClusterCost> clusterCosts) { + public DeploymentCost(Map<String, ClusterCost> clusterCosts) { clusters = new HashMap<>(clusterCosts); double tco = 0; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentSpecValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentSpecValidator.java index ce7904dc829..5c4d5874e53 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentSpecValidator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentSpecValidator.java @@ -39,7 +39,7 @@ public class DeploymentSpecValidator { /** Verify that each of the production zones listed in the deployment spec exist in this system */ private void validateSteps(DeploymentSpec deploymentSpec) { new DeploymentSteps(deploymentSpec, controller::system).jobs(); - deploymentSpec.zones().stream() + deploymentSpec.instances().stream().flatMap(instance -> instance.zones().stream()) .filter(zone -> zone.environment() == Environment.prod) .forEach(zone -> { if ( ! controller.zoneRegistry().hasZone(ZoneId.from(zone.environment(), @@ -51,16 +51,19 @@ public class DeploymentSpecValidator { /** Verify that no single endpoint contains regions in different clouds */ private void validateEndpoints(DeploymentSpec deploymentSpec) { - for (var endpoint : deploymentSpec.endpoints()) { - var clouds = new HashSet<CloudName>(); - for (var region : endpoint.regions()) { - for (ZoneApi zone : controller.zoneRegistry().zones().all().in(region).zones()) { - clouds.add(zone.getCloudName()); + for (var instance : deploymentSpec.instances()) { + for (var endpoint : instance.endpoints()) { + var clouds = new HashSet<CloudName>(); + for (var region : endpoint.regions()) { + for (ZoneApi zone : controller.zoneRegistry().zones().all().in(region).zones()) { + clouds.add(zone.getCloudName()); + } + } + if (clouds.size() != 1) { + throw new IllegalArgumentException("Endpoint '" + endpoint.endpointId() + "' in " + instance + + " cannot contain regions in different clouds: " + + endpoint.regions().stream().sorted().collect(Collectors.toList())); } - } - if (clouds.size() != 1) { - throw new IllegalArgumentException("Endpoint '" + endpoint.endpointId() + "' cannot contain regions in different clouds: " + - endpoint.regions().stream().sorted().collect(Collectors.toList())); } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 376048143d9..3df889d7a88 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -372,9 +372,8 @@ public class DeploymentTrigger { } else { // All jobs are complete; find the time of completion of this step. if (stepJobs.isEmpty()) { // No jobs means this is a delay step. - Duration delay = ((DeploymentSpec.Delay) step).duration(); - completedAt = completedAt.map(at -> at.plus(delay)).filter(at -> !at.isAfter(clock.instant())); - reason += " after a delay of " + delay; + completedAt = completedAt.map(at -> at.plus(step.delay())).filter(at -> !at.isAfter(clock.instant())); + reason += " after a delay of " + step.delay(); } else { completedAt = stepJobs.stream().map(job -> instance.deploymentJobs().statusOf(job).get().lastCompleted().get().at()).max(naturalOrder()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 42e270edd5e..50af8bd8611 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -1,8 +1,6 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; @@ -468,7 +466,7 @@ public class InternalStepRunner implements StepRunner { testConfigSerializer.configJson(id.application(), id.type(), endpoints, - controller.applications().listClusters(id.application(), zones))); + controller.applications().contentClustersByZone(id.application(), zones))); return Optional.of(running); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 9253e249765..361cc43da50 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobList; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.rotation.RotationLock; +import com.yahoo.vespa.hosted.controller.versions.NodeVersions; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import java.time.Clock; @@ -20,6 +21,7 @@ import java.time.Instant; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.function.Function; import java.util.stream.Collectors; /** @@ -36,6 +38,7 @@ public class MetricsReporter extends Maintainer { public static final String DEPLOYMENT_BUILD_AGE_SECONDS = "deployment.buildAgeSeconds"; public static final String DEPLOYMENT_WARNINGS = "deployment.warnings"; public static final String NODES_FAILING_SYSTEM_UPGRADE = "deployment.nodesFailingSystemUpgrade"; + public static final String NODES_FAILING_OS_UPGRADE = "deployment.nodesFailingOsUpgrade"; public static final String REMAINING_ROTATIONS = "remaining_rotations"; public static final String NAME_SERVICE_REQUESTS_QUEUED = "dns.queuedRequests"; @@ -56,6 +59,7 @@ public class MetricsReporter extends Maintainer { reportRemainingRotations(); reportQueuedNameServiceRequests(); reportNodesFailingSystemUpgrade(); + reportNodesFailingOsUpgrade(); } private void reportRemainingRotations() { @@ -103,13 +107,31 @@ public class MetricsReporter extends Maintainer { metric.set(NODES_FAILING_SYSTEM_UPGRADE, nodesFailingSystemUpgrade(), metric.createContext(Map.of())); } + private void reportNodesFailingOsUpgrade() { + metric.set(NODES_FAILING_OS_UPGRADE, nodesFailingOsUpgrade(), metric.createContext(Map.of())); + } + private int nodesFailingSystemUpgrade() { if (!controller().versionStatus().isUpgrading()) return 0; + return nodesFailingUpgrade(controller().versionStatus().versions(), (vespaVersion) -> { + if (vespaVersion.confidence() == VespaVersion.Confidence.broken) return NodeVersions.EMPTY; + return vespaVersion.nodeVersions(); + }); + } + + private int nodesFailingOsUpgrade() { + return nodesFailingUpgrade(controller().osVersionStatus().versions().entrySet(), (kv) -> { + var osVersion = kv.getKey(); + if (osVersion.version().isEmpty()) return NodeVersions.EMPTY; + return kv.getValue(); + }); + } + + private <V> int nodesFailingUpgrade(Collection<V> collection, Function<V, NodeVersions> nodeVersionsFunction) { var nodesFailingUpgrade = 0; var acceptableInstant = clock.instant().minus(NODE_UPGRADE_TIMEOUT); - for (var vespaVersion : controller().versionStatus().versions()) { - if (vespaVersion.confidence() == VespaVersion.Confidence.broken) continue; - for (var nodeVersion : vespaVersion.nodeVersions().asMap().values()) { + for (var object : collection) { + for (var nodeVersion : nodeVersionsFunction.apply(object).asMap().values()) { if (!nodeVersion.changing()) continue; if (nodeVersion.changedAt().isBefore(acceptableInstant)) nodesFailingUpgrade++; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java index 60bc3d15ec6..93d1dac7382 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java @@ -61,7 +61,7 @@ public class OsUpgrader extends InfrastructureUpgrader { // Return target if we have nodes in this cloud on a lower version return controller().osVersion(cloud) .filter(target -> controller().osVersionStatus().nodesIn(cloud).stream() - .anyMatch(node -> node.version().isBefore(target.version()))) + .anyMatch(node -> node.currentVersion().isBefore(target.version()))) .map(OsVersion::version); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index c700ddac51c..0e14b61c5c5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; @@ -15,6 +17,7 @@ import java.time.Duration; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.function.Predicate; import java.util.stream.Collectors; /** @@ -47,33 +50,44 @@ public class ResourceMeterMaintainer extends Maintainer { @Override protected void maintain() { - Collection<ResourceSnapshot> resourceSnapshots = getResourceSnapshots(allocatedNodes()); + + Collection<ResourceSnapshot> resourceSnapshots = getAllResourceSnapshots(); meteringClient.consume(resourceSnapshots); metric.set(METERING_LAST_REPORTED, clock.millis() / 1000, metric.createContext(Collections.emptyMap())); // total metered resource usage, for alerting on drastic changes metric.set(METERING_TOTAL_REPORTED, - resourceSnapshots.stream().mapToDouble(r -> r.getCpuCores() + r.getMemoryGb() + r.getDiskGb()).sum(), + resourceSnapshots.stream() + .mapToDouble(r -> r.getCpuCores() + r.getMemoryGb() + r.getDiskGb()).sum(), metric.createContext(Collections.emptyMap())); } - private List<Node> allocatedNodes() { + private Collection<ResourceSnapshot> getAllResourceSnapshots() { return controller().zoneRegistry().zones() .ofCloud(CloudName.from("aws")) .reachable().zones().stream() - .flatMap(zone -> nodeRepository.list(zone.getId()).stream()) - .filter(node -> node.owner().isPresent()) - .filter(node -> ! node.owner().get().tenant().value().equals("hosted-vespa")) + .map(ZoneApi::getId) + .map(zoneId -> createResourceSnapshotsFromNodes(zoneId, nodeRepository.list(zoneId))) + .flatMap(Collection::stream) .collect(Collectors.toList()); } - private Collection<ResourceSnapshot> getResourceSnapshots(List<Node> nodes) { + private Collection<ResourceSnapshot> createResourceSnapshotsFromNodes(ZoneId zoneId, List<Node> nodes) { return nodes.stream() - .collect(Collectors.groupingBy(node -> node.owner().get(), - Collectors.collectingAndThen(Collectors.toList(), - nodeList -> ResourceSnapshot.from(nodeList, - clock.instant())) - )).values(); + .filter(unlessNodeOwnerIsHostedVespa()) + .collect(Collectors.groupingBy(node -> + node.owner().get(), + Collectors.collectingAndThen(Collectors.toList(), + nodeList -> ResourceSnapshot.from( + nodeList, + clock.instant(), + zoneId)) + )).values(); } + private Predicate<Node> unlessNodeOwnerIsHostedVespa() { + return node -> node.owner().map(owner -> + !owner.tenant().value().equals("hosted-vespa") + ).orElse(false); + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 61fd0b67ec9..e67d5aea45d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -23,7 +23,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; -import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentActivity; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; @@ -88,7 +87,6 @@ public class ApplicationSerializer { private static final String pemDeployKeysField = "pemDeployKeys"; private static final String assignedRotationClusterField = "clusterId"; private static final String assignedRotationRotationField = "rotationId"; - private static final String applicationCertificateField = "applicationCertificate"; // Instance fields private static final String instanceNameField = "instanceName"; @@ -147,13 +145,6 @@ public class ApplicationSerializer { private static final String clusterInfoTypeField = "clusterType"; private static final String clusterInfoHostnamesField = "hostnames"; - // ClusterUtils fields - private static final String clusterUtilsField = "clusterUtils"; - private static final String clusterUtilsCpuField = "cpu"; - private static final String clusterUtilsMemField = "mem"; - private static final String clusterUtilsDiskField = "disk"; - private static final String clusterUtilsDiskBusyField = "diskbusy"; - // Deployment metrics fields private static final String deploymentMetricsField = "metrics"; private static final String deploymentMetricsQPSField = "queriesPerSecond"; @@ -220,7 +211,6 @@ public class ApplicationSerializer { object.setLong(deployTimeField, deployment.at().toEpochMilli()); toSlime(deployment.applicationVersion(), object.setObject(applicationPackageRevisionField)); clusterInfoToSlime(deployment.clusterInfo(), object); - clusterUtilsToSlime(deployment.clusterUtils(), object); deploymentMetricsToSlime(deployment.metrics(), object); deployment.activity().lastQueried().ifPresent(instant -> object.setLong(lastQueriedField, instant.toEpochMilli())); deployment.activity().lastWritten().ifPresent(instant -> object.setLong(lastWrittenField, instant.toEpochMilli())); @@ -262,20 +252,6 @@ public class ApplicationSerializer { } } - private void clusterUtilsToSlime(Map<ClusterSpec.Id, ClusterUtilization> clusters, Cursor object) { - Cursor root = object.setObject(clusterUtilsField); - for (Map.Entry<ClusterSpec.Id, ClusterUtilization> entry : clusters.entrySet()) { - toSlime(entry.getValue(), root.setObject(entry.getKey().value())); - } - } - - private void toSlime(ClusterUtilization utils, Cursor object) { - object.setDouble(clusterUtilsCpuField, utils.getCpu()); - object.setDouble(clusterUtilsMemField, utils.getMemory()); - object.setDouble(clusterUtilsDiskField, utils.getDisk()); - object.setDouble(clusterUtilsDiskBusyField, utils.getDiskBusy()); - } - private void zoneIdToSlime(ZoneId zone, Cursor object) { object.setString(environmentField, zone.environment().value()); object.setString(regionField, zone.region().value()); @@ -425,7 +401,6 @@ public class ApplicationSerializer { applicationVersionFromSlime(deploymentObject.field(applicationPackageRevisionField)), Version.fromString(deploymentObject.field(versionField).asString()), Instant.ofEpochMilli(deploymentObject.field(deployTimeField).asLong()), - Map.of(), clusterInfoMapFromSlime(deploymentObject.field(clusterInfoField)), deploymentMetricsFromSlime(deploymentObject.field(deploymentMetricsField)), DeploymentActivity.create(Serializers.optionalInstant(deploymentObject.field(lastQueriedField)), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 357dbb37b27..dbd52fc6d02 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -6,7 +6,6 @@ import com.google.inject.Inject; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.path.Path; @@ -82,14 +81,15 @@ public class CuratorDb { private static final Path applicationCertificateRoot = root.append("applicationCertificates"); private final StringSetSerializer stringSetSerializer = new StringSetSerializer(); - private final VersionStatusSerializer versionStatusSerializer = new VersionStatusSerializer(); + private final NodeVersionSerializer nodeVersionSerializer = new NodeVersionSerializer(); + private final VersionStatusSerializer versionStatusSerializer = new VersionStatusSerializer(nodeVersionSerializer); private final ControllerVersionSerializer controllerVersionSerializer = new ControllerVersionSerializer(); private final ConfidenceOverrideSerializer confidenceOverrideSerializer = new ConfidenceOverrideSerializer(); private final TenantSerializer tenantSerializer = new TenantSerializer(); private final ApplicationSerializer applicationSerializer = new ApplicationSerializer(); private final RunSerializer runSerializer = new RunSerializer(); private final OsVersionSerializer osVersionSerializer = new OsVersionSerializer(); - private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer(osVersionSerializer); + private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer(osVersionSerializer, nodeVersionSerializer); private final RoutingPolicySerializer routingPolicySerializer = new RoutingPolicySerializer(); private final AuditLogSerializer auditLogSerializer = new AuditLogSerializer(); private final NameServiceQueueSerializer nameServiceQueueSerializer = new NameServiceQueueSerializer(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java new file mode 100644 index 00000000000..4b6e997241d --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java @@ -0,0 +1,78 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.component.Version; +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.vespa.hosted.controller.versions.NodeVersion; +import com.yahoo.vespa.hosted.controller.versions.NodeVersions; + +import java.time.Instant; + +/** + * Serializer for {@link com.yahoo.vespa.hosted.controller.versions.NodeVersion}. + * + * @author mpolden + */ +public class NodeVersionSerializer { + + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + + private static final String hostnameField = "hostname"; + private static final String zoneField = "zone"; + private static final String wantedVersionField = "wantedVersion"; + private static final String changedAtField = "changedAt"; + + // Legacy fields + private static final String environmentField = "environment"; + private static final String regionField = "region"; + + public void nodeVersionsToSlime(NodeVersions nodeVersions, Cursor array) { + for (var nodeVersion : nodeVersions.asMap().values()) { + var nodeVersionObject = array.addObject(); + nodeVersionObject.setString(hostnameField, nodeVersion.hostname().value()); + nodeVersionObject.setString(zoneField, nodeVersion.zone().value()); + nodeVersionObject.setString(wantedVersionField, nodeVersion.wantedVersion().toFullString()); + nodeVersionObject.setLong(changedAtField, nodeVersion.changedAt().toEpochMilli()); + } + } + + public NodeVersions nodeVersionsFromSlime(Inspector array, Version version) { + var nodeVersions = ImmutableMap.<HostName, NodeVersion>builder(); + array.traverse((ArrayTraverser) (i, entry) -> { + var hostname = HostName.from(entry.field(hostnameField).asString()); + var zone = zoneFromSlime(entry); + // TODO(mpolden): Make the following fields non-optional after September 2019 + var wantedVersion = Serializers.optionalString(entry.field(wantedVersionField)) + .map(Version::fromString) + .orElse(Version.emptyVersion); + var changedAt = Serializers.optionalInstant(entry.field(changedAtField)).orElse(Instant.EPOCH); + nodeVersions.put(hostname, new NodeVersion(hostname, zone, version, wantedVersion, changedAt)); + }); + return new NodeVersions(nodeVersions.build()); + } + + // TODO(mpolden): Simplify and in-line after September 2019 + private ZoneId zoneFromSlime(Inspector object) { + var zoneInspector = object.field(zoneField); + if (zoneInspector.valid()) { + return ZoneId.from(zoneInspector.asString()); + } + var regionInspector = object.field(regionField); + var environmentInspector = object.field(environmentField); + if (regionInspector.valid() && environmentInspector.valid()) { + return ZoneId.from(environmentInspector.asString(), regionInspector.asString()); + } + return ZoneId.defaultId(); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java index 88805f54d65..fa29969f166 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java @@ -1,23 +1,19 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSortedMap; import com.yahoo.component.Version; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.RegionName; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.versions.NodeVersion; +import com.yahoo.vespa.hosted.controller.versions.NodeVersions; import com.yahoo.vespa.hosted.controller.versions.OsVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; import java.util.Objects; -import java.util.TreeMap; /** * Serializer for {@link OsVersionStatus}. @@ -39,11 +35,14 @@ public class OsVersionStatusSerializer { private static final String hostnameField = "hostname"; private static final String regionField = "region"; private static final String environmentField = "environment"; + private static final String nodeVersionsField = "nodeVersions"; private final OsVersionSerializer osVersionSerializer; + private final NodeVersionSerializer nodeVersionSerializer; - public OsVersionStatusSerializer(OsVersionSerializer osVersionSerializer) { + public OsVersionStatusSerializer(OsVersionSerializer osVersionSerializer, NodeVersionSerializer nodeVersionSerializer) { this.osVersionSerializer = Objects.requireNonNull(osVersionSerializer, "osVersionSerializer must be non-null"); + this.nodeVersionSerializer = Objects.requireNonNull(nodeVersionSerializer, "nodeVersionSerializer must be non-null"); } public Slime toSlime(OsVersionStatus status) { @@ -53,6 +52,8 @@ public class OsVersionStatusSerializer { status.versions().forEach((version, nodes) -> { Cursor object = versions.addObject(); osVersionSerializer.toSlime(version, object); + nodeVersionSerializer.nodeVersionsToSlime(nodes, object.setArray(nodeVersionsField)); + // TODO(mpolden): Stop writing this after September 2019 nodesToSlime(nodes, object.setArray(nodesField)); }); return slime; @@ -62,40 +63,33 @@ public class OsVersionStatusSerializer { return new OsVersionStatus(osVersionsFromSlime(slime.get().field(versionsField))); } - private void nodesToSlime(List<OsVersionStatus.Node> nodes, Cursor array) { - nodes.forEach(node -> nodeToSlime(node, array.addObject())); + private void nodesToSlime(NodeVersions nodeVersions, Cursor array) { + nodeVersions.asMap().values().forEach(node -> nodeToSlime(node, array.addObject())); } - private void nodeToSlime(OsVersionStatus.Node node, Cursor object) { + private void nodeToSlime(NodeVersion node, Cursor object) { object.setString(hostnameField, node.hostname().value()); - object.setString(versionField, node.version().toFullString()); - object.setString(regionField, node.region().value()); - object.setString(environmentField, node.environment().value()); + object.setString(versionField, node.currentVersion().toFullString()); + object.setString(regionField, node.zone().region().value()); + object.setString(environmentField, node.zone().environment().value()); } - private Map<OsVersion, List<OsVersionStatus.Node>> osVersionsFromSlime(Inspector array) { - Map<OsVersion, List<OsVersionStatus.Node>> versions = new TreeMap<>(); + private ImmutableMap<OsVersion, NodeVersions> osVersionsFromSlime(Inspector array) { + var versions = ImmutableSortedMap.<OsVersion, NodeVersions>naturalOrder(); array.traverse((ArrayTraverser) (i, object) -> { OsVersion osVersion = osVersionSerializer.fromSlime(object); - List<OsVersionStatus.Node> nodes = nodesFromSlime(object.field(nodesField)); - versions.put(osVersion, nodes); + versions.put(osVersion, nodesFromSlime(object, osVersion.version())); }); - return Collections.unmodifiableMap(versions); + return versions.build(); } - private List<OsVersionStatus.Node> nodesFromSlime(Inspector array) { - List<OsVersionStatus.Node> nodes = new ArrayList<>(); - array.traverse((ArrayTraverser) (i, object) -> nodes.add(nodeFromSlime(object))); - return Collections.unmodifiableList(nodes); - } - - private OsVersionStatus.Node nodeFromSlime(Inspector object) { - return new OsVersionStatus.Node( - HostName.from(object.field(hostnameField).asString()), - Version.fromString(object.field(versionField).asString()), - Environment.from(object.field(environmentField).asString()), - RegionName.from(object.field(regionField).asString()) - ); + // TODO(mpolden): Simplify and in-line after September 2019 + private NodeVersions nodesFromSlime(Inspector object, Version version) { + var newField = object.field(nodeVersionsField); + if (newField.valid()) { + return nodeVersionSerializer.nodeVersionsFromSlime(newField, version); + } + return nodeVersionSerializer.nodeVersionsFromSlime(object.field(nodesField), version); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java index 5061f32da68..366e2c9af4b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java @@ -1,16 +1,13 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; -import com.google.common.collect.ImmutableMap; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics; -import com.yahoo.vespa.hosted.controller.versions.NodeVersion; import com.yahoo.vespa.hosted.controller.versions.NodeVersions; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; @@ -19,9 +16,8 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.LinkedHashSet; import java.util.List; -import java.util.Set; +import java.util.Objects; /** * Serializer for {@link VersionStatus}. @@ -53,17 +49,18 @@ public class VersionStatusSerializer { // NodeVersions fields private static final String nodeVersionsField = "nodeVersions"; - // NodeVersion fields - private static final String hostnameField = "hostname"; - private static final String wantedVersionField = "wantedVersion"; - private static final String changedAtField = "changedAt"; - // DeploymentStatistics fields private static final String versionField = "version"; private static final String failingField = "failing"; private static final String productionField = "production"; private static final String deployingField = "deploying"; + private final NodeVersionSerializer nodeVersionSerializer; + + public VersionStatusSerializer(NodeVersionSerializer nodeVersionSerializer) { + this.nodeVersionSerializer = Objects.requireNonNull(nodeVersionSerializer, "nodeVersionSerializer must be non-null"); + } + public Slime toSlime(VersionStatus status) { Slime slime = new Slime(); Cursor root = slime.setObject(); @@ -88,22 +85,11 @@ public class VersionStatusSerializer { object.setBool(isReleasedField, version.isReleased()); deploymentStatisticsToSlime(version.statistics(), object.setObject(deploymentStatisticsField)); object.setString(confidenceField, version.confidence().name()); - configServersToSlime(version.nodeVersions().hostnames(), object.setArray(configServersField)); nodeVersionsToSlime(version.nodeVersions(), object.setArray(nodeVersionsField)); } private void nodeVersionsToSlime(NodeVersions nodeVersions, Cursor array) { - for (NodeVersion nodeVersion : nodeVersions.asMap().values()) { - var nodeVersionObject = array.addObject(); - nodeVersionObject.setString(hostnameField, nodeVersion.hostname().value()); - nodeVersionObject.setString(wantedVersionField, nodeVersion.wantedVersion().toFullString()); - nodeVersionObject.setLong(changedAtField, nodeVersion.changedAt().toEpochMilli()); - } - } - - // TODO(mpolden): Remove after October 2019 - private void configServersToSlime(Set<HostName> configServerHostnames, Cursor array) { - configServerHostnames.stream().map(HostName::value).forEach(array::addString); + nodeVersionSerializer.nodeVersionsToSlime(nodeVersions, array); } private void deploymentStatisticsToSlime(DeploymentStatistics statistics, Cursor object) { @@ -131,37 +117,11 @@ public class VersionStatusSerializer { object.field(isControllerVersionField).asBool(), object.field(isSystemVersionField).asBool(), object.field(isReleasedField).asBool(), - nodeVersionsFromSlime(object, deploymentStatistics.version()), + nodeVersionSerializer.nodeVersionsFromSlime(object.field(nodeVersionsField), deploymentStatistics.version()), VespaVersion.Confidence.valueOf(object.field(confidenceField).asString()) ); } - private NodeVersions nodeVersionsFromSlime(Inspector root, Version version) { - var nodeVersions = ImmutableMap.<HostName, NodeVersion>builder(); - var nodeVersionsRoot = root.field(nodeVersionsField); - if (nodeVersionsRoot.valid()) { - nodeVersionsRoot.traverse((ArrayTraverser) (i, entry) -> { - var hostname = HostName.from(entry.field(hostnameField).asString()); - var wantedVersion = Version.fromString(entry.field(wantedVersionField).asString()); - var changedAt = Instant.ofEpochMilli(entry.field(changedAtField).asLong()); - nodeVersions.put(hostname, new NodeVersion(hostname, version, wantedVersion, changedAt)); - }); - } else { - // TODO(mpolden): Remove after October 2019 - var configServerHostnames = configServersFromSlime(root.field(configServersField)); - for (var hostname : configServerHostnames) { - nodeVersions.put(hostname, NodeVersion.empty(hostname)); - } - } - return new NodeVersions(nodeVersions.build()); - } - - private Set<HostName> configServersFromSlime(Inspector array) { - Set<HostName> configServerHostnames = new LinkedHashSet<>(); - array.traverse((ArrayTraverser) (i, entry) -> configServerHostnames.add(HostName.from(entry.asString()))); - return Collections.unmodifiableSet(configServerHostnames); - } - private DeploymentStatistics deploymentStatisticsFromSlime(Inspector object) { return new DeploymentStatistics(Version.fromString(object.field(versionField).asString()), applicationsFromSlime(object.field(failingField)), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index b76d0ae1094..c37309b87ad 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -102,7 +102,6 @@ import java.time.Duration; import java.time.Instant; import java.util.Arrays; import java.util.Base64; -import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -381,9 +380,13 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Principal user = request.getJDiscRequest().getUserPrincipal(); String pemDeveloperKey = toSlime(request.getData()).get().field("key").asString(); PublicKey developerKey = KeyUtils.fromPemEncodedPublicKey(pemDeveloperKey); - controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, tenant -> - controller.tenants().store(tenant.withDeveloperKey(developerKey, user))); - return new MessageResponse("Set developer key " + pemDeveloperKey + " for " + user); + Slime root = new Slime(); + controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, tenant -> { + tenant = tenant.withDeveloperKey(developerKey, user); + toSlime(root.setObject().setArray("keys"), tenant.get().developerKeys()); + controller.tenants().store(tenant); + }); + return new SlimeJsonResponse(root); } private HttpResponse removeDeveloperKey(String tenantName, HttpRequest request) { @@ -393,27 +396,49 @@ public class ApplicationApiHandler extends LoggingRequestHandler { String pemDeveloperKey = toSlime(request.getData()).get().field("key").asString(); PublicKey developerKey = KeyUtils.fromPemEncodedPublicKey(pemDeveloperKey); Principal user = ((CloudTenant) controller.tenants().require(TenantName.from(tenantName))).developerKeys().get(developerKey); - controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, tenant -> - controller.tenants().store(tenant.withoutDeveloperKey(developerKey))); - return new MessageResponse("Removed developer key " + pemDeveloperKey + " for " + user); + Slime root = new Slime(); + controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, tenant -> { + tenant = tenant.withoutDeveloperKey(developerKey); + toSlime(root.setObject().setArray("keys"), tenant.get().developerKeys()); + controller.tenants().store(tenant); + }); + return new SlimeJsonResponse(root); + } + + private void toSlime(Cursor keysArray, Map<PublicKey, Principal> keys) { + keys.forEach((key, principal) -> { + Cursor keyObject = keysArray.addObject(); + keyObject.setString("key", KeyUtils.toPem(key)); + keyObject.setString("user", principal.getName()); + }); } private HttpResponse addDeployKey(String tenantName, String applicationName, HttpRequest request) { String pemDeployKey = toSlime(request.getData()).get().field("key").asString(); PublicKey deployKey = KeyUtils.fromPemEncodedPublicKey(pemDeployKey); - controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(tenantName, applicationName), application -> - controller.applications().store(application.withDeployKey(deployKey))); - - return new MessageResponse("Added deploy key " + pemDeployKey); + Slime root = new Slime(); + controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(tenantName, applicationName), application -> { + application = application.withDeployKey(deployKey); + application.get().deployKeys().stream() + .map(KeyUtils::toPem) + .forEach(root.setObject().setArray("keys")::addString); + controller.applications().store(application); + }); + return new SlimeJsonResponse(root); } private HttpResponse removeDeployKey(String tenantName, String applicationName, HttpRequest request) { String pemDeployKey = toSlime(request.getData()).get().field("key").asString(); PublicKey deployKey = KeyUtils.fromPemEncodedPublicKey(pemDeployKey); - controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(tenantName, applicationName), application -> - controller.applications().store(application.withoutDeployKey(deployKey))); - - return new MessageResponse("Removed deploy key " + pemDeployKey); + Slime root = new Slime(); + controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(tenantName, applicationName), application -> { + application = application.withoutDeployKey(deployKey); + application.get().deployKeys().stream() + .map(KeyUtils::toPem) + .forEach(root.setObject().setArray("keys")::addString); + controller.applications().store(application); + }); + return new SlimeJsonResponse(root); } private HttpResponse patchApplication(String tenantName, String applicationName, HttpRequest request) { @@ -752,7 +777,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { deployment.activity().lastWritesPerSecond().ifPresent(value -> activity.setDouble("lastWritesPerSecond", value)); // Cost - DeploymentCost appCost = deployment.calculateCost(); + DeploymentCost appCost = new DeploymentCost(Map.of()); Cursor costObject = response.setObject("cost"); toSlime(appCost, costObject); @@ -1321,7 +1346,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(testConfigSerializer.configSlime(id, type, controller.applications().clusterEndpoints(id, zones), - controller.applications().listClusters(id, zones))); + controller.applications().contentClustersByZone(id, zones))); } private static DeploymentJobs.JobReport toJobReport(String tenantName, String applicationName, Inspector report) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java index 450f4481c5f..c168a057bfb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java @@ -160,17 +160,17 @@ public class OsApiHandler extends AuditLoggingRequestHandler { Set<OsVersion> osVersions = controller.osVersions(); Cursor versions = root.setArray("versions"); - controller.osVersionStatus().versions().forEach((osVersion, nodes) -> { + controller.osVersionStatus().versions().forEach((osVersion, nodeVersions) -> { Cursor currentVersionObject = versions.addObject(); currentVersionObject.setString("version", osVersion.version().toFullString()); currentVersionObject.setBool("targetVersion", osVersions.contains(osVersion)); currentVersionObject.setString("cloud", osVersion.cloud().value()); Cursor nodesArray = currentVersionObject.setArray("nodes"); - nodes.forEach(node -> { + nodeVersions.asMap().values().forEach(nodeVersion -> { Cursor nodeObject = nodesArray.addObject(); - nodeObject.setString("hostname", node.hostname().value()); - nodeObject.setString("environment", node.environment().value()); - nodeObject.setString("region", node.region().value()); + nodeObject.setString("hostname", nodeVersion.hostname().value()); + nodeObject.setString("environment", nodeVersion.zone().environment().value()); + nodeObject.setString("region", nodeVersion.zone().region().value()); }); }); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java index a16ca5cb201..9f6bbcd2a5a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java @@ -77,22 +77,25 @@ public class RotationRepository { * If a rotation is already assigned to the application, that rotation will be returned. * If no rotation is assigned, return an available rotation. The caller is responsible for assigning the rotation. * - * @param deploymentSpec The deployment spec for the application - * @param instance The instance requesting a rotation - * @param lock Lock which must be acquired by the caller + * @param deploymentSpec the deployment spec for the application + * @param instance the instance requesting a rotation + * @param lock lock which must be acquired by the caller */ public Rotation getOrAssignRotation(DeploymentSpec deploymentSpec, Instance instance, RotationLock lock) { if ( ! instance.rotations().isEmpty()) { return allRotations.get(instance.rotations().get(0).rotationId()); } - if (deploymentSpec.globalServiceId().isEmpty()) { - throw new IllegalArgumentException("global-service-id is not set in deployment spec"); + + if (deploymentSpec.requireInstance(instance.name()).globalServiceId().isEmpty()) { + throw new IllegalArgumentException("global-service-id is not set in deployment spec for instance '" + + instance.name() + "'"); } - long productionZones = deploymentSpec.zones().stream() - .filter(zone -> zone.deploysTo(Environment.prod)) - .count(); + long productionZones = deploymentSpec.requireInstance(instance.name()).zones().stream() + .filter(zone -> zone.deploysTo(Environment.prod)) + .count(); if (productionZones < 2) { - throw new IllegalArgumentException("global-service-id is set but less than 2 prod zones are defined"); + throw new IllegalArgumentException("global-service-id is set but less than 2 prod zones are defined " + + "in instance '" + instance.name() + "'"); } return findAvailableRotation(instance.id(), lock); } @@ -110,22 +113,23 @@ public class RotationRepository { * @return List of rotation assignments - either new or existing */ public List<AssignedRotation> getOrAssignRotations(DeploymentSpec deploymentSpec, Instance instance, RotationLock lock) { - if (deploymentSpec.globalServiceId().isPresent() && ! deploymentSpec.endpoints().isEmpty()) { + if (deploymentSpec.requireInstance(instance.name()).globalServiceId().isPresent() + && ! deploymentSpec.requireInstance(instance.name()).endpoints().isEmpty()) { throw new IllegalArgumentException("Cannot provision rotations with both global-service-id and 'endpoints'"); } // Support the older case of setting global-service-id - if (deploymentSpec.globalServiceId().isPresent()) { - final var regions = deploymentSpec.zones().stream() - .filter(zone -> zone.environment().isProduction()) - .flatMap(zone -> zone.region().stream()) - .collect(Collectors.toSet()); + if (deploymentSpec.requireInstance(instance.name()).globalServiceId().isPresent()) { + var regions = deploymentSpec.requireInstance(instance.name()).zones().stream() + .filter(zone -> zone.environment().isProduction()) + .flatMap(zone -> zone.region().stream()) + .collect(Collectors.toSet()); - final var rotation = getOrAssignRotation(deploymentSpec, instance, lock); + var rotation = getOrAssignRotation(deploymentSpec, instance, lock); return List.of( new AssignedRotation( - new ClusterSpec.Id(deploymentSpec.globalServiceId().get()), + new ClusterSpec.Id(deploymentSpec.requireInstance(instance.name()).globalServiceId().get()), EndpointId.default_(), rotation.id(), regions @@ -133,8 +137,8 @@ public class RotationRepository { ); } - final Map<EndpointId, AssignedRotation> existingAssignments = existingEndpointAssignments(deploymentSpec, instance); - final Map<EndpointId, AssignedRotation> updatedAssignments = assignRotationsToEndpoints(deploymentSpec, existingAssignments, lock); + Map<EndpointId, AssignedRotation> existingAssignments = existingEndpointAssignments(deploymentSpec, instance); + Map<EndpointId, AssignedRotation> updatedAssignments = assignRotationsToEndpoints(deploymentSpec, existingAssignments, lock); existingAssignments.putAll(updatedAssignments); @@ -142,11 +146,11 @@ public class RotationRepository { } private Map<EndpointId, AssignedRotation> assignRotationsToEndpoints(DeploymentSpec deploymentSpec, Map<EndpointId, AssignedRotation> existingAssignments, RotationLock lock) { - final var availableRotations = new ArrayList<>(availableRotations(lock).values()); + var availableRotations = new ArrayList<>(availableRotations(lock).values()); - final var neededRotations = deploymentSpec.endpoints().stream() - .filter(Predicate.not(endpoint -> existingAssignments.containsKey(EndpointId.of(endpoint.endpointId())))) - .collect(Collectors.toSet()); + var neededRotations = deploymentSpec.endpoints().stream() + .filter(Predicate.not(endpoint -> existingAssignments.containsKey(EndpointId.of(endpoint.endpointId())))) + .collect(Collectors.toSet()); if (neededRotations.size() > availableRotations.size()) { throw new IllegalStateException("Hosted Vespa ran out of rotations, unable to assign rotation: need " + neededRotations.size() + ", have " + availableRotations.size()); @@ -172,34 +176,26 @@ public class RotationRepository { } private Map<EndpointId, AssignedRotation> existingEndpointAssignments(DeploymentSpec deploymentSpec, Instance instance) { - // // Get the regions that has been configured for an endpoint. Empty set if the endpoint // is no longer mentioned in the configuration file. - // - final Function<EndpointId, Set<RegionName>> configuredRegionsForEndpoint = endpointId -> { - return deploymentSpec.endpoints().stream() + Function<EndpointId, Set<RegionName>> configuredRegionsForEndpoint = endpointId -> + deploymentSpec.requireInstance(instance.name()).endpoints().stream() .filter(endpoint -> endpointId.id().equals(endpoint.endpointId())) .map(Endpoint::regions) .findFirst() .orElse(Set.of()); - }; - // // Build a new AssignedRotation instance where we update set of regions from the configuration instead - // of using the one already mentioned in the assignment. This allows us to overwrite the set of regions - // when - final Function<AssignedRotation, AssignedRotation> assignedRotationWithConfiguredRegions = assignedRotation -> { - return new AssignedRotation( + // of using the one already mentioned in the assignment. This allows us to overwrite the set of regions. + Function<AssignedRotation, AssignedRotation> assignedRotationWithConfiguredRegions = assignedRotation -> + new AssignedRotation( assignedRotation.clusterId(), assignedRotation.endpointId(), assignedRotation.rotationId(), - configuredRegionsForEndpoint.apply(assignedRotation.endpointId()) - ); - }; + configuredRegionsForEndpoint.apply(assignedRotation.endpointId())); return instance.rotations().stream() - .collect( - Collectors.toMap( + .collect(Collectors.toMap( AssignedRotation::endpointId, assignedRotationWithConfiguredRegions, (a, b) -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java index 0a690b90410..8d0232afa58 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.versions; import com.yahoo.component.Version; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.zone.ZoneId; import java.time.Instant; import java.util.Objects; @@ -17,12 +18,14 @@ import java.util.Objects; public class NodeVersion { private final HostName hostname; + private final ZoneId zone; private final Version currentVersion; private final Version wantedVersion; private final Instant changedAt; - public NodeVersion(HostName hostname, Version currentVersion, Version wantedVersion, Instant changedAt) { + public NodeVersion(HostName hostname, ZoneId zone, Version currentVersion, Version wantedVersion, Instant changedAt) { this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); + this.zone = Objects.requireNonNull(zone, "zone must be non-null"); this.currentVersion = Objects.requireNonNull(currentVersion, "version must be non-null"); this.wantedVersion = Objects.requireNonNull(wantedVersion, "wantedVersion must be non-null"); this.changedAt = Objects.requireNonNull(changedAt, "changedAt must be non-null"); @@ -33,6 +36,11 @@ public class NodeVersion { return hostname; } + /** Zone of this */ + public ZoneId zone() { + return zone; + } + /** Current version of this */ public Version currentVersion() { return currentVersion; @@ -56,18 +64,18 @@ public class NodeVersion { /** Returns a copy of this with current version set to given version */ public NodeVersion withCurrentVersion(Version version, Instant changedAt) { if (currentVersion.equals(version)) return this; - return new NodeVersion(hostname, version, wantedVersion, changedAt); + return new NodeVersion(hostname, zone, version, wantedVersion, changedAt); } /** Returns a copy of this with wanted version set to given version */ public NodeVersion withWantedVersion(Version version) { if (wantedVersion.equals(version)) return this; - return new NodeVersion(hostname, currentVersion, version, changedAt); + return new NodeVersion(hostname, zone, currentVersion, version, changedAt); } @Override public String toString() { - return hostname + ": " + currentVersion + " -> " + wantedVersion + " [changedAt=" + changedAt + "]"; + return hostname + ": " + currentVersion + " -> " + wantedVersion + " [zone=" + zone + ", changedAt=" + changedAt + "]"; } @Override @@ -76,6 +84,7 @@ public class NodeVersion { if (o == null || getClass() != o.getClass()) return false; NodeVersion that = (NodeVersion) o; return hostname.equals(that.hostname) && + zone.equals(that.zone) && currentVersion.equals(that.currentVersion) && wantedVersion.equals(that.wantedVersion) && changedAt.equals(that.changedAt); @@ -83,11 +92,11 @@ public class NodeVersion { @Override public int hashCode() { - return Objects.hash(hostname, currentVersion, wantedVersion, changedAt); + return Objects.hash(hostname, zone, currentVersion, wantedVersion, changedAt); } public static NodeVersion empty(HostName hostname) { - return new NodeVersion(hostname, Version.emptyVersion, Version.emptyVersion, Instant.EPOCH); + return new NodeVersion(hostname, ZoneId.defaultId(), Version.emptyVersion, Version.emptyVersion, Instant.EPOCH); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java index a73a20198f0..d5e83d99cdd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java @@ -4,9 +4,7 @@ package com.yahoo.vespa.hosted.controller.versions; import com.google.common.collect.ImmutableMap; import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.SystemApplication; @@ -14,11 +12,11 @@ import com.yahoo.vespa.hosted.controller.maintenance.OsUpgrader; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -29,25 +27,25 @@ import java.util.stream.Collectors; */ public class OsVersionStatus { - public static final OsVersionStatus empty = new OsVersionStatus(Collections.emptyMap()); + public static final OsVersionStatus empty = new OsVersionStatus(ImmutableMap.of()); - private final Map<OsVersion, List<Node>> versions; + private final Map<OsVersion, NodeVersions> versions; /** Public for serialization purpose only. Use {@link OsVersionStatus#compute(Controller)} for an up-to-date status */ - public OsVersionStatus(Map<OsVersion, List<Node>> versions) { + public OsVersionStatus(ImmutableMap<OsVersion, NodeVersions> versions) { this.versions = ImmutableMap.copyOf(Objects.requireNonNull(versions, "versions must be non-null")); } /** All known OS versions and their nodes */ - public Map<OsVersion, List<Node>> versions() { + public Map<OsVersion, NodeVersions> versions() { return versions; } /** Returns nodes eligible for OS upgrades that exist in given cloud */ - public List<Node> nodesIn(CloudName cloud) { + public List<NodeVersion> nodesIn(CloudName cloud) { return versions.entrySet().stream() .filter(entry -> entry.getKey().cloud().equals(cloud)) - .flatMap(entry -> entry.getValue().stream()) + .flatMap(entry -> entry.getValue().asMap().values().stream()) .collect(Collectors.toUnmodifiableList()); } @@ -61,28 +59,52 @@ public class OsVersionStatus { /** Compute the current OS versions in this system. This is expensive and should be called infrequently */ public static OsVersionStatus compute(Controller controller) { - Map<OsVersion, List<Node>> versions = new HashMap<>(); - - // Always include all target versions - controller.osVersions().forEach(osVersion -> versions.put(osVersion, new ArrayList<>())); - - for (SystemApplication application : SystemApplication.all()) { - if (!application.isEligibleForOsUpgrades()) { - continue; // Avoid querying applications that are not eligible for OS upgrades - } - for (ZoneApi zone : zonesToUpgrade(controller)) { - controller.serviceRegistry().configServer().nodeRepository().list(zone.getId(), application.id()).stream() + var osVersionStatus = controller.osVersionStatus(); + var osVersions = new HashMap<OsVersion, List<NodeVersion>>(); + var now = controller.clock().instant(); + controller.osVersions().forEach(osVersion -> osVersions.put(osVersion, new ArrayList<>())); + + for (var application : SystemApplication.all()) { + if (!application.isEligibleForOsUpgrades()) continue; + for (var zone : zonesToUpgrade(controller)) { + var targetOsVersion = controller.serviceRegistry().configServer().nodeRepository() + .targetVersionsOf(zone.getId()) + .osVersion(application.nodeType()) + .orElse(Version.emptyVersion); + controller.serviceRegistry().configServer().nodeRepository() + .list(zone.getId(), application.id()).stream() .filter(node -> OsUpgrader.eligibleForUpgrade(node, application)) - .map(node -> new Node(node.hostname(), node.currentOsVersion(), zone.getEnvironment(), zone.getRegionName())) - .forEach(node -> { - var version = new OsVersion(node.version(), zone.getCloudName()); - versions.putIfAbsent(version, new ArrayList<>()); - versions.get(version).add(node); + .map(node -> new NodeVersion(node.hostname(), zone.getId(), node.currentOsVersion(), targetOsVersion, now)) + .forEach(nodeVersion -> { + var newNodeVersion = osVersionStatus.of(nodeVersion.hostname()) + .map(nv -> nv.withCurrentVersion(nodeVersion.currentVersion(), now) + .withWantedVersion(nodeVersion.wantedVersion())) + .orElse(nodeVersion); + var version = new OsVersion(newNodeVersion.currentVersion(), zone.getCloudName()); + osVersions.putIfAbsent(version, new ArrayList<>()); + osVersions.get(version).add(newNodeVersion); }); } } - return new OsVersionStatus(versions); + var newOsVersions = ImmutableMap.<OsVersion, NodeVersions>builder(); + for (var osVersion : osVersions.entrySet()) { + var nodeVersions = ImmutableMap.<HostName, NodeVersion>builder(); + for (var nodeVersion : osVersion.getValue()) { + nodeVersions.put(nodeVersion.hostname(), nodeVersion); + } + newOsVersions.put(osVersion.getKey(), new NodeVersions(nodeVersions.build())); + } + return new OsVersionStatus(newOsVersions.build()); + } + + /** Returns version of node identified by given host name */ + private Optional<NodeVersion> of(HostName hostname) { + return versions.values().stream() + .map(nodeVersions -> nodeVersions.asMap().get(hostname)) + .map(Optional::ofNullable) + .flatMap(Optional::stream) + .findFirst(); } private static List<ZoneApi> zonesToUpgrade(Controller controller) { @@ -92,52 +114,4 @@ public class OsVersionStatus { .collect(Collectors.toUnmodifiableList()); } - /** A node in this system and its current OS version */ - public static class Node { - - private final HostName hostname; - private final Version version; - private final Environment environment; - private final RegionName region; - - public Node(HostName hostname, Version version, Environment environment, RegionName region) { - this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); - this.version = Objects.requireNonNull(version, "version must be non-null"); - this.environment = Objects.requireNonNull(environment, "environment must be non-null"); - this.region = Objects.requireNonNull(region, "region must be non-null"); - } - - public HostName hostname() { - return hostname; - } - - public Version version() { - return version; - } - - public Environment environment() { - return environment; - } - - public RegionName region() { - return region; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Node node = (Node) o; - return Objects.equals(hostname, node.hostname) && - Objects.equals(version, node.version) && - environment == node.environment && - Objects.equals(region, node.region); - } - - @Override - public int hashCode() { - return Objects.hash(hostname, version, environment, region); - } - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index bb43ec20234..ab445de5a7f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -172,7 +172,7 @@ public class VersionStatus { for (var node : nodes) { // Only use current node version if config has converged Version version = configConverged ? node.currentVersion() : controller.systemVersion(); - newNodeVersions.add(new NodeVersion(node.hostname(), version, node.wantedVersion(), now)); + newNodeVersions.add(new NodeVersion(node.hostname(), zone.getId(), version, node.wantedVersion(), now)); } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index ebf80eb9daa..e3682a78b7d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -72,7 +72,6 @@ public class ControllerTest { @Test public void testDeployment() { // Setup system - ApplicationController applications = tester.controller().applications(); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) .region("us-west-1") @@ -753,7 +752,7 @@ public class ControllerTest { tester.deployCompletely(application, applicationPackage); fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("Endpoint 'default' cannot contain regions in different clouds: [aws-us-east-1, us-west-1]", e.getMessage()); + assertEquals("Endpoint 'default' in instance 'default' cannot contain regions in different clouds: [aws-us-east-1, us-west-1]", e.getMessage()); } var applicationPackage2 = new ApplicationPackageBuilder() @@ -766,7 +765,7 @@ public class ControllerTest { tester.deployCompletely(application, applicationPackage2); fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("Endpoint 'foo' cannot contain regions in different clouds: [aws-us-east-1, us-west-1]", e.getMessage()); + assertEquals("Endpoint 'foo' in instance 'default' cannot contain regions in different clouds: [aws-us-east-1, us-west-1]", e.getMessage()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index 25e562ed046..9449f2b0854 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -47,6 +47,7 @@ public class ApplicationPackageBuilder { private final List<X509Certificate> trustedCertificates = new ArrayList<>(); private OptionalInt majorVersion = OptionalInt.empty(); + private String instances = "default"; private String upgradePolicy = null; private Environment environment = Environment.prod; private String globalServiceId = null; @@ -58,6 +59,11 @@ public class ApplicationPackageBuilder { return this; } + public ApplicationPackageBuilder instances(String instances) { + this.instances = instances; + return this; + } + public ApplicationPackageBuilder upgradePolicy(String upgradePolicy) { this.upgradePolicy = upgradePolicy; return this; @@ -90,7 +96,7 @@ public class ApplicationPackageBuilder { } public ApplicationPackageBuilder region(String regionName) { - environmentBody.append(" <region active='true'>"); + environmentBody.append(" <region active='true'>"); environmentBody.append(regionName); environmentBody.append("</region>\n"); return this; @@ -112,7 +118,7 @@ public class ApplicationPackageBuilder { public ApplicationPackageBuilder blockChange(boolean revision, boolean version, String daySpec, String hourSpec, String zoneSpec) { - blockChange.append(" <block-change"); + blockChange.append(" <block-change"); blockChange.append(" revision='").append(revision).append("'"); blockChange.append(" version='").append(version).append("'"); blockChange.append(" days='").append(daySpec).append("'"); @@ -166,14 +172,15 @@ public class ApplicationPackageBuilder { xml.append(athenzIdentityAttributes); } xml.append(">\n"); + xml.append(" <instance id='").append(instances).append("'>\n"); if (upgradePolicy != null) { - xml.append("<upgrade policy='"); + xml.append(" <upgrade policy='"); xml.append(upgradePolicy); xml.append("'/>\n"); } xml.append(notifications); xml.append(blockChange); - xml.append(" <"); + xml.append(" <"); xml.append(environment.value()); if (globalServiceId != null) { xml.append(" global-service-id='"); @@ -182,13 +189,14 @@ public class ApplicationPackageBuilder { } xml.append(">\n"); xml.append(environmentBody); - xml.append(" </"); + xml.append(" </"); xml.append(environment.value()); xml.append(">\n"); - xml.append(" <endpoints>\n"); + xml.append(" <endpoints>\n"); xml.append(endpointsBody); - xml.append(" </endpoints>\n"); - xml.append("</deployment>"); + xml.append(" </endpoints>\n"); + xml.append(" </instance>\n"); + xml.append("</deployment>\n"); return xml.toString().getBytes(UTF_8); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 6da77a967f1..6e7a50b5f81 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -151,15 +151,44 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer /** Set version for an application in a given zone */ public void setVersion(ApplicationId application, ZoneId zone, Version version) { - setVersion(application, zone, version, -1); + setVersion(application, zone, version, -1, false); } /** Set version for nodeCount number of nodes in application in a given zone */ public void setVersion(ApplicationId application, ZoneId zone, Version version, int nodeCount) { + setVersion(application, zone, version, nodeCount, false); + } + + /** Set OS version for an application in a given zone */ + public void setOsVersion(ApplicationId application, ZoneId zone, Version version) { + setOsVersion(application, zone, version, -1); + } + + /** Set OS version for an application in a given zone */ + public void setOsVersion(ApplicationId application, ZoneId zone, Version version, int nodeCount) { + setVersion(application, zone, version, nodeCount, true); + } + + private void setVersion(ApplicationId application, ZoneId zone, Version version, int nodeCount, boolean osVersion) { int n = 0; for (Node node : nodeRepository().list(zone, application)) { - nodeRepository().putByHostname(zone, new Node(node.hostname(), node.state(), node.type(), node.owner(), - version, version)); + Node newNode; + if (osVersion) { + newNode = new Node(node.hostname(), node.state(), node.type(), node.owner(), node.currentVersion(), + node.wantedVersion(), version, version, node.serviceState(), + node.restartGeneration(), node.wantedRestartGeneration(), node.rebootGeneration(), + node.wantedRebootGeneration(), node.vcpu(), node.memoryGb(), node.diskGb(), + node.bandwidthGbps(), node.fastDisk(), node.cost(), node.canonicalFlavor(), + node.clusterId(), node.clusterType()); + } else { + newNode = new Node(node.hostname(), node.state(), node.type(), node.owner(), version, + version, node.currentOsVersion(), node.wantedOsVersion(), node.serviceState(), + node.restartGeneration(), node.wantedRestartGeneration(), node.rebootGeneration(), + node.wantedRebootGeneration(), node.vcpu(), node.memoryGb(), node.diskGb(), + node.bandwidthGbps(), node.fastDisk(), node.cost(), node.canonicalFlavor(), + node.clusterId(), node.clusterType()); + } + nodeRepository().putByHostname(zone, newNode); if (++n == nodeCount) break; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 9cb40d60677..44785407874 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneId; @@ -262,6 +263,57 @@ public class MetricsReporterTest { } } + @Test + public void test_nodes_failing_os_upgrade() { + var tester = new DeploymentTester(); + var reporter = createReporter(tester.controller()); + var zone = ZoneApiMock.fromId("prod.eu-west-1"); + var cloud = CloudName.defaultName(); + tester.controllerTester().zoneRegistry().setOsUpgradePolicy(cloud, UpgradePolicy.create().upgrade(zone)); + var osUpgrader = new OsUpgrader(tester.controller(), Duration.ofDays(1), + new JobControl(tester.controllerTester().curator()), CloudName.defaultName());; + var statusUpdater = new OsVersionStatusUpdater(tester.controller(), Duration.ofDays(1), + new JobControl(tester.controller().curator())); + tester.configServer().bootstrap(List.of(zone.getId()), SystemApplication.tenantHost); + + // All nodes upgrade to initial OS version + var version0 = Version.fromString("8.0"); + tester.controller().upgradeOsIn(cloud, version0, false); + osUpgrader.maintain(); + tester.configServer().setOsVersion(SystemApplication.tenantHost.id(), zone.getId(), version0); + statusUpdater.maintain(); + reporter.maintain(); + assertEquals(0, getNodesFailingOsUpgrade()); + + for (var version : List.of(Version.fromString("8.1"), Version.fromString("8.2"))) { + // System starts upgrading to next OS version + tester.controller().upgradeOsIn(cloud, version, false); + osUpgrader.maintain(); + statusUpdater.maintain(); + reporter.maintain(); + assertEquals(0, getNodesFailingOsUpgrade()); + + // 30 minutes pass and nothing happens + tester.clock().advance(Duration.ofMinutes(30)); + statusUpdater.maintain(); + reporter.maintain(); + assertEquals(0, getNodesFailingOsUpgrade()); + + // 1/3 nodes upgrade within timeout + tester.configServer().setOsVersion(SystemApplication.tenantHost.id(), zone.getId(), version, 1); + tester.clock().advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1))); + statusUpdater.maintain(); + reporter.maintain(); + assertEquals(2, getNodesFailingOsUpgrade()); + + // 3/3 nodes upgrade + tester.configServer().setOsVersion(SystemApplication.tenantHost.id(), zone.getId(), version); + statusUpdater.maintain(); + reporter.maintain(); + assertEquals(0, getNodesFailingOsUpgrade()); + } + } + private Duration getAverageDeploymentDuration(ApplicationId id) { return Duration.ofSeconds(getMetric(MetricsReporter.DEPLOYMENT_AVERAGE_DURATION, id).longValue()); } @@ -278,6 +330,10 @@ public class MetricsReporterTest { return metrics.getMetric(MetricsReporter.NODES_FAILING_SYSTEM_UPGRADE).intValue(); } + private int getNodesFailingOsUpgrade() { + return metrics.getMetric(MetricsReporter.NODES_FAILING_OS_UPGRADE).intValue(); + } + private Number getMetric(String name, ApplicationId id) { return metrics.getMetric((dimensions) -> id.tenant().value().equals(dimensions.get("tenant")) && appDimension(id).equals(dimensions.get("app")), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index 1af5fafbb79..5e92112d465 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -12,7 +12,7 @@ import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; -import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; +import com.yahoo.vespa.hosted.controller.versions.NodeVersion; import org.junit.Before; import org.junit.Test; @@ -111,13 +111,13 @@ public class OsUpgraderTest { assertWanted(version1, SystemApplication.tenantHost, zone1.getId(), zone2.getId(), zone3.getId(), zone4.getId()); statusUpdater.maintain(); assertTrue("All nodes on target version", tester.controller().osVersionStatus().nodesIn(cloud).stream() - .allMatch(node -> node.version().equals(version1))); + .allMatch(node -> node.currentVersion().equals(version1))); } - private List<OsVersionStatus.Node> nodesOn(Version version) { + private List<NodeVersion> nodesOn(Version version) { return tester.controller().osVersionStatus().versions().entrySet().stream() .filter(entry -> entry.getKey().version().equals(version)) - .flatMap(entry -> entry.getValue().stream()) + .flatMap(entry -> entry.getValue().asMap().values().stream()) .collect(Collectors.toList()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java index fe7f39fd66d..e51fcff33d1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java @@ -3,18 +3,15 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.config.provision.zone.UpgradePolicy; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import com.yahoo.vespa.hosted.controller.versions.OsVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; import org.junit.Test; import java.time.Duration; -import java.util.List; -import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -47,10 +44,10 @@ public class OsVersionStatusUpdaterTest { tester.controller().upgradeOsIn(cloud, version1, false); statusUpdater.maintain(); - Map<OsVersion, List<OsVersionStatus.Node>> osVersions = tester.controller().osVersionStatus().versions(); + var osVersions = tester.controller().osVersionStatus().versions(); assertEquals(2, osVersions.size()); - assertFalse("All nodes on unknown version", osVersions.get(new OsVersion(Version.emptyVersion, cloud)).isEmpty()); - assertTrue("No nodes on current target", osVersions.get(new OsVersion(version1, cloud)).isEmpty()); + assertFalse("All nodes on unknown version", osVersions.get(new OsVersion(Version.emptyVersion, cloud)).asMap().isEmpty()); + assertTrue("No nodes on current target", osVersions.get(new OsVersion(version1, cloud)).asMap().isEmpty()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java index f28ce83e643..0245e7475f7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java @@ -11,7 +11,6 @@ import org.junit.Test; import java.time.Duration; import java.util.Collection; -import java.util.List; import static org.junit.Assert.assertEquals; @@ -27,20 +26,14 @@ public class ResourceMeterMaintainerTest { @Test public void testMaintainer() { - var awsZone = ZoneApiMock.newBuilder().withId("prod.aws-us-east-1").withCloud("aws").build(); - tester.zoneRegistry().setZones( - ZoneApiMock.newBuilder().withId("prod.us-east-3").build(), - ZoneApiMock.newBuilder().withId("prod.us-west-1").build(), - ZoneApiMock.newBuilder().withId("prod.us-central-1").build(), - awsZone); - tester.configServer().nodeRepository().addFixedNodes(awsZone.getId()); + setUpZones(); ResourceMeterMaintainer resourceMeterMaintainer = new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), new JobControl(tester.curator()), metrics, snapshotConsumer); resourceMeterMaintainer.maintain(); Collection<ResourceSnapshot> consumedResources = snapshotConsumer.consumedResources(); // The mocked repository contains two applications, so we should also consume two ResourceSnapshots - assertEquals(2, consumedResources.size()); + assertEquals(4, consumedResources.size()); ResourceSnapshot app1 = consumedResources.stream().filter(snapshot -> snapshot.getApplicationId().equals(ApplicationId.from("tenant1", "app1", "default"))).findFirst().orElseThrow(); ResourceSnapshot app2 = consumedResources.stream().filter(snapshot -> snapshot.getApplicationId().equals(ApplicationId.from("tenant2", "app2", "default"))).findFirst().orElseThrow(); @@ -53,7 +46,19 @@ public class ResourceMeterMaintainerTest { assertEquals(500, app2.getDiskGb(), DELTA); assertEquals(tester.clock().millis()/1000, metrics.getMetric("metering_last_reported")); - assertEquals(1112.0d, (Double) metrics.getMetric("metering_total_reported"), DELTA); + assertEquals(2224.0d, (Double) metrics.getMetric("metering_total_reported"), DELTA); } + private void setUpZones() { + ZoneApiMock nonAwsZone = ZoneApiMock.newBuilder().withId("test.region-1").build(); + ZoneApiMock awsZone1 = ZoneApiMock.newBuilder().withId("prod.region-2").withCloud("aws").build(); + ZoneApiMock awsZone2 = ZoneApiMock.newBuilder().withId("test.region-3").withCloud("aws").build(); + tester.zoneRegistry().setZones( + nonAwsZone, + awsZone1, + awsZone2); + tester.configServer().nodeRepository().addFixedNodes(nonAwsZone.getId()); + tester.configServer().nodeRepository().addFixedNodes(awsZone1.getId()); + tester.configServer().nodeRepository().addFixedNodes(awsZone2.getId()); + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 08963b9fec7..447bce0a544 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -19,7 +19,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; -import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentActivity; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; @@ -92,7 +91,7 @@ public class ApplicationSerializerTest { Instant activityAt = Instant.parse("2018-06-01T10:15:30.00Z"); deployments.add(new Deployment(zone1, applicationVersion1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3))); // One deployment without cluster info and utils deployments.add(new Deployment(zone2, applicationVersion2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5), - createClusterUtils(3, 0.2), createClusterInfo(3, 4), + createClusterInfo(3, 4), new DeploymentMetrics(2, 3, 4, 5, 6, Optional.of(Instant.now().truncatedTo(ChronoUnit.MILLIS)), Map.of(DeploymentMetrics.Warning.all, 3)), @@ -191,10 +190,6 @@ public class ApplicationSerializerTest { assertEquals(original.require(id1.instance()).rotations(), serialized.require(id1.instance()).rotations()); assertEquals(original.require(id1.instance()).rotationStatus(), serialized.require(id1.instance()).rotationStatus()); - // Test cluster utilization - assertEquals(0, serialized.require(id1.instance()).deployments().get(zone1).clusterUtils().size()); - assertEquals(0, serialized.require(id1.instance()).deployments().get(zone2).clusterUtils().size()); - // Test cluster info assertEquals(3, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().size()); assertEquals(10, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorCost()); @@ -232,21 +227,6 @@ public class ApplicationSerializerTest { return result; } - private Map<ClusterSpec.Id, ClusterUtilization> createClusterUtils(int clusters, double inc) { - Map<ClusterSpec.Id, ClusterUtilization> result = new HashMap<>(); - - ClusterUtilization util = new ClusterUtilization(0,0,0,0); - for (int cluster = 0; cluster < clusters; cluster++) { - double agg = cluster*inc; - result.put(ClusterSpec.Id.from("id" + cluster), new ClusterUtilization( - util.getMemory()+ agg, - util.getCpu()+ agg, - util.getDisk() + agg, - util.getDiskBusy() + agg)); - } - return result; - } - @Test public void testCompleteApplicationDeserialization() throws Exception { byte[] applicationJson = Files.readAllBytes(testData.resolve("complete-application.json")); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java index 5073f651fd3..ba771d70d26 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java @@ -1,18 +1,23 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; +import com.google.common.collect.ImmutableMap; import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.controller.versions.NodeVersion; +import com.yahoo.vespa.hosted.controller.versions.NodeVersions; import com.yahoo.vespa.hosted.controller.versions.OsVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; import org.junit.Test; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.Instant; import java.util.List; -import java.util.Map; -import java.util.TreeMap; import static org.junit.Assert.assertEquals; @@ -25,22 +30,41 @@ public class OsVersionStatusSerializerTest { public void test_serialization() { Version version1 = Version.fromString("7.1"); Version version2 = Version.fromString("7.2"); - Map<OsVersion, List<OsVersionStatus.Node>> versions = new TreeMap<>(); + var versions = ImmutableMap.<OsVersion, NodeVersions>builder(); - versions.put(new OsVersion(version1, CloudName.defaultName()), List.of( - new OsVersionStatus.Node(HostName.from("node1"), version1, Environment.prod, RegionName.from("us-west")), - new OsVersionStatus.Node(HostName.from("node2"), version1, Environment.prod, RegionName.from("us-east")) - )); - versions.put(new OsVersion(version2, CloudName.defaultName()), List.of( - new OsVersionStatus.Node(HostName.from("node3"), version2, Environment.prod, RegionName.from("us-west")), - new OsVersionStatus.Node(HostName.from("node4"), version2, Environment.prod, RegionName.from("us-east")) + versions.put(new OsVersion(version1, CloudName.defaultName()), NodeVersions.EMPTY.with(List.of( + new NodeVersion(HostName.from("node1"), ZoneId.from("prod", "us-west"), version1, version2, Instant.ofEpochMilli(1)), + new NodeVersion(HostName.from("node2"), ZoneId.from("prod", "us-east"), version1, version2, Instant.ofEpochMilli(2)) + ))); + versions.put(new OsVersion(version2, CloudName.defaultName()), NodeVersions.EMPTY.with(List.of( + new NodeVersion(HostName.from("node3"), ZoneId.from("prod", "us-west"), version2, version2, Instant.ofEpochMilli(3)), + new NodeVersion(HostName.from("node4"), ZoneId.from("prod", "us-east"), version2, version2, Instant.ofEpochMilli(4)) + ))); - )); - - OsVersionStatusSerializer serializer = new OsVersionStatusSerializer(new OsVersionSerializer()); - OsVersionStatus status = new OsVersionStatus(versions); + OsVersionStatusSerializer serializer = new OsVersionStatusSerializer(new OsVersionSerializer(), new NodeVersionSerializer()); + OsVersionStatus status = new OsVersionStatus(versions.build()); OsVersionStatus serialized = serializer.fromSlime(serializer.toSlime(status)); assertEquals(status.versions(), serialized.versions()); } + @Test + public void testLegacySerialization() throws Exception { + var data = Files.readAllBytes(Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/os-version-status-legacy-format.json")); + var serializer = new OsVersionStatusSerializer(new OsVersionSerializer(), new NodeVersionSerializer()); + var versions = ImmutableMap.of( + new OsVersion(Version.fromString("7.42"), CloudName.from("yahoo")), + NodeVersions.EMPTY.with(List.of(new NodeVersion(HostName.from("node1"), ZoneId.from("prod", "us-north-1"), + Version.fromString("7.42"), Version.emptyVersion, Instant.EPOCH), + new NodeVersion(HostName.from("node2"), ZoneId.from("test", "us-north-2"), + Version.fromString("7.42"), Version.emptyVersion, Instant.EPOCH)))); + + var deserialized = serializer.fromSlime(SlimeUtils.jsonToSlime(data)); + assertEquals(versions, deserialized.versions()); + + + var serialized = new String(SlimeUtils.toJsonBytes(serializer.toSlime(new OsVersionStatus(versions))), StandardCharsets.UTF_8); + assertEquals("{\"versions\":[{\"version\":\"7.42.0\",\"cloud\":\"yahoo\",\"nodeVersions\":[{\"hostname\":\"node1\",\"zone\":\"prod.us-north-1\",\"wantedVersion\":\"0.0.0\",\"changedAt\":0},{\"hostname\":\"node2\",\"zone\":\"test.us-north-2\",\"wantedVersion\":\"0.0.0\",\"changedAt\":0}],\"nodes\":[{\"hostname\":\"node1\",\"version\":\"7.42.0\",\"region\":\"us-north-1\",\"environment\":\"prod\"},{\"hostname\":\"node2\",\"version\":\"7.42.0\",\"region\":\"us-north-2\",\"environment\":\"test\"}]}]}", + serialized); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java index 5d65cf0381e..a80dcc118dc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics; import com.yahoo.vespa.hosted.controller.versions.NodeVersion; @@ -45,7 +46,7 @@ public class VersionStatusSerializerTest { false, nodeVersions(Version.fromString("5.0"), Version.fromString("5.1"), Instant.ofEpochMilli(456), "cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); VersionStatus status = new VersionStatus(vespaVersions); - VersionStatusSerializer serializer = new VersionStatusSerializer(); + VersionStatusSerializer serializer = new VersionStatusSerializer(new NodeVersionSerializer()); VersionStatus deserialized = serializer.fromSlime(serializer.toSlime(status)); assertEquals(status.versions().size(), deserialized.versions().size()); @@ -67,7 +68,7 @@ public class VersionStatusSerializerTest { @Test public void testLegacySerialization() throws Exception { var data = Files.readAllBytes(Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/version-status-legacy-format.json")); - var serializer = new VersionStatusSerializer(); + var serializer = new VersionStatusSerializer(new NodeVersionSerializer()); var deserializedStatus = serializer.fromSlime(SlimeUtils.jsonToSlime(data)); var statistics = new DeploymentStatistics( @@ -76,11 +77,16 @@ public class VersionStatusSerializerTest { List.of(), List.of() ); + var nodeVersions = List.of(new NodeVersion(HostName.from("cfg1"), ZoneId.defaultId(), Version.fromString("7.0"), + Version.fromString("7.1"), Instant.ofEpochMilli(1111)), + new NodeVersion(HostName.from("cfg2"), ZoneId.defaultId(), Version.fromString("7.0"), + Version.fromString("7.1"), Instant.ofEpochMilli(2222)), + new NodeVersion(HostName.from("cfg3"), ZoneId.defaultId(), Version.fromString("7.0"), + Version.fromString("7.1"), Instant.ofEpochMilli(3333))); var vespaVersion = new VespaVersion(statistics, "badc0ffee", Instant.ofEpochMilli(123), true, true, true, - nodeVersions(Version.emptyVersion, Version.emptyVersion, - Instant.EPOCH, "cfg1", "cfg2", "cfg3"), + NodeVersions.EMPTY.with(nodeVersions), VespaVersion.Confidence.normal); VespaVersion deserialized = deserializedStatus.versions().get(0); @@ -97,7 +103,7 @@ public class VersionStatusSerializerTest { private static NodeVersions nodeVersions(Version version, Version wantedVersion, Instant changedAt, String... hostnames) { var nodeVersions = new ArrayList<NodeVersion>(); for (var hostname : hostnames) { - nodeVersions.add(new NodeVersion(HostName.from(hostname), version, wantedVersion, changedAt)); + nodeVersions.add(new NodeVersion(HostName.from(hostname), ZoneId.from("prod", "us-north-1"), version, wantedVersion, changedAt)); } return NodeVersions.EMPTY.with(nodeVersions); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/os-version-status-legacy-format.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/os-version-status-legacy-format.json new file mode 100644 index 00000000000..5a6a864cbf8 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/os-version-status-legacy-format.json @@ -0,0 +1,22 @@ +{ + "versions": [ + { + "version": "7.42", + "cloud": "yahoo", + "nodes": [ + { + "hostname": "node1", + "version": "7.42", + "region": "us-north-1", + "environment": "prod" + }, + { + "hostname": "node2", + "version": "7.42", + "region": "us-north-2", + "environment": "test" + } + ] + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/version-status-legacy-format.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/version-status-legacy-format.json index 96ca22e1c1a..08463ed7cb4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/version-status-legacy-format.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/version-status-legacy-format.json @@ -13,10 +13,22 @@ "deploying": [] }, "confidence": "normal", - "configServerHostnames": [ - "cfg1", - "cfg2", - "cfg3" + "nodeVersions": [ + { + "hostname": "cfg1", + "wantedVersion": "7.1", + "changedAt": 1111 + }, + { + "hostname": "cfg2", + "wantedVersion": "7.1", + "changedAt": 2222 + }, + { + "hostname": "cfg3", + "wantedVersion": "7.1", + "changedAt": 3333 + } ] } ] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 7cacd91a5c4..9c957785606 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -45,7 +45,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClien import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; -import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; @@ -65,8 +64,6 @@ import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; -import com.yahoo.vespa.hosted.controller.rotation.RotationState; -import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; @@ -84,7 +81,6 @@ import java.util.ArrayList; import java.util.Base64; import java.util.Collections; import java.util.HashMap; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -117,7 +113,18 @@ public class ApplicationApiTest extends ControllerContainerTest { "-----END PUBLIC KEY-----\n"; private static final String quotedPemPublicKey = pemPublicKey.replaceAll("\\n", "\\\\n"); - private static final ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + private static final ApplicationPackage applicationPackageDefault = new ApplicationPackageBuilder() + .instances("default") + .environment(Environment.prod) + .globalServiceId("foo") + .region("us-central-1") + .region("us-east-3") + .region("us-west-1") + .blockChange(false, true, "mon-fri", "0-8", "UTC") + .build(); + + private static final ApplicationPackage applicationPackageInstance1 = new ApplicationPackageBuilder() + .instances("instance1") .environment(Environment.prod) .globalServiceId("foo") .region("us-central-1") @@ -225,7 +232,7 @@ public class ApplicationApiTest extends ControllerContainerTest { addUserToHostedOperatorRole(HostedAthenzIdentities.from(HOSTED_VESPA_OPERATOR)); // POST (deploy) an application to a zone - manual user deployment (includes a content hash for verification) - MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1, true); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1/deploy", POST) .data(entity) .header("X-Content-Hash", Base64.getEncoder().encodeToString(Signatures.sha256Digest(entity::data))) @@ -245,7 +252,7 @@ public class ApplicationApiTest extends ControllerContainerTest { controllerTester.jobCompletion(JobType.component) .application(id) .projectId(screwdriverProjectId) - .uploadArtifact(applicationPackage) + .uploadArtifact(applicationPackageInstance1) .submit(); // ... systemtest @@ -309,6 +316,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST (create) another application ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .instances("instance1") .environment(Environment.prod) .region("us-west-1") .build(); @@ -354,7 +362,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/key", POST) .userIdentity(USER_ID) .data("{\"key\":\"" + pemPublicKey + "\"}"), - "{\"message\":\"Added deploy key " + quotedPemPublicKey + "\"}"); + "{\"keys\":[\"-----BEGIN PUBLIC KEY-----\\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\\n-----END PUBLIC KEY-----\\n\"]}"); // PATCH in a pem deploy key at deprecated path tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/instance/default", PATCH) @@ -377,7 +385,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/key", DELETE) .userIdentity(USER_ID) .data("{\"key\":\"" + pemPublicKey + "\"}"), - "{\"message\":\"Removed deploy key " + quotedPemPublicKey + "\"}"); + "{\"keys\":[]}"); tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", GET) .userIdentity(USER_ID), @@ -585,6 +593,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // Second attempt has a service under a different domain than the tenant of the application, and fails. ApplicationPackage packageWithServiceForWrongDomain = new ApplicationPackageBuilder() + .instances("instance1") .environment(Environment.prod) .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from(ATHENZ_TENANT_DOMAIN_2.getName()), AthenzService.from("service")) .region("us-west-1") @@ -597,6 +606,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // Third attempt finally has a service under the domain of the tenant, and succeeds. ApplicationPackage packageWithService = new ApplicationPackageBuilder() + .instances("instance1") .environment(Environment.prod) .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from(ATHENZ_TENANT_DOMAIN.getName()), AthenzService.from("service")) .region("us-west-1") @@ -710,6 +720,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.computeVersionStatus(); createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .instances("instance1") .globalServiceId("foo") .region("us-west-1") .region("us-east-3") @@ -718,7 +729,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // Create tenant and deploy ApplicationId id = createTenantAndApplication(); long projectId = 1; - MultiPartStreamer deployData = createApplicationDeployData(Optional.empty(), false); + MultiPartStreamer deployData = createApplicationDeployData(Optional.of(applicationPackage), false); startAndTestChange(controllerTester, id, projectId, applicationPackage, deployData, 100); // us-west-1 @@ -781,6 +792,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.computeVersionStatus(); createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .instances("instance1") .region("us-west-1") .region("us-east-3") .region("eu-west-1") @@ -857,7 +869,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId("application1")); // POST (deploy) an application to a prod zone - allowed when project ID is not specified - MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1, true); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/deploy", POST) .data(entity) .screwdriverIdentity(SCREWDRIVER_ID), @@ -889,6 +901,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // Deploy ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .instances("instance1") .region("us-east-3") .build(); ApplicationId id = createTenantAndApplication(); @@ -908,6 +921,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // New zone is added before us-east-3 applicationPackage = new ApplicationPackageBuilder() + .instances("instance1") .globalServiceId("foo") // These decides the ordering of deploymentJobs and instances in the response .region("us-west-1") @@ -953,9 +967,9 @@ public class ApplicationApiTest extends ControllerContainerTest { ResourceAllocation lastMonth = new ResourceAllocation(24, 48, 2000); ApplicationId applicationId = ApplicationId.from("doesnotexist", "doesnotexist", "default"); Map<ApplicationId, List<ResourceSnapshot>> snapshotHistory = Map.of(applicationId, List.of( - new ResourceSnapshot(applicationId, 1, 2,3, Instant.ofEpochMilli(123)), - new ResourceSnapshot(applicationId, 1, 2,3, Instant.ofEpochMilli(246)), - new ResourceSnapshot(applicationId, 1, 2,3, Instant.ofEpochMilli(492)))); + new ResourceSnapshot(applicationId, 1, 2,3, Instant.ofEpochMilli(123), ZoneId.defaultId()), + new ResourceSnapshot(applicationId, 1, 2,3, Instant.ofEpochMilli(246), ZoneId.defaultId()), + new ResourceSnapshot(applicationId, 1, 2,3, Instant.ofEpochMilli(492), ZoneId.defaultId()))); mockMeteringClient.setMeteringInfo(new MeteringInfo(thisMonth, lastMonth, currentSnapshot, snapshotHistory)); @@ -1060,7 +1074,7 @@ public class ApplicationApiTest extends ControllerContainerTest { configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Failed to prepare application", ConfigServerException.ErrorCode.INVALID_APPLICATION_PACKAGE, null)); // POST (deploy) an application with an invalid application package - MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1, true); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1/deploy", POST) .data(entity) .userIdentity(USER_ID), @@ -1180,7 +1194,7 @@ public class ApplicationApiTest extends ControllerContainerTest { 200); // Deploy to an authorized zone by a user tenant is disallowed - MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackageDefault, true); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/deploy", POST) .data(entity) .userIdentity(USER_ID), @@ -1593,7 +1607,7 @@ public class ApplicationApiTest extends ControllerContainerTest { } private MultiPartStreamer createApplicationDeployData(Optional<ApplicationPackage> applicationPackage, - Optional<ApplicationVersion> applicationVersion, boolean deployDirectly) { + Optional<ApplicationVersion> applicationVersion, boolean deployDirectly) { MultiPartStreamer streamer = new MultiPartStreamer(); streamer.addJson("deployOptions", deployOptions(deployDirectly, applicationVersion)); applicationPackage.ifPresent(ap -> streamer.addBytes("applicationZip", ap.zippedContent())); @@ -1745,14 +1759,11 @@ public class ApplicationApiTest extends ControllerContainerTest { clusterInfo.put(ClusterSpec.Id.from("cluster1"), new ClusterInfo("flavor1", 37, 2, 4, 50, ClusterSpec.Type.content, hostnames)); - Map<ClusterSpec.Id, ClusterUtilization> clusterUtils = new HashMap<>(); - clusterUtils.put(ClusterSpec.Id.from("cluster1"), new ClusterUtilization(0.3, 0.6, 0.4, 0.3)); DeploymentMetrics metrics = new DeploymentMetrics(1, 2, 3, 4, 5, Optional.of(Instant.ofEpochMilli(123123)), Map.of()); lockedApplication = lockedApplication.with(instance.name(), lockedInstance -> lockedInstance.withClusterInfo(deployment.zone(), clusterInfo) - .withClusterUtilization(deployment.zone(), clusterUtils) .with(deployment.zone(), metrics) .recordActivityAt(Instant.parse("2018-06-01T10:15:30.00Z"), deployment.zone())); } @@ -1771,17 +1782,6 @@ public class ApplicationApiTest extends ControllerContainerTest { new RotationStatusUpdater(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator())).run(); } - private RotationStatus rotationStatus(Instance instance) { - return controllerTester.controller().applications().rotationRepository().getRotation(instance) - .map(rotation -> { - var rotationStatus = controllerTester.controller().serviceRegistry().globalRoutingService().getHealthStatus(rotation.name()); - var statusMap = new LinkedHashMap<ZoneId, RotationState>(); - rotationStatus.forEach((zone, status) -> statusMap.put(zone, RotationState.in)); - return RotationStatus.from(Map.of(rotation.id(), statusMap)); - }) - .orElse(RotationStatus.EMPTY); - } - private void updateContactInformation() { Contact contact = new Contact(URI.create("www.contacts.tld/1234"), URI.create("www.properties.tld/1234"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index 0a4d046e318..bb1e6b6256a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -76,8 +76,8 @@ public class DeploymentApiTest extends ControllerContainerTest { version.isControllerVersion(), version.isSystemVersion(), version.isReleased(), - NodeVersions.EMPTY.with(List.of(new NodeVersion(HostName.from("config1.test"), version.versionNumber(), version.versionNumber(), Instant.EPOCH), - new NodeVersion(HostName.from("config2.test"), version.versionNumber(), version.versionNumber(), Instant.EPOCH))), + NodeVersions.EMPTY.with(List.of(new NodeVersion(HostName.from("config1.test"), ZoneId.defaultId(), version.versionNumber(), version.versionNumber(), Instant.EPOCH), + new NodeVersion(HostName.from("config2.test"), ZoneId.defaultId(), version.versionNumber(), version.versionNumber(), Instant.EPOCH))), VespaVersion.confidenceFrom(version.statistics(), controller) ); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index f2410c47908..b1f5f33b960 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -143,14 +143,14 @@ public class UserApiTest extends ControllerContainerCloudTest { tester.assertResponse(request("/application/v4/tenant/my-tenant/application/my-app/key", POST) .roles(Set.of(Role.tenantOperator(id.tenant()))) .data("{\"key\":\"" + pemPublicKey + "\"}"), - "{\"message\":\"Added deploy key " + quotedPemPublicKey + "\"}"); + new File("first-deploy-key.json")); // POST a pem developer key tester.assertResponse(request("/application/v4/tenant/my-tenant/key", POST) .user("joe@dev") .roles(Set.of(Role.tenantOperator(id.tenant()))) .data("{\"key\":\"" + pemPublicKey + "\"}"), - "{\"message\":\"Set developer key " + quotedPemPublicKey + " for joe@dev\"}"); + new File("first-developer-key.json")); // POST the same pem developer key for a different user is forbidden tester.assertResponse(request("/application/v4/tenant/my-tenant/key", POST) @@ -165,7 +165,7 @@ public class UserApiTest extends ControllerContainerCloudTest { .user("operator@tenant") .roles(Set.of(Role.tenantOperator(id.tenant()))) .data("{\"key\":\"" + otherPemPublicKey + "\"}"), - "{\"message\":\"Set developer key " + otherQuotedPemPublicKey + " for operator@tenant\"}"); + new File("both-developer-keys.json")); // GET tenant information with keys tester.assertResponse(request("/application/v4/tenant/my-tenant/") @@ -176,7 +176,7 @@ public class UserApiTest extends ControllerContainerCloudTest { tester.assertResponse(request("/application/v4/tenant/my-tenant/key", DELETE) .roles(Set.of(Role.tenantOperator(id.tenant()))) .data("{\"key\":\"" + pemPublicKey + "\"}"), - "{\"message\":\"Removed developer key " + quotedPemPublicKey + " for joe@dev\"}"); + new File("second-developer-key.json")); // DELETE an application role is allowed for an application admin. tester.assertResponse(request("/user/v1/tenant/my-tenant/application/my-app", DELETE) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/both-developer-keys.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/both-developer-keys.json new file mode 100644 index 00000000000..2ff1c29fe29 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/both-developer-keys.json @@ -0,0 +1,12 @@ +{ + "keys": [ + { + "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n", + "user": "joe@dev" + }, + { + "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\npDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n-----END PUBLIC KEY-----\n", + "user": "operator@tenant" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-deploy-key.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-deploy-key.json new file mode 100644 index 00000000000..1c86877b77d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-deploy-key.json @@ -0,0 +1,5 @@ +{ + "keys": [ + "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n" + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-developer-key.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-developer-key.json new file mode 100644 index 00000000000..b7d48f283f3 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-developer-key.json @@ -0,0 +1,9 @@ +{ + "keys": [ + { + "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n", + "user": "joe@dev" + } + ] +} + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/second-developer-key.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/second-developer-key.json new file mode 100644 index 00000000000..f7d90f31116 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/second-developer-key.json @@ -0,0 +1,8 @@ +{ + "keys": [ + { + "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\npDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n-----END PUBLIC KEY-----\n", + "user": "operator@tenant" + } + ] +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index ae782bf32ff..4b1befc1770 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -82,9 +82,14 @@ public class Flags { public static final UnboundBooleanFlag INCLUDE_SIS_IN_TRUSTSTORE = defineFeatureFlag( "include-sis-in-truststore", false, - "Whether to use the trust store backed by Athenz and Service Identity certificates.", - "Takes effect on next tick, but may get throttled due to orchestration.", - HOSTNAME); + "Whether to use the trust store backed by Athenz and (in public) Service Identity certificates in " + + "host-admin and/or Docker containers", + "Takes effect on restart of host-admin (for host-admin), and restart of Docker container.", + // For host-admin, HOSTNAME and NODE_TYPE is available + // For Docker containers, HOSTNAME and APPLICATION_ID is available + // WARNING: Having different sets of dimensions is DISCOURAGED in general, but needed for here since + // trust store for host-admin is determined before having access to application ID from node repo. + HOSTNAME, NODE_TYPE, APPLICATION_ID); public static final UnboundStringFlag TLS_INSECURE_MIXED_MODE = defineStringFlag( "tls-insecure-mixed-mode", "tls_client_mixed_server", diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java b/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java index d578f937485..68656f06d7d 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java @@ -16,42 +16,67 @@ import java.util.Optional; */ public class Properties { + /** + * Returns the relevant application ID. This is the 'tenant', 'application' and 'instance' properties. + * The instance defaults to the user name of the current user, if not explicitly set. + */ public static ApplicationId application() { return ApplicationId.from(requireNonBlankProperty("tenant"), requireNonBlankProperty("application"), - getNonBlankProperty("instance").orElse("default")); + getNonBlankProperty("instance").orElse(user())); } + /** Returns the relevant environment, if this is set with the 'environment' property */ public static Optional<Environment> environment() { return getNonBlankProperty("environment").map(Environment::from); } + /** Returns the relevant region, if this is set with the 'region' property */ public static Optional<RegionName> region() { return getNonBlankProperty("region").map(RegionName::from); } - public static URI endpoint() { + /** Returns the URL of the API endpoint of the Vespa cloud. This must be set with the 'endpoint' property. */ + public static URI apiEndpoint() { return URI.create(requireNonBlankProperty("endpoint")); } - public static Path privateKeyFile() { + /** Returns the path of the API private key. This must be set with the 'privateKeyFile' property. */ + public static Path apiPrivateKeyFile() { return Paths.get(requireNonBlankProperty("privateKeyFile")); } - public static Optional<Path> certificateFile() { + /** Returns the path of the API certificate, if this is set with the 'certificateFile' property. */ + public static Optional<Path> apiCertificateFile() { return getNonBlankProperty("certificateFile").map(Paths::get); } + /** Returns the actual private key as a string */ public static Optional<String> privateKey() { return getNonBlankProperty("privateKey"); } + /** Returns the path of the data plane certificate file, if this is set with the 'dataPlaneCertificateFile' property. */ + public static Optional<Path> dataPlaneCertificateFile() { + return getNonBlankProperty("dataPlaneCertificateFile").map(Paths::get); + } + + /** Returns the path of the data plane private key file, if this is set with the 'dataPlanePrivateKeyFile' property. */ + public static Optional<Path> dataPlanePrivateKeyFile() { + return getNonBlankProperty("dataPlaneKeyFile").map(Paths::get); + } + + /** Returns the user name of the current user. This is set with the 'user.name' property. */ + public static String user() { + return System.getProperty("user.name"); + } + /** Returns the system property with the given name if it is set, or empty. */ public static Optional<String> getNonBlankProperty(String name) { return Optional.ofNullable(System.getProperty(name)).filter(value -> ! value.isBlank()); } - /** Returns the system property with the given name if it is set, or throws. */ + /** Returns the system property with the given name if it is set, or throws an IllegalStateException. */ public static String requireNonBlankProperty(String name) { return getNonBlankProperty(name).orElseThrow(() -> new IllegalStateException("Missing required property '" + name + "'")); } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java index 9a10c70ceab..0074f5cfe89 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java @@ -31,6 +31,7 @@ public class AccessLogRequestLog extends AbstractLifeCycle implements RequestLog // TODO These hardcoded headers should be provided by config instead private static final String HEADER_NAME_X_FORWARDED_FOR = "x-forwarded-for"; + private static final String HEADER_NAME_X_FORWARDED_PORT = "X-Forwarded-Port"; private static final String HEADER_NAME_Y_RA = "y-ra"; private static final String HEADER_NAME_Y_RP = "y-rp"; private static final String HEADER_NAME_YAHOOREMOTEIP = "yahooremoteip"; @@ -58,23 +59,24 @@ public class AccessLogRequestLog extends AbstractLifeCycle implements RequestLog accessLogEntry.setRawQuery(queryString); } - final String remoteAddress = getRemoteAddress(request); - final int remotePort = getRemotePort(request); - final String peerAddress = request.getRemoteAddr(); - final int peerPort = request.getRemotePort(); - accessLogEntry.setUserAgent(request.getHeader("User-Agent")); accessLogEntry.setHttpMethod(request.getMethod()); accessLogEntry.setHostString(request.getHeader("Host")); accessLogEntry.setReferer(request.getHeader("Referer")); + + String peerAddress = request.getRemoteAddr(); accessLogEntry.setIpV4Address(peerAddress); - accessLogEntry.setRemoteAddress(remoteAddress); - accessLogEntry.setRemotePort(remotePort); + accessLogEntry.setPeerAddress(peerAddress); + String remoteAddress = getRemoteAddress(request); if (!Objects.equal(remoteAddress, peerAddress)) { - accessLogEntry.setPeerAddress(peerAddress); + accessLogEntry.setRemoteAddress(remoteAddress); } + + int peerPort = request.getRemotePort(); + accessLogEntry.setPeerPort(peerPort); + int remotePort = getRemotePort(request); if (remotePort != peerPort) { - accessLogEntry.setPeerPort(peerPort); + accessLogEntry.setRemotePort(remotePort); } accessLogEntry.setHttpVersion(request.getProtocol()); accessLogEntry.setScheme(request.getScheme()); @@ -118,15 +120,16 @@ public class AccessLogRequestLog extends AbstractLifeCycle implements RequestLog } private static String getRemoteAddress(final HttpServletRequest request) { - return Alternative.preferred(request.getHeader(HEADER_NAME_X_FORWARDED_FOR)) - .alternatively(() -> request.getHeader(HEADER_NAME_Y_RA)) - .alternatively(() -> request.getHeader(HEADER_NAME_YAHOOREMOTEIP)) - .alternatively(() -> request.getHeader(HEADER_NAME_CLIENT_IP)) + return Optional.ofNullable(request.getHeader(HEADER_NAME_X_FORWARDED_FOR)) + .or(() -> Optional.ofNullable(request.getHeader(HEADER_NAME_Y_RA))) + .or(() -> Optional.ofNullable(request.getHeader(HEADER_NAME_YAHOOREMOTEIP))) + .or(() -> Optional.ofNullable(request.getHeader(HEADER_NAME_CLIENT_IP))) .orElseGet(request::getRemoteAddr); } private static int getRemotePort(final HttpServletRequest request) { - return Optional.ofNullable(request.getHeader(HEADER_NAME_Y_RP)) + return Optional.ofNullable(request.getHeader(HEADER_NAME_X_FORWARDED_PORT)) + .or(() -> Optional.ofNullable(request.getHeader(HEADER_NAME_Y_RP))) .map(Integer::valueOf) .orElseGet(request::getRemotePort); } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/Alternative.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/Alternative.java deleted file mode 100644 index 441082e95c1..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/Alternative.java +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.server.jetty; - -import java.util.Objects; -import java.util.function.Supplier; - -/** - * Simple monad class, like Optional but with support for chaining alternatives in preferred order. - * - * Holds a current value (immutably), but if the current value is null provides an easy way to obtain an instance - * with another value, ad infinitum. - * - * Instances of this class are immutable and thread-safe. - * - * @author bakksjo - */ -public class Alternative<T> { - private final T value; - - private Alternative(final T value) { - this.value = value; - } - - /** - * Creates an instance with the supplied value. - */ - public static <T> Alternative<T> preferred(final T value) { - return new Alternative<>(value); - } - - /** - * Returns itself (unchanged) iff current value != null, - * otherwise returns a new instance with the value supplied by the supplier. - */ - public Alternative<T> alternatively(final Supplier<? extends T> supplier) { - if (value != null) { - return this; - } - - return new Alternative<>(supplier.get()); - } - - /** - * Returns the held value iff != null, otherwise invokes the supplier and returns its value. - */ - public T orElseGet(final Supplier<? extends T> supplier) { - if (value != null) { - return value; - } - return supplier.get(); - } - - @Override - public boolean equals(final Object o) { - if (!(o instanceof Alternative<?>)) { - return false; - } - - final Alternative<?> other = (Alternative<?>) o; - - return Objects.equals(value, other.value); - } - - @Override - public int hashCode() { - return Objects.hashCode(value); - } -} diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLogTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLogTest.java index 3a605040742..580533be4c3 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLogTest.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLogTest.java @@ -82,6 +82,19 @@ public class AccessLogRequestLogTest { assertThat(accessLogEntry.getRemoteAddress(), is("1.2.3.4")); } + @Test + public void verify_x_forwarded_port_precedence () { + AccessLogEntry accessLogEntry = new AccessLogEntry(); + Request jettyRequest = createRequestMock(accessLogEntry); + when(jettyRequest.getRequestURI()).thenReturn("//search/"); + when(jettyRequest.getQueryString()).thenReturn("q=%%2"); + when(jettyRequest.getHeader("X-Forwarded-Port")).thenReturn("80"); + when(jettyRequest.getHeader("y-rp")).thenReturn("8080"); + + new AccessLogRequestLog(mock(AccessLog.class)).log(jettyRequest, createResponseMock()); + assertThat(accessLogEntry.getRemotePort(), is(80)); + } + private static Request createRequestMock(AccessLogEntry entry) { Request request = mock(Request.class); when(request.getAttribute(JDiscHttpServlet.ATTRIBUTE_NAME_ACCESS_LOG_ENTRY)).thenReturn(entry); diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/AlternativeTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/AlternativeTest.java deleted file mode 100644 index 966c8d418b1..00000000000 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/AlternativeTest.java +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.server.jetty; - -import org.junit.Test; - -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.greaterThan; - -/** - * @author bakksjo - */ -public class AlternativeTest { - private static final String MAN = "man"; - private static final String BEAR = "bear"; - private static final String PIG = "pig"; - - @Test - public void singleValue() { - assertThat( - Alternative.preferred(MAN) - .orElseGet(() -> BEAR), - is(MAN)); - } - - @Test - public void singleNull() { - assertThat( - Alternative.preferred(null) - .orElseGet(() -> BEAR), - is(BEAR)); - } - - @Test - public void twoValues() { - assertThat( - Alternative.preferred(MAN) - .alternatively(() -> BEAR) - .orElseGet(() -> PIG), - is(MAN)); - } - - @Test - public void oneNullOneValue() { - assertThat( - Alternative.preferred(null) - .alternatively(() -> MAN) - .orElseGet(() -> BEAR), - is(MAN)); - } - - @Test - public void twoNulls() { - assertThat( - Alternative.preferred(null) - .alternatively(() -> null) - .orElseGet(() -> MAN), - is(MAN)); - } - - @Test - public void singleNullLastResortIsNull() { - assertThat( - Alternative.preferred(null) - .orElseGet(() -> null), - is(nullValue())); - } - - @Test - public void twoNullsLastResortIsNull() { - assertThat( - Alternative.preferred(null) - .alternatively(() -> null) - .orElseGet(() -> null), - is(nullValue())); - } - - @Test - public void oneNullTwoValues() { - assertThat( - Alternative.preferred(null) - .alternatively(() -> MAN) - .alternatively(() -> BEAR) - .orElseGet(() -> PIG), - is(MAN)); - } - - @Test - public void equalValuesMakeEqualAlternatives() { - assertThat(Alternative.preferred(MAN), is(equalTo(Alternative.preferred(MAN)))); - assertThat(Alternative.preferred(BEAR), is(equalTo(Alternative.preferred(BEAR)))); - assertThat(Alternative.preferred(PIG), is(equalTo(Alternative.preferred(PIG)))); - assertThat(Alternative.preferred(null), is(equalTo(Alternative.preferred(null)))); - } - - @Test - public void equalValuesMakeEqualHashCodes() { - assertThat(Alternative.preferred(MAN).hashCode(), is(equalTo(Alternative.preferred(MAN).hashCode()))); - assertThat(Alternative.preferred(BEAR).hashCode(), is(equalTo(Alternative.preferred(BEAR).hashCode()))); - assertThat(Alternative.preferred(PIG).hashCode(), is(equalTo(Alternative.preferred(PIG).hashCode()))); - assertThat(Alternative.preferred(null).hashCode(), is(equalTo(Alternative.preferred(null).hashCode()))); - } - - @Test - public void unequalValuesMakeUnequalAlternatives() { - assertThat(Alternative.preferred(MAN), is(not(equalTo(Alternative.preferred(BEAR))))); - assertThat(Alternative.preferred(MAN), is(not(equalTo(Alternative.preferred(PIG))))); - assertThat(Alternative.preferred(MAN), is(not(equalTo(Alternative.preferred(null))))); - assertThat(Alternative.preferred(BEAR), is(not(equalTo(Alternative.preferred(MAN))))); - assertThat(Alternative.preferred(BEAR), is(not(equalTo(Alternative.preferred(PIG))))); - assertThat(Alternative.preferred(BEAR), is(not(equalTo(Alternative.preferred(null))))); - assertThat(Alternative.preferred(PIG), is(not(equalTo(Alternative.preferred(MAN))))); - assertThat(Alternative.preferred(PIG), is(not(equalTo(Alternative.preferred(BEAR))))); - assertThat(Alternative.preferred(PIG), is(not(equalTo(Alternative.preferred(null))))); - assertThat(Alternative.preferred(null), is(not(equalTo(Alternative.preferred(MAN))))); - assertThat(Alternative.preferred(null), is(not(equalTo(Alternative.preferred(BEAR))))); - assertThat(Alternative.preferred(null), is(not(equalTo(Alternative.preferred(PIG))))); - } - - @Test - public void hashValuesAreDecent() { - final String[] animals = { MAN, BEAR, PIG, "squirrel", "aardvark", "porcupine", "sasquatch", null }; - final Set<Integer> hashCodes = Stream.of(animals) - .map(Alternative::preferred) - .map(Alternative::hashCode) - .collect(Collectors.toSet()); - assertThat(hashCodes.size(), is(greaterThan(animals.length / 2))); // A modest requirement. - } -} diff --git a/metrics/src/vespa/metrics/metric.cpp b/metrics/src/vespa/metrics/metric.cpp index 20083300271..a7a212d759c 100644 --- a/metrics/src/vespa/metrics/metric.cpp +++ b/metrics/src/vespa/metrics/metric.cpp @@ -13,6 +13,7 @@ #include <cassert> #include <algorithm> #include <ostream> +#include <regex> namespace metrics { @@ -39,10 +40,9 @@ MetricVisitor::visitMetric(const Metric&, bool) namespace { std::string namePattern = "[a-zA-Z][_a-zA-Z0-9]*"; + std::regex name_pattern_regex(namePattern); } -vespalib::Regexp Metric::_namePattern(namePattern); - Tag::Tag(vespalib::stringref k) : _key(NameRepo::tagKeyId(k)), _value(TagValueId::empty_handle) @@ -143,7 +143,8 @@ Metric::verifyConstructionParameters() throw vespalib::IllegalArgumentException( "Metric cannot have empty name", VESPA_STRLOC); } - if (!_namePattern.match(getName())) { + const auto &name = getName(); + if (!std::regex_search(name.c_str(), name.c_str() + name.size(), name_pattern_regex)) { throw vespalib::IllegalArgumentException( "Illegal metric name '" + getName() + "'. Names must match pattern " + namePattern, VESPA_STRLOC); diff --git a/metrics/src/vespa/metrics/metric.h b/metrics/src/vespa/metrics/metric.h index 85832ba08d1..845f40a335b 100644 --- a/metrics/src/vespa/metrics/metric.h +++ b/metrics/src/vespa/metrics/metric.h @@ -3,7 +3,6 @@ #include <vespa/vespalib/util/printable.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/util/regexp.h> #include "name_repo.h" namespace metrics { @@ -110,8 +109,6 @@ public: using SP = std::shared_ptr<Metric>; using Tags = std::vector<Tag>; - static vespalib::Regexp _namePattern; - Metric(const String& name, Tags dimensions, const String& description, diff --git a/metrics/src/vespa/metrics/textwriter.cpp b/metrics/src/vespa/metrics/textwriter.cpp index 9ce1005821f..4edfb93b452 100644 --- a/metrics/src/vespa/metrics/textwriter.cpp +++ b/metrics/src/vespa/metrics/textwriter.cpp @@ -11,8 +11,13 @@ namespace metrics { TextWriter::TextWriter(std::ostream& out, uint32_t period, const std::string& regex, bool verbose) - : _period(period), _out(out), _regex(regex), _verbose(verbose) -{ } + : _period(period), _out(out), _regex(), _verbose(verbose) +{ + try { + _regex = std::regex(regex); + } catch (std::regex_error &) { + } +} TextWriter::~TextWriter() { } @@ -50,7 +55,7 @@ TextWriter::writeCommon(const Metric& metric) } std::string mypath(path.str()); path << metric.getMangledName(); - if (_regex.match(path.str())) { + if (_regex && std::regex_search(path.str(), *_regex)) { if (metric.used() || _verbose) { _out << "\n" << mypath; return true; diff --git a/metrics/src/vespa/metrics/textwriter.h b/metrics/src/vespa/metrics/textwriter.h index c4267f07197..b1f09d1f0ed 100644 --- a/metrics/src/vespa/metrics/textwriter.h +++ b/metrics/src/vespa/metrics/textwriter.h @@ -3,7 +3,8 @@ #pragma once #include "metric.h" -#include <vespa/vespalib/util/regexp.h> +#include <regex> +#include <optional> namespace metrics { @@ -11,7 +12,7 @@ class TextWriter : public MetricVisitor { uint32_t _period; std::ostream& _out; std::vector<std::string> _path; - vespalib::Regexp _regex; + std::optional<std::regex> _regex; bool _verbose; public: diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java index 78f720074dc..dc13ea1c9ab 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSync.java @@ -4,11 +4,15 @@ package com.yahoo.vespa.hosted.node.admin.task.util.file; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import java.nio.file.FileSystems; +import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; import java.util.Optional; import java.util.logging.Logger; +import static com.yahoo.yolean.Exceptions.uncheck; + /** * Class to minimize resource usage with repetitive and mostly identical, idempotent, and * mutating file operations, e.g. setting file content, setting owner, etc. @@ -29,15 +33,22 @@ public class FileSync { this.contentCache = new FileContentCache(this.path); } + public boolean convergeTo(TaskContext taskContext, PartialFileData partialFileData) { + return convergeTo(taskContext, partialFileData, false); + } + /** * CPU, I/O, and memory usage is optimized for repeated calls with the same arguments. + * + * @param atomicWrite Whether to write updates to a temporary file in the same directory, and atomically move it + * to path. Ensures the file cannot be read while in the middle of writing it. * @return true if the system was modified: content was written, or owner was set, etc. * system is only modified if necessary (different). */ - public boolean convergeTo(TaskContext taskContext, PartialFileData partialFileData) { + public boolean convergeTo(TaskContext taskContext, PartialFileData partialFileData, boolean atomicWrite) { FileAttributesCache currentAttributes = new FileAttributesCache(path); - boolean modifiedSystem = maybeUpdateContent(taskContext, partialFileData.getContent(), currentAttributes); + boolean modifiedSystem = maybeUpdateContent(taskContext, partialFileData.getContent(), currentAttributes, atomicWrite); AttributeSync attributeSync = new AttributeSync(path.toPath()).with(partialFileData); modifiedSystem |= attributeSync.converge(taskContext, currentAttributes); @@ -47,7 +58,8 @@ public class FileSync { private boolean maybeUpdateContent(TaskContext taskContext, Optional<byte[]> content, - FileAttributesCache currentAttributes) { + FileAttributesCache currentAttributes, + boolean atomicWrite) { if (!content.isPresent()) { return false; } @@ -55,7 +67,7 @@ public class FileSync { if (!currentAttributes.exists()) { taskContext.recordSystemModification(logger, "Creating file " + path); path.createParents(); - path.writeBytes(content.get()); + writeBytes(content.get(), atomicWrite); contentCache.updateWith(content.get(), currentAttributes.forceGet().lastModifiedTime()); return true; } @@ -64,9 +76,20 @@ public class FileSync { return false; } else { taskContext.recordSystemModification(logger, "Patching file " + path); - path.writeBytes(content.get()); + writeBytes(content.get(), atomicWrite); contentCache.updateWith(content.get(), currentAttributes.forceGet().lastModifiedTime()); return true; } } + + private void writeBytes(byte[] content, boolean atomic) { + if (atomic) { + String tmpPath = path.toPath().toString() + ".FileSyncTmp"; + new UnixPath(path.toPath().getFileSystem().getPath(tmpPath)) + .writeBytes(content) + .atomicMove(path.toPath()); + } else { + path.writeBytes(content); + } + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java index afc0e7b5c22..57f3417b789 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.node.admin.task.util.file; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import java.io.File; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -21,6 +20,7 @@ public class FileWriter { private final PartialFileData.Builder fileDataBuilder = PartialFileData.builder(); private final Optional<ByteArraySupplier> contentProducer; + private boolean atomicWrite = false; private boolean overwriteExistingFile = true; public FileWriter(Path path) { @@ -58,6 +58,11 @@ public class FileWriter { return this; } + public FileWriter atomicWrite(boolean atomicWrite) { + this.atomicWrite = atomicWrite; + return this; + } + public FileWriter onlyIfFileDoesNotAlreadyExist() { overwriteExistingFile = false; return this; @@ -78,7 +83,7 @@ public class FileWriter { fileDataBuilder.withContent(content); PartialFileData fileData = fileDataBuilder.create(); - return fileSync.convergeTo(context, fileData); + return fileSync.convergeTo(context, fileData, atomicWrite); } @FunctionalInterface diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java index 2cc74742463..de3555b24a5 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java @@ -206,7 +206,7 @@ public class UnixPath { /** This path must be on the same file system as the to-path. Returns UnixPath of 'to'. */ public UnixPath atomicMove(Path to) { - uncheck(() -> Files.move(path, to, StandardCopyOption.ATOMIC_MOVE)); + uncheck(() -> Files.move(path, to, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING)); return new UnixPath(to); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java index 2bc64a3fdb3..e1a0ed5d972 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java @@ -21,6 +21,7 @@ import static org.mockito.Mockito.verify; public class FileWriterTest { private final FileSystem fileSystem = TestFileSystem.create(); + private final TaskContext context = mock(TaskContext.class); @Test public void testWrite() { @@ -35,7 +36,6 @@ public class FileWriterTest { .withOwner(owner) .withGroup(group) .onlyIfFileDoesNotAlreadyExist(); - TaskContext context = mock(TaskContext.class); assertTrue(writer.converge(context)); verify(context, times(1)).recordSystemModification(any(), eq("Creating file " + path)); @@ -50,4 +50,15 @@ public class FileWriterTest { assertFalse(writer.converge(context)); assertEquals(fileTime, unixPath.getLastModifiedTime()); } + + @Test + public void testAtomicWrite() { + FileWriter writer = new FileWriter(fileSystem.getPath("/foo/bar")) + .atomicWrite(true); + + assertTrue(writer.converge(context, "content")); + + verify(context).recordSystemModification(any(), eq("Creating file /foo/bar")); + assertEquals("content", new UnixPath(writer.path()).readUtf8File()); + } } diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index 144f4ca4ff7..2bb17c54367 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -1232,13 +1232,13 @@ TEST_F("require that commit is not called when inside a commit interval", TEST_F("require that commit is called when crossing a commit interval", SearchableFeedViewFixture(SHORT_DELAY)) { - FastOS_Thread::Sleep(SHORT_DELAY.ms() + 10); + FastOS_Thread::Sleep(SHORT_DELAY.ms() + 100); DocumentContext dc = f.doc1(); f.putAndWait(dc); EXPECT_EQUAL(1u, f.miw._commitCount); EXPECT_EQUAL(1u, f.maw._commitCount); EXPECT_EQUAL(2u, f._docIdLimit.get()); - FastOS_Thread::Sleep(SHORT_DELAY.ms() + 10); + FastOS_Thread::Sleep(SHORT_DELAY.ms() + 100); f.removeAndWait(dc); EXPECT_EQUAL(2u, f.miw._commitCount); EXPECT_EQUAL(2u, f.maw._commitCount); @@ -1257,13 +1257,13 @@ TEST_F("require that commit is not implicitly called after handover to maintenan SearchableFeedViewFixture(SHORT_DELAY)) { f._commitTimeTracker.setReplayDone(); - FastOS_Thread::Sleep(SHORT_DELAY.ms() + 10); + FastOS_Thread::Sleep(SHORT_DELAY.ms() + 100); DocumentContext dc = f.doc1(); f.putAndWait(dc); EXPECT_EQUAL(0u, f.miw._commitCount); EXPECT_EQUAL(0u, f.maw._commitCount); EXPECT_EQUAL(0u, f._docIdLimit.get()); - FastOS_Thread::Sleep(SHORT_DELAY.ms() + 10); + FastOS_Thread::Sleep(SHORT_DELAY.ms() + 100); f.removeAndWait(dc); EXPECT_EQUAL(0u, f.miw._commitCount); EXPECT_EQUAL(0u, f.maw._commitCount); diff --git a/searchlib/src/tests/fef/properties/properties_test.cpp b/searchlib/src/tests/fef/properties/properties_test.cpp index df868de3a97..b7478da3f71 100644 --- a/searchlib/src/tests/fef/properties/properties_test.cpp +++ b/searchlib/src/tests/fef/properties/properties_test.cpp @@ -226,6 +226,14 @@ TEST("test stuff") { EXPECT_TRUE(!eval::LazyExpressions::check(p, true)); EXPECT_TRUE(!eval::LazyExpressions::check(p, false)); } + { // vespa.eval.use_fast_forest + EXPECT_EQUAL(eval::UseFastForest::NAME, vespalib::string("vespa.eval.use_fast_forest")); + EXPECT_EQUAL(eval::UseFastForest::DEFAULT_VALUE, false); + Properties p; + EXPECT_EQUAL(eval::UseFastForest::check(p), false); + p.add("vespa.eval.use_fast_forest", "true"); + EXPECT_EQUAL(eval::UseFastForest::check(p), true); + } { // vespa.rank.firstphase EXPECT_EQUAL(rank::FirstPhase::NAME, vespalib::string("vespa.rank.firstphase")); EXPECT_EQUAL(rank::FirstPhase::DEFAULT_VALUE, vespalib::string("nativeRank")); diff --git a/searchlib/src/tests/fef/rank_program/rank_program_test.cpp b/searchlib/src/tests/fef/rank_program/rank_program_test.cpp index 7e28178e5f7..d1b0f8112f3 100644 --- a/searchlib/src/tests/fef/rank_program/rank_program_test.cpp +++ b/searchlib/src/tests/fef/rank_program/rank_program_test.cpp @@ -90,6 +90,10 @@ struct Fixture { value ? "true" : "false"); return *this; } + Fixture &use_fast_forest() { + indexEnv.getProperties().add(indexproperties::eval::UseFastForest::NAME, "true"); + return *this; + } Fixture &add_expr(const vespalib::string &name, const vespalib::string &expr) { vespalib::string feature_name = expr_feature(name); vespalib::string expr_name = feature_name + ".rankingScript"; @@ -113,6 +117,11 @@ struct Fixture { program.setup(*match_data, queryEnv, overrides); return *this; } + vespalib::string final_executor_name() const { + size_t n = program.num_executors(); + ASSERT_TRUE(n > 0); + return program.get_executor(n-1).getClassName(); + } double get(uint32_t docid = default_docid) { auto result = program.get_seeds(); EXPECT_EQUAL(1u, result.num_features()); @@ -360,4 +369,26 @@ TEST_F("require that interpreted ranking expressions are pure", Fixture()) { EXPECT_EQUAL(f1.get(), 7.0); } +const vespalib::string tree_expr = "if(value(1)<2,1,2)+if(value(2)<1,10,20)"; + +TEST_F("require that fast-forest gbdt evaluation can be enabled", Fixture()) { + f1.use_fast_forest().add_expr("rank", tree_expr).compile(); + EXPECT_EQUAL(f1.get(), 21.0); + EXPECT_EQUAL(f1.final_executor_name(), "search::features::FastForestExecutor"); +} + +TEST_F("require that fast-forest gbdt evaluation is disabled by default", Fixture()) { + f1.add_expr("rank", tree_expr).compile(); + EXPECT_EQUAL(f1.get(), 21.0); + EXPECT_EQUAL(f1.final_executor_name(), "search::features::CompiledRankingExpressionExecutor"); +} + +TEST_F("require that fast-forest gbdt evaluation is pure", Fixture()) { + f1.use_fast_forest().add_expr("rank", tree_expr).compile(); + EXPECT_EQUAL(3u, count_features(f1.program)); + EXPECT_EQUAL(3u, count_const_features(f1.program)); + EXPECT_EQUAL(f1.get(), 21.0); + EXPECT_EQUAL(f1.final_executor_name(), "search::features::FastForestExecutor"); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp index 2733ec62105..a4b2280fa57 100644 --- a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp @@ -11,20 +11,21 @@ #include <vespa/log/log.h> LOG_SETUP(".features.rankingexpression"); -using vespalib::eval::Function; -using vespalib::eval::PassParams; +using search::fef::FeatureType; +using vespalib::ArrayRef; +using vespalib::ConstArrayRef; using vespalib::eval::CompileCache; using vespalib::eval::CompiledFunction; +using vespalib::eval::DoubleValue; +using vespalib::eval::Function; using vespalib::eval::InterpretedFunction; using vespalib::eval::LazyParams; -using vespalib::eval::ValueType; -using vespalib::eval::Value; -using vespalib::eval::DoubleValue; using vespalib::eval::NodeTypes; +using vespalib::eval::PassParams; +using vespalib::eval::Value; +using vespalib::eval::ValueType; +using vespalib::eval::gbdt::FastForest; using vespalib::tensor::DefaultTensorEngine; -using search::fef::FeatureType; -using vespalib::ArrayRef; -using vespalib::ConstArrayRef; namespace search::features { @@ -43,6 +44,23 @@ vespalib::string list_issues(const std::vector<vespalib::string> &issues) { //----------------------------------------------------------------------------- /** + * Implements the executor for fast forest gbdt evaluation + **/ +class FastForestExecutor : public fef::FeatureExecutor +{ +private: + const FastForest &_forest; + FastForest::Context _ctx; + +public: + FastForestExecutor(const FastForest &forest); + bool isPure() override { return true; } + void execute(uint32_t docId) override; +}; + +//----------------------------------------------------------------------------- + +/** * Implements the executor for compiled ranking expressions **/ class CompiledRankingExpressionExecutor : public fef::FeatureExecutor @@ -110,6 +128,22 @@ public: //----------------------------------------------------------------------------- +FastForestExecutor::FastForestExecutor(const FastForest &forest) + : _forest(forest), + _ctx(_forest) +{ +} + +void +FastForestExecutor::execute(uint32_t) +{ + const auto ¶ms = inputs(); + double result = _forest.eval(_ctx, [¶ms](size_t p){ return params.get_number(p); }); + outputs().set_number(0, result); +} + +//----------------------------------------------------------------------------- + CompiledRankingExpressionExecutor::CompiledRankingExpressionExecutor(const CompiledFunction &compiled_function) : _ranking_function(compiled_function.get_function()), _params(compiled_function.num_params(), 0.0) @@ -178,6 +212,7 @@ RankingExpressionBlueprint::RankingExpressionBlueprint(rankingexpression::Expres : fef::Blueprint("rankingExpression"), _expression_replacer(std::move(replacer)), _intrinsic_expression(), + _fast_forest(), _interpreted_function(), _compile_token(), _input_is_object() @@ -259,11 +294,17 @@ RankingExpressionBlueprint::setup(const fef::IIndexEnvironment &env, // avoid costly compilation when only verifying setup if (env.getFeatureMotivation() != env.FeatureMotivation::VERIFY_SETUP) { if (do_compile) { - bool suggest_lazy = CompiledFunction::should_use_lazy_params(rank_function); - if (fef::indexproperties::eval::LazyExpressions::check(env.getProperties(), suggest_lazy)) { - _compile_token = CompileCache::compile(rank_function, PassParams::LAZY); - } else { - _compile_token = CompileCache::compile(rank_function, PassParams::ARRAY); + // fast forest evaluation is a possible replacement for compiled tree models + if (fef::indexproperties::eval::UseFastForest::check(env.getProperties())) { + _fast_forest = FastForest::try_convert(rank_function); + } + if (!_fast_forest) { + bool suggest_lazy = CompiledFunction::should_use_lazy_params(rank_function); + if (fef::indexproperties::eval::LazyExpressions::check(env.getProperties(), suggest_lazy)) { + _compile_token = CompileCache::compile(rank_function, PassParams::LAZY); + } else { + _compile_token = CompileCache::compile(rank_function, PassParams::ARRAY); + } } } else { _interpreted_function.reset(new InterpretedFunction(DefaultTensorEngine::ref(), rank_function, node_types)); @@ -300,6 +341,9 @@ RankingExpressionBlueprint::createExecutor(const fef::IQueryEnvironment &env, ve ConstArrayRef<char> input_is_object = stash.copy_array<char>(_input_is_object); return stash.create<InterpretedRankingExpressionExecutor>(*_interpreted_function, input_is_object); } + if (_fast_forest) { + return stash.create<FastForestExecutor>(*_fast_forest); + } assert(_compile_token.get() != nullptr); // will be nullptr for VERIFY_SETUP feature motivation if (_compile_token->get().pass_params() == PassParams::ARRAY) { return stash.create<CompiledRankingExpressionExecutor>(_compile_token->get()); diff --git a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h index 104e8d63a70..579c8cf91a7 100644 --- a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h +++ b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h @@ -2,6 +2,7 @@ #pragma once #include <vespa/searchlib/fef/blueprint.h> +#include <vespa/eval/eval/fast_forest.h> #include <vespa/eval/eval/interpreted_function.h> #include <vespa/eval/eval/llvm/compile_cache.h> #include <vespa/searchlib/features/rankingexpression/expression_replacer.h> @@ -19,6 +20,7 @@ class RankingExpressionBlueprint : public fef::Blueprint private: rankingexpression::ExpressionReplacer::SP _expression_replacer; rankingexpression::IntrinsicExpression::UP _intrinsic_expression; + vespalib::eval::gbdt::FastForest::UP _fast_forest; vespalib::eval::InterpretedFunction::UP _interpreted_function; vespalib::eval::CompileCache::Token::UP _compile_token; std::vector<char> _input_is_object; diff --git a/searchlib/src/vespa/searchlib/fef/indexproperties.cpp b/searchlib/src/vespa/searchlib/fef/indexproperties.cpp index a7df39faf2f..ce1bd69cc4c 100644 --- a/searchlib/src/vespa/searchlib/fef/indexproperties.cpp +++ b/searchlib/src/vespa/searchlib/fef/indexproperties.cpp @@ -84,6 +84,10 @@ LazyExpressions::check(const Properties &props, bool default_value) return lookupBool(props, NAME, default_value); } +const vespalib::string UseFastForest::NAME("vespa.eval.use_fast_forest"); +const bool UseFastForest::DEFAULT_VALUE(false); +bool UseFastForest::check(const Properties &props) { return lookupBool(props, NAME, DEFAULT_VALUE); } + } // namespace eval namespace rank { diff --git a/searchlib/src/vespa/searchlib/fef/indexproperties.h b/searchlib/src/vespa/searchlib/fef/indexproperties.h index 9adf4487ec5..57aa24222a3 100644 --- a/searchlib/src/vespa/searchlib/fef/indexproperties.h +++ b/searchlib/src/vespa/searchlib/fef/indexproperties.h @@ -26,6 +26,13 @@ struct LazyExpressions { static bool check(const Properties &props, bool default_value); }; +// use fast-forest evaluation for gbdt expressions. affects rank/summary/dump +struct UseFastForest { + static const vespalib::string NAME; + static const bool DEFAULT_VALUE; + static bool check(const Properties &props); +}; + } // namespace eval namespace rank { diff --git a/searchlib/src/vespa/searchlib/fef/rank_program.h b/searchlib/src/vespa/searchlib/fef/rank_program.h index 3a92fc874a4..e1014df5ee5 100644 --- a/searchlib/src/vespa/searchlib/fef/rank_program.h +++ b/searchlib/src/vespa/searchlib/fef/rank_program.h @@ -59,6 +59,7 @@ public: ~RankProgram(); size_t num_executors() const { return _executors.size(); } + const FeatureExecutor &get_executor(size_t i) const { return *_executors[i]; } /** * Set up this rank program by creating the needed feature diff --git a/security-utils/src/main/java/com/yahoo/security/SubjectAlternativeName.java b/security-utils/src/main/java/com/yahoo/security/SubjectAlternativeName.java index 29395c75e70..81581c8146c 100644 --- a/security-utils/src/main/java/com/yahoo/security/SubjectAlternativeName.java +++ b/security-utils/src/main/java/com/yahoo/security/SubjectAlternativeName.java @@ -3,10 +3,13 @@ package com.yahoo.security; import org.bouncycastle.asn1.ASN1Encodable; import org.bouncycastle.asn1.DERIA5String; +import org.bouncycastle.asn1.DEROctetString; import org.bouncycastle.asn1.x500.X500Name; import org.bouncycastle.asn1.x509.GeneralName; import org.bouncycastle.asn1.x509.GeneralNames; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.Arrays; import java.util.List; import java.util.Objects; @@ -43,6 +46,10 @@ public class SubjectAlternativeName { return new GeneralName(type.tag, value); } + public SubjectAlternativeName decode() { + return new SubjectAlternativeName(new GeneralName(type.tag, value)); + } + static List<SubjectAlternativeName> fromGeneralNames(GeneralNames generalNames) { return Arrays.stream(generalNames.getNames()).map(SubjectAlternativeName::new).collect(toList()); } @@ -56,6 +63,14 @@ public class SubjectAlternativeName { return DERIA5String.getInstance(name).getString(); case GeneralName.directoryName: return X500Name.getInstance(name).toString(); + case GeneralName.iPAddress: + var octets = DEROctetString.getInstance(name.toASN1Primitive()).getOctets(); + try { + return InetAddress.getByAddress(octets).getHostAddress(); + } catch (UnknownHostException e) { + // Only thrown if IP address is of invalid length, which is an illegal argument + throw new IllegalArgumentException(e); + } default: return name.toString(); } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializer.java b/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializer.java index a6291477942..5487bad24e7 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializer.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializer.java @@ -20,6 +20,7 @@ import java.io.UncheckedIOException; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Set; @@ -132,25 +133,28 @@ public class TransportSecurityOptionsJsonSerializer { options.getCaCertificatesFile().ifPresent(value -> entity.files.caCertificatesFile = value.toString()); options.getCertificatesFile().ifPresent(value -> entity.files.certificatesFile = value.toString()); options.getPrivateKeyFile().ifPresent(value -> entity.files.privateKeyFile = value.toString()); - options.getAuthorizedPeers().ifPresent( authorizedPeers -> { - entity.authorizedPeers = new ArrayList<>(); - for (PeerPolicy peerPolicy : authorizedPeers.peerPolicies()) { - AuthorizedPeer authorizedPeer = new AuthorizedPeer(); - authorizedPeer.name = peerPolicy.policyName(); - authorizedPeer.requiredCredentials = new ArrayList<>(); - for (RequiredPeerCredential requiredPeerCredential : peerPolicy.requiredCredentials()) { - RequiredCredential requiredCredential = new RequiredCredential(); - requiredCredential.field = toField(requiredPeerCredential.field()); - requiredCredential.matchExpression = requiredPeerCredential.pattern().asString(); - authorizedPeer.requiredCredentials.add(requiredCredential); - } - if (!peerPolicy.assumedRoles().isEmpty()) { - authorizedPeer.roles = new ArrayList<>(); - peerPolicy.assumedRoles().forEach(role -> authorizedPeer.roles.add(role.name())); - } - entity.authorizedPeers.add(authorizedPeer); - } - }); + options.getAuthorizedPeers().ifPresent( authorizedPeers -> entity.authorizedPeers = + authorizedPeers.peerPolicies().stream() + // Makes tests stable + .sorted(Comparator.comparing(PeerPolicy::policyName)) + .map(peerPolicy -> { + AuthorizedPeer authorizedPeer = new AuthorizedPeer(); + authorizedPeer.name = peerPolicy.policyName(); + authorizedPeer.requiredCredentials = new ArrayList<>(); + for (RequiredPeerCredential requiredPeerCredential : peerPolicy.requiredCredentials()) { + RequiredCredential requiredCredential = new RequiredCredential(); + requiredCredential.field = toField(requiredPeerCredential.field()); + requiredCredential.matchExpression = requiredPeerCredential.pattern().asString(); + authorizedPeer.requiredCredentials.add(requiredCredential); + } + if (!peerPolicy.assumedRoles().isEmpty()) { + authorizedPeer.roles = new ArrayList<>(); + peerPolicy.assumedRoles().forEach(role -> authorizedPeer.roles.add(role.name())); + } + + return authorizedPeer; + }) + .collect(toList())); if (!options.getAcceptedCiphers().isEmpty()) { entity.acceptedCiphers = options.getAcceptedCiphers(); } diff --git a/storage/src/tests/distributor/updateoperationtest.cpp b/storage/src/tests/distributor/updateoperationtest.cpp index 6bc000f6780..1d6fb5fe2ea 100644 --- a/storage/src/tests/distributor/updateoperationtest.cpp +++ b/storage/src/tests/distributor/updateoperationtest.cpp @@ -42,7 +42,8 @@ struct UpdateOperationTest : Test, DistributorTestUtil { } void replyToMessage(UpdateOperation& callback, DistributorMessageSenderStub& sender, uint32_t index, - uint64_t oldTimestamp, const api::BucketInfo& info = api::BucketInfo(2,4,6)); + uint64_t oldTimestamp, const api::BucketInfo& info = api::BucketInfo(2,4,6), + const api::ReturnCode& result = api::ReturnCode()); std::shared_ptr<UpdateOperation> sendUpdate(const std::string& bucketState); @@ -72,7 +73,7 @@ UpdateOperationTest::sendUpdate(const std::string& bucketState) void UpdateOperationTest::replyToMessage(UpdateOperation& callback, DistributorMessageSenderStub& sender, uint32_t index, - uint64_t oldTimestamp, const api::BucketInfo& info) + uint64_t oldTimestamp, const api::BucketInfo& info, const api::ReturnCode& result) { std::shared_ptr<api::StorageMessage> msg2 = sender.command(index); auto* updatec = dynamic_cast<UpdateCommand*>(msg2.get()); @@ -80,6 +81,7 @@ UpdateOperationTest::replyToMessage(UpdateOperation& callback, DistributorMessag auto* updateR = static_cast<api::UpdateReply*>(reply.get()); updateR->setOldTimestamp(oldTimestamp); updateR->setBucketInfo(info); + updateR->setResult(result); callback.onReceive(sender, std::shared_ptr<StorageReply>(reply.release())); } @@ -163,3 +165,21 @@ TEST_F(UpdateOperationTest, multi_node_inconsistent_timestamp) { EXPECT_EQ(1, metrics.diverging_timestamp_updates.getValue()); } +TEST_F(UpdateOperationTest, test_and_set_failures_increment_tas_metric) { + setupDistributor(2, 2, "distributor:1 storage:1"); + std::shared_ptr<UpdateOperation> cb(sendUpdate("0=1/2/3")); + DistributorMessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + ASSERT_EQ("Update => 0", sender.getCommands(true)); + api::ReturnCode result(api::ReturnCode::TEST_AND_SET_CONDITION_FAILED, "bork bork"); + replyToMessage(*cb, sender, 0, 1234, api::BucketInfo(), result); + + ASSERT_EQ("UpdateReply(id:ns:text/html::1, BucketId(0x0000000000000000), " + "timestamp 100, timestamp of updated doc: 0) " + "ReturnCode(TEST_AND_SET_CONDITION_FAILED, bork bork)", + sender.getLastReply(true)); + + auto& metrics = getDistributor().getMetrics().updates[documentapi::LoadType::DEFAULT]; + EXPECT_EQ(1, metrics.failures.test_and_set_failed.getValue()); +} + diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp index 4adbdd32669..ab6776717aa 100644 --- a/storage/src/vespa/storage/distributor/distributor.cpp +++ b/storage/src/vespa/storage/distributor/distributor.cpp @@ -77,8 +77,7 @@ Distributor::Distributor(DistributorComponentRegister& compReg, _distributorStatusDelegate(compReg, *this, *this), _bucketDBStatusDelegate(compReg, *this, _bucketDBUpdater), _idealStateManager(*this, *_bucketSpaceRepo, *_readOnlyBucketSpaceRepo, compReg, manageActiveBucketCopies), - _externalOperationHandler(*this, *_bucketSpaceRepo, *_readOnlyBucketSpaceRepo, - _idealStateManager, compReg, use_btree_database), + _externalOperationHandler(*this, *_bucketSpaceRepo, *_readOnlyBucketSpaceRepo, _idealStateManager, compReg), _threadPool(threadPool), _initializingIsUp(true), _doneInitializeHandler(doneInitHandler), diff --git a/storage/src/vespa/storage/distributor/externaloperationhandler.cpp b/storage/src/vespa/storage/distributor/externaloperationhandler.cpp index 221c516a56e..6b476ae37c5 100644 --- a/storage/src/vespa/storage/distributor/externaloperationhandler.cpp +++ b/storage/src/vespa/storage/distributor/externaloperationhandler.cpp @@ -30,14 +30,12 @@ ExternalOperationHandler::ExternalOperationHandler(Distributor& owner, DistributorBucketSpaceRepo& bucketSpaceRepo, DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo, const MaintenanceOperationGenerator& gen, - DistributorComponentRegister& compReg, - bool enable_concurrent_gets) + DistributorComponentRegister& compReg) : DistributorComponent(owner, bucketSpaceRepo, readOnlyBucketSpaceRepo, compReg, "External operation handler"), _operationGenerator(gen), _rejectFeedBeforeTimeReached(), // At epoch _non_main_thread_ops_mutex(), - _non_main_thread_ops_owner(owner, getClock()), - _enable_concurrent_gets(enable_concurrent_gets) + _non_main_thread_ops_owner(owner, getClock()) { } diff --git a/storage/src/vespa/storage/distributor/externaloperationhandler.h b/storage/src/vespa/storage/distributor/externaloperationhandler.h index 9db078af198..b64b4bc90cd 100644 --- a/storage/src/vespa/storage/distributor/externaloperationhandler.h +++ b/storage/src/vespa/storage/distributor/externaloperationhandler.h @@ -40,8 +40,7 @@ public: DistributorBucketSpaceRepo& bucketSpaceRepo, DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo, const MaintenanceOperationGenerator&, - DistributorComponentRegister& compReg, - bool enable_concurrent_gets); + DistributorComponentRegister& compReg); ~ExternalOperationHandler() override; @@ -59,7 +58,6 @@ private: TimePoint _rejectFeedBeforeTimeReached; mutable std::mutex _non_main_thread_ops_mutex; OperationOwner _non_main_thread_ops_owner; - bool _enable_concurrent_gets; template <typename Func> void bounce_or_invoke_read_only_op(api::StorageCommand& cmd, diff --git a/storage/src/vespa/storage/distributor/persistence_operation_metric_set.cpp b/storage/src/vespa/storage/distributor/persistence_operation_metric_set.cpp index d3ae5d547ed..457d1e051b9 100644 --- a/storage/src/vespa/storage/distributor/persistence_operation_metric_set.cpp +++ b/storage/src/vespa/storage/distributor/persistence_operation_metric_set.cpp @@ -28,7 +28,9 @@ PersistenceFailuresMetricSet::PersistenceFailuresMetricSet(MetricSet* owner) "being in an inconsistent state or not found", this), notfound("notfound", {}, "The number of operations that failed because the document did not exist", this), concurrent_mutations("concurrent_mutations", {}, "The number of operations that were transiently failed due " - "to a mutating operation already being in progress for its document ID", this) + "to a mutating operation already being in progress for its document ID", this), + test_and_set_failed("test_and_set_failed", {}, "The number of mutating operations that failed because " + "they specified a test-and-set condition that did not match the existing document", this) { sum.addMetricToSum(notready); sum.addMetricToSum(notconnected); @@ -39,6 +41,8 @@ PersistenceFailuresMetricSet::PersistenceFailuresMetricSet(MetricSet* owner) sum.addMetricToSum(busy); sum.addMetricToSum(inconsistent_bucket); sum.addMetricToSum(notfound); + // TaS/concurrent mutation failures not added to the main failure metric, as they're not "failures" as per se. + // TODO introduce separate aggregate for such metrics } PersistenceFailuresMetricSet::~PersistenceFailuresMetricSet() = default; @@ -61,7 +65,7 @@ PersistenceOperationMetricSet::PersistenceOperationMetricSet(const std::string& failures(this) { } -PersistenceOperationMetricSet::~PersistenceOperationMetricSet() { } +PersistenceOperationMetricSet::~PersistenceOperationMetricSet() = default; MetricSet * PersistenceOperationMetricSet::clone(std::vector<Metric::UP>& ownerList, CopyType copyType, @@ -84,6 +88,8 @@ PersistenceOperationMetricSet::updateFromResult(const api::ReturnCode& result) failures.wrongdistributor.inc(); } else if (result.getResult() == api::ReturnCode::TIMEOUT) { failures.timeout.inc(); + } else if (result.getResult() == api::ReturnCode::TEST_AND_SET_CONDITION_FAILED) { + failures.test_and_set_failed.inc(); } else if (result.isBusy()) { failures.busy.inc(); } else if (result.isBucketDisappearance()) { diff --git a/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h b/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h index 4f51c664daf..52249529e4f 100644 --- a/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h +++ b/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h @@ -28,6 +28,7 @@ public: metrics::LongCountMetric inconsistent_bucket; metrics::LongCountMetric notfound; metrics::LongCountMetric concurrent_mutations; + metrics::LongCountMetric test_and_set_failed; MetricSet * clone(std::vector<Metric::UP>& ownerList, CopyType copyType, metrics::MetricSet* owner, bool includeUnused) const override; diff --git a/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java b/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java index 9de06e7f4da..f6a88ec83c2 100644 --- a/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java +++ b/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java @@ -8,13 +8,13 @@ public class ApiAuthenticator implements ai.vespa.hosted.api.ApiAuthenticator { /** Returns a controller client using mTLS if a key and certificate pair is provided, or signed requests otherwise. */ @Override public ControllerHttpClient controller() { - return Properties.certificateFile() - .map(certificateFile -> ControllerHttpClient.withKeyAndCertificate(Properties.endpoint(), - Properties.privateKeyFile(), + return Properties.apiCertificateFile() + .map(certificateFile -> ControllerHttpClient.withKeyAndCertificate(Properties.apiEndpoint(), + Properties.apiPrivateKeyFile(), certificateFile)) .orElseGet(() -> - ControllerHttpClient.withSignatureKey(Properties.endpoint(), - Properties.privateKeyFile(), + ControllerHttpClient.withSignatureKey(Properties.apiEndpoint(), + Properties.apiPrivateKeyFile(), Properties.application())); } diff --git a/tenant-auth/src/main/java/ai/vespa/hosted/auth/EndpointAuthenticator.java b/tenant-auth/src/main/java/ai/vespa/hosted/auth/EndpointAuthenticator.java index c9640763ac8..e51476907e2 100644 --- a/tenant-auth/src/main/java/ai/vespa/hosted/auth/EndpointAuthenticator.java +++ b/tenant-auth/src/main/java/ai/vespa/hosted/auth/EndpointAuthenticator.java @@ -1,5 +1,6 @@ package ai.vespa.hosted.auth; +import ai.vespa.hosted.api.Properties; import com.yahoo.config.provision.SystemName; import com.yahoo.security.KeyUtils; import com.yahoo.security.SslContextBuilder; @@ -47,12 +48,10 @@ public class EndpointAuthenticator implements ai.vespa.hosted.api.EndpointAuthen privateKeyFile = credentialsRoot.resolve("key"); } else { - Optional<String> certificateFileProperty = getNonBlankProperty("dataPlaneCertificateFile"); - if (certificateFileProperty.isPresent()) - certificateFile = Path.of(certificateFileProperty.get()); - Optional<String> privateKeyFileProperty = getNonBlankProperty("dataPlaneKeyFile"); - if (privateKeyFileProperty.isPresent()) - privateKeyFile = Path.of(privateKeyFileProperty.get()); + if (Properties.dataPlaneCertificateFile().isPresent()) + certificateFile = Properties.dataPlaneCertificateFile().get(); + if (Properties.dataPlanePrivateKeyFile().isPresent()) + privateKeyFile = Properties.dataPlanePrivateKeyFile().get(); } if (certificateFile != null && privateKeyFile != null) { X509Certificate certificate = X509CertificateUtils.fromPem(new String(Files.readAllBytes(certificateFile))); @@ -67,7 +66,7 @@ public class EndpointAuthenticator implements ai.vespa.hosted.api.EndpointAuthen logger.warning( "##################################################################################\n" + "# Data plane key and/or certificate missing; please specify #\n" + "# '-DdataPlaneCertificateFile=/path/to/certificate' and #\n" - + "# '-DdataPlaneKeyFile=/path/to/private_key. #\n" + + "# '-DdataPlaneKeyFile=/path/to/private_key'. #\n" + "# Trying the default SSLContext, but this will most likely cause HTTP error 401. #\n" + "##################################################################################"); return SSLContext.getDefault(); diff --git a/vdslib/src/tests/distribution/distributiontest.cpp b/vdslib/src/tests/distribution/distributiontest.cpp index 80f28af17b5..c43735e7e41 100644 --- a/vdslib/src/tests/distribution/distributiontest.cpp +++ b/vdslib/src/tests/distribution/distributiontest.cpp @@ -13,7 +13,6 @@ #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/stllike/lexical_cast.h> #include <vespa/vespalib/text/stringtokenizer.h> -#include <vespa/vespalib/util/regexp.h> #include <chrono> #include <thread> #include <fstream> diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java index 9bd995ef106..845d0ba4c1b 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java @@ -1,6 +1,7 @@ package ai.vespa.hosted.plugin; import ai.vespa.hosted.api.ControllerHttpClient; +import ai.vespa.hosted.api.Properties; import com.yahoo.config.provision.ApplicationId; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; @@ -69,7 +70,7 @@ public abstract class AbstractVespaMojo extends AbstractMojo { protected void setup() { tenant = firstNonBlank(tenant, project.getProperties().getProperty("tenant")); application = firstNonBlank(application, project.getProperties().getProperty("application")); - instance = firstNonBlank(instance, project.getProperties().getProperty("instance", "default")); + instance = firstNonBlank(instance, project.getProperties().getProperty("instance", Properties.user())); id = ApplicationId.from(tenant, application, instance); if (privateKey != null) { |