diff options
author | Jon Bratseth <jonbratseth@yahoo.com> | 2017-10-15 11:34:33 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-10-15 11:34:33 +0200 |
commit | 44f2511975cf77bd8bfb931154ba088d65eedefb (patch) | |
tree | 883eb51cb6504129087daffb9d910bd2ac076a84 | |
parent | 9fc068e627ae71c5478a91fc06fff4d62933efa1 (diff) | |
parent | 5e8192a1919137822b4b042a68b633b36e86436c (diff) |
Merge pull request #3746 from vespa-engine/mpolden/freeze-confidence
Freeze confidence for previous versions
14 files changed, 361 insertions, 161 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 4f5c2ead4da..35551b50953 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; +import com.yahoo.component.Vtag; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; @@ -40,7 +41,6 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Logger; /** @@ -64,15 +64,7 @@ public class Controller extends AbstractComponent { private final CuratorDb curator; private final ApplicationController applicationController; private final TenantController tenantController; - - /** - * Status of Vespa versions across the system. - * This is expensive to maintain so that is done periodically by a maintenance job - */ - private final AtomicReference<VersionStatus> versionStatus; - private final Clock clock; - private final RotationRepository rotationRepository; private final GitHub gitHub; private final EntityService entityService; @@ -141,7 +133,6 @@ public class Controller extends AbstractComponent { applicationController = new ApplicationController(this, db, curator, rotationRepository, athens.zmsClientFactory(), nameService, configServerClient, routingGenerator, clock); tenantController = new TenantController(this, db, curator, entityService); - versionStatus = new AtomicReference<>(VersionStatus.empty()); } /** Returns the instance controlling tenants */ @@ -191,11 +182,7 @@ public class Controller extends AbstractComponent { "sort:!('@timestamp',desc))"; URI kibanaPath = URI.create(kibanaQuery); - if (kibanaHost.isPresent()) { - return kibanaHost.get().resolve(kibanaPath); - } else { - return null; - } + return kibanaHost.map(uri -> uri.resolve(kibanaPath)).orElse(null); } public Set<URI> getRotationUris(ApplicationId id) { @@ -235,17 +222,19 @@ public class Controller extends AbstractComponent { ! newStatus.systemVersion().equals(currentStatus.systemVersion())) { log.info("Changing system version from " + printableVersion(currentStatus.systemVersion()) + " to " + printableVersion(newStatus.systemVersion())); - curator.writeSystemVersion(newStatus.systemVersion().get().versionNumber()); } - - this.versionStatus.set(newStatus); + curator.writeVersionStatus(newStatus); } /** Returns the latest known version status. Calling this is free but the status may be slightly out of date. */ - public VersionStatus versionStatus() { return versionStatus.get(); } + public VersionStatus versionStatus() { return curator.readVersionStatus(); } /** Returns the current system version: The controller should drive towards running all applications on this version */ - public Version systemVersion() { return curator.readSystemVersion(); } + public Version systemVersion() { + return versionStatus().systemVersion() + .map(VespaVersion::versionNumber) + .orElse(Vtag.currentVersion); + } public MetricsService metricsService() { return metricsService; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index a70e31d9de8..2ca662ccb66 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -4,20 +4,21 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.google.inject.Inject; import com.yahoo.cloud.config.ClusterInfoConfig; import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.component.Version; -import com.yahoo.component.Vtag; import com.yahoo.config.provision.ApplicationId; import com.yahoo.net.HostName; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.zookeeper.ZooKeeperServer; +import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayDeque; import java.util.Collections; @@ -148,18 +149,6 @@ public class CuratorDb { // -------------- Read and write -------------------------------------------------- - public Version readSystemVersion() { - Optional<byte[]> data = curator.getData(systemVersionPath()); - if (! data.isPresent() || data.get().length == 0) return Vtag.currentVersion; - return Version.fromString(new String(data.get(), StandardCharsets.UTF_8)); - } - - public void writeSystemVersion(Version version) { - NestedTransaction transaction = new NestedTransaction(); - curator.set(systemVersionPath(), version.toString().getBytes(StandardCharsets.UTF_8)); - transaction.commit(); - } - public Set<String> readInactiveJobs() { try { Optional<byte[]> data = curator.getData(inactiveJobsPath()); @@ -199,7 +188,7 @@ public class CuratorDb { } public double readUpgradesPerMinute() { - Optional<byte[]> n = curator.getData(upgradePerMinutePath()); + Optional<byte[]> n = curator.getData(upgradesPerMinutePath()); if (!n.isPresent() || n.get().length == 0) { return 0.5; // Default if value has never been written } @@ -211,10 +200,34 @@ public class CuratorDb { throw new IllegalArgumentException("Upgrades per minute must be >= 0"); } NestedTransaction transaction = new NestedTransaction(); - curator.set(upgradePerMinutePath(), ByteBuffer.allocate(Double.BYTES).putDouble(n).array()); + curator.set(upgradesPerMinutePath(), ByteBuffer.allocate(Double.BYTES).putDouble(n).array()); + transaction.commit(); + } + + public void writeVersionStatus(VersionStatus status) { + VersionStatusSerializer serializer = new VersionStatusSerializer(); + NestedTransaction transaction = new NestedTransaction(); + try { + // TODO: Removes unused data. Remove after October 2017 + if (curator.getData(systemVersionPath()).isPresent()) { + curator.delete(systemVersionPath()); + } + curator.set(versionStatusPath(), SlimeUtils.toJsonBytes(serializer.toSlime(status))); + } catch (IOException e) { + throw new UncheckedIOException("Failed to serialize version status", e); + } transaction.commit(); } + public VersionStatus readVersionStatus() { + Optional<byte[]> data = curator.getData(versionStatusPath()); + if (!data.isPresent() || data.get().length == 0) { + return VersionStatus.empty(); // Default if status has never been written + } + VersionStatusSerializer serializer = new VersionStatusSerializer(); + return serializer.fromSlime(SlimeUtils.jsonToSlime(data.get())); + } + public Optional<byte[]> readProvisionState(String provisionId) { return curator.getData(provisionStatePath().append(provisionId)); } @@ -264,10 +277,12 @@ public class CuratorDb { return root.append("jobQueues").append(jobType.name()); } - private Path upgradePerMinutePath() { + private Path upgradesPerMinutePath() { return root.append("upgrader").append("upgradesPerMinute"); } + private Path versionStatusPath() { return root.append("versionStatus"); } + private Path provisionStatePath() { return root.append("provisioning").append("states"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java new file mode 100644 index 00000000000..e0bc592c82c --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java @@ -0,0 +1,118 @@ +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics; +import com.yahoo.vespa.hosted.controller.versions.VersionStatus; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * Serializes VersionStatus to and from slime + * + * @author mpolden + */ +public class VersionStatusSerializer { + + // VersionStatus fields + private static final String versionsField = "versions"; + + // VespaVersion fields + private static final String releaseCommitField = "releaseCommit"; + private static final String releasedAtField = "releasedAt"; + private static final String isCurrentSystemVersionField = "isCurrentSystemVersion"; + private static final String deploymentStatisticsField = "deploymentStatistics"; + private static final String confidenceField = "confidence"; + private static final String configServersField = "configServerHostnames"; + + // DeploymentStatistics fields + private static final String versionField = "version"; + private static final String failingField = "failing"; + private static final String productionField = "production"; + + public Slime toSlime(VersionStatus status) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + versionsToSlime(status.versions(), root.setArray(versionsField)); + return slime; + } + + public VersionStatus fromSlime(Slime slime) { + Inspector root = slime.get(); + return new VersionStatus(vespaVersionsFromSlime(root.field(versionsField))); + } + + private void versionsToSlime(List<VespaVersion> versions, Cursor array) { + versions.forEach(version -> vespaVersionToSlime(version, array.addObject())); + } + + private void vespaVersionToSlime(VespaVersion version, Cursor object) { + object.setString(releaseCommitField, version.releaseCommit()); + object.setLong(releasedAtField, version.releasedAt().toEpochMilli()); + object.setBool(isCurrentSystemVersionField, version.isCurrentSystemVersion()); + deploymentStatisticsToSlime(version.statistics(), object.setObject(deploymentStatisticsField)); + object.setString(confidenceField, version.confidence().name()); + configServersToSlime(version.configServerHostnames(), object.setArray(configServersField)); + } + + private void configServersToSlime(Set<String> configServerHostnames, Cursor array) { + configServerHostnames.forEach(array::addString); + } + + private void deploymentStatisticsToSlime(DeploymentStatistics statistics, Cursor object) { + object.setString(versionField, statistics.version().toString()); + applicationsToSlime(statistics.failing(), object.setArray(failingField)); + applicationsToSlime(statistics.production(), object.setArray(productionField)); + } + + private void applicationsToSlime(List<ApplicationId> applications, Cursor array) { + applications.forEach(application -> array.addString(application.serializedForm())); + } + + private List<VespaVersion> vespaVersionsFromSlime(Inspector array) { + List<VespaVersion> versions = new ArrayList<>(); + array.traverse((ArrayTraverser) (i, object) -> versions.add(vespaVersionFromSlime(object))); + return Collections.unmodifiableList(versions); + } + + private VespaVersion vespaVersionFromSlime(Inspector object) { + return new VespaVersion(deploymentStatisticsFromSlime(object.field(deploymentStatisticsField)), + object.field(releaseCommitField).asString(), + Instant.ofEpochMilli(object.field(releasedAtField).asLong()), + object.field(isCurrentSystemVersionField).asBool(), + configServersFromSlime(object.field(configServersField)), + VespaVersion.Confidence.valueOf(object.field(confidenceField).asString()) + ); + } + + private Set<String> configServersFromSlime(Inspector array) { + Set<String> configServerHostnames = new LinkedHashSet<>(); + array.traverse((ArrayTraverser) (i, entry) -> configServerHostnames.add(entry.asString())); + return Collections.unmodifiableSet(configServerHostnames); + } + + private DeploymentStatistics deploymentStatisticsFromSlime(Inspector object) { + return new DeploymentStatistics(Version.fromString(object.field(versionField).asString()), + applicationsFromSlime(object.field(failingField)), + applicationsFromSlime(object.field(productionField))); + } + + private List<ApplicationId> applicationsFromSlime(Inspector array) { + List<ApplicationId> applications = new ArrayList<>(); + array.traverse((ArrayTraverser) (i, entry) -> applications.add( + ApplicationId.fromSerializedForm(entry.asString())) + ); + return Collections.unmodifiableList(applications); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java index fbd1a74c12c..6174a017a54 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java @@ -6,6 +6,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import java.util.List; +import java.util.Objects; /** * Statistics about deployments on a platform version. This is immutable. @@ -18,8 +19,9 @@ public class DeploymentStatistics { private final ImmutableList<ApplicationId> failing; private final ImmutableList<ApplicationId> production; - private DeploymentStatistics(Version version, - List<ApplicationId> failingApplications, List<ApplicationId> production) { + /** DO NOT USE. Public for serialization purposes */ + public DeploymentStatistics(Version version, List<ApplicationId> failingApplications, + List<ApplicationId> production) { this.version = version; this.failing = ImmutableList.copyOf(failingApplications); this.production = ImmutableList.copyOf(production); @@ -59,4 +61,18 @@ public class DeploymentStatistics { return b.build(); } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof DeploymentStatistics)) return false; + DeploymentStatistics that = (DeploymentStatistics) o; + return Objects.equals(version, that.version) && + Objects.equals(failing, that.failing) && + Objects.equals(production, that.production); + } + + @Override + public int hashCode() { + return Objects.hash(version, failing, production); + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index c3f3773cde6..c3b2da42861 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -5,9 +5,9 @@ import com.google.common.collect.ImmutableList; import com.yahoo.collections.ListMap; import com.yahoo.component.Version; import com.yahoo.component.Vtag; -import com.yahoo.vespa.hosted.controller.api.integration.github.GitSha; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.github.GitSha; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; @@ -46,7 +46,7 @@ public class VersionStatus { private final ImmutableList<VespaVersion> versions; - /** Create a version status. DO NOT USE: Public for testing only */ + /** Create a version status. DO NOT USE: Public for testing and serialization only */ public VersionStatus(List<VespaVersion> versions) { this.versions = ImmutableList.copyOf(versions); } @@ -175,11 +175,30 @@ public class VersionStatus { Collection<String> configServerHostnames, Controller controller) { GitSha gitSha = controller.gitHub().getCommit(VESPA_REPO_OWNER, VESPA_REPO, statistics.version().toFullString()); + Instant releasedAt = Instant.ofEpochMilli(gitSha.commit.author.date.getTime()); + VespaVersion.Confidence confidence; + // Always compute confidence for system version + if (isSystemVersion) { + confidence = VespaVersion.confidenceFrom(statistics, controller, releasedAt); + } else { + // Keep existing confidence for non-system versions if already computed + confidence = confidenceFor(statistics.version(), controller) + .orElse(VespaVersion.confidenceFrom(statistics, controller, releasedAt)); + } return new VespaVersion(statistics, - gitSha.sha, Instant.ofEpochMilli(gitSha.commit.author.date.getTime()), + gitSha.sha, releasedAt, isSystemVersion, configServerHostnames, - controller); + confidence + ); + } + + /** Returns the current confidence for the given version */ + private static Optional<VespaVersion.Confidence> confidenceFor(Version version, Controller controller) { + return controller.versionStatus().versions().stream() + .filter(v -> version.equals(v.versionNumber())) + .map(VespaVersion::confidence) + .findFirst(); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index 3208f4d09c6..c1b9c045fbe 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.versions; import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.component.Vtag; -import com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.ApplicationList; @@ -12,6 +11,8 @@ import java.time.Instant; import java.util.Collection; import java.util.Set; +import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy; + /** * Information about a particular Vespa version. * VespaVersions are identified by their version number and ordered by increasing version numbers. @@ -26,23 +27,22 @@ public class VespaVersion implements Comparable<VespaVersion> { private final Instant releasedAt; private final boolean isCurrentSystemVersion; private final DeploymentStatistics statistics; - private final Confidence confidence; private final ImmutableSet<String> configServerHostnames; + private final Confidence confidence; - public VespaVersion(DeploymentStatistics statistics, String releaseCommit, Instant releasedAt, + public VespaVersion(DeploymentStatistics statistics, String releaseCommit, Instant releasedAt, boolean isCurrentSystemVersion, Collection<String> configServerHostnames, - Controller controller) { + Confidence confidence) { this.statistics = statistics; this.releaseCommit = releaseCommit; this.releasedAt = releasedAt; this.isCurrentSystemVersion = isCurrentSystemVersion; this.configServerHostnames = ImmutableSet.copyOf(configServerHostnames); - this.confidence = deduceConfidenceFrom(statistics, controller, releasedAt); + this.confidence = confidence; } - private static Confidence deduceConfidenceFrom(DeploymentStatistics statistics, - Controller controller, - Instant releasedAt) { + public static Confidence confidenceFrom(DeploymentStatistics statistics, Controller controller, + Instant releasedAt) { // 'production on this': All deployment jobs upgrading to this version have completed without failure ApplicationList productionOnThis = ApplicationList.from(statistics.production(), controller.applications()) .notUpgradingTo(statistics.version()) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java index 9db852374b8..9228e83bbc6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java @@ -7,16 +7,16 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeList; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeList; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.rotation.Rotation; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.vespa.serviceview.bindings.ClusterView; @@ -28,7 +28,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -40,15 +39,11 @@ import java.util.UUID; */ public class ConfigServerClientMock extends AbstractComponent implements ConfigServerClient { - private Map<ApplicationId, byte[]> applicationContent = new HashMap<>(); - private Map<ApplicationId, String> applicationInstances = new HashMap<>(); - private Map<ApplicationId, Boolean> applicationActivated = new HashMap<>(); - private Set<ApplicationId> applicationRestarted = new HashSet<>(); - private Set<String> hostsExplicitlyRestarted = new HashSet<>(); - private Map<String, EndpointStatus> endpoints = new HashMap<>(); - - private Map<URI, Version> configServerVersions = new HashMap<>(); - private Version defaultConfigServerVersion = new Version(6, 1, 0); + private final Map<ApplicationId, String> applicationInstances = new HashMap<>(); + private final Map<ApplicationId, Boolean> applicationActivated = new HashMap<>(); + private final Map<String, EndpointStatus> endpoints = new HashMap<>(); + private final Map<URI, Version> versions = new HashMap<>(); + private Version defaultVersion = new Version(6, 1, 0); /** The exception to throw on the next prepare run, or null to continue normally */ private RuntimeException prepareException = null; @@ -64,18 +59,27 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS public Map<ApplicationId, Boolean> activated() { return Collections.unmodifiableMap(applicationActivated); } + + public void throwOnNextPrepare(RuntimeException prepareException) { + this.prepareException = prepareException; + } + + /** + * Returns the (initially empty) mutable map of config server urls to versions. + * This API will return defaultVersion as response to any version(url) call for versions not added to the map. + */ + public Map<URI, Version> versions() { + return versions; + } @Override public PreparedApplication prepare(DeploymentId deployment, DeployOptions deployOptions, Set<String> rotationCnames, Set<Rotation> rotations, byte[] content) { lastPrepareVersion = deployOptions.vespaVersion.map(Version::new); - if (prepareException != null) { RuntimeException prepareException = this.prepareException; this.prepareException = null; throw prepareException; } - - applicationContent.put(deployment.applicationId(), content); applicationActivated.put(deployment.applicationId(), false); applicationInstances.put(deployment.applicationId(), UUID.randomUUID() + ":4080"); @@ -111,20 +115,8 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS }; } - public void throwOnNextPrepare(RuntimeException prepareException) { - this.prepareException = prepareException; - } - - /** - * Returns the (initially empty) mutable map of config server urls to versions. - * This API will return defaultConfigserverVersion as response to any version(url) call for versions not added to the map. - */ - public Map<URI, Version> configServerVersions() { - return configServerVersions; - } - - public Version getDefaultConfigServerVersion() { return defaultConfigServerVersion; } - public void setDefaultConfigServerVersion(Version version) { defaultConfigServerVersion = version; } + /** Set the default config server version */ + public void setDefaultVersion(Version version) { this.defaultVersion = version; } @Override public List<String> getNodeQueryHost(DeploymentId deployment, String type) { @@ -137,16 +129,11 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS @Override public void restart(DeploymentId deployment, Optional<Hostname> hostname) { - applicationRestarted.add(deployment.applicationId()); - if (hostname.isPresent()) { - hostsExplicitlyRestarted.add(hostname.get().id()); - } } @Override public void deactivate(DeploymentId deployment) { applicationActivated.remove(deployment.applicationId()); - applicationContent.remove(deployment.applicationId()); applicationInstances.remove(deployment.applicationId()); } @@ -197,7 +184,7 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS @Override public Version version(URI configServerURI) { - return configServerVersions.getOrDefault(configServerURI, defaultConfigServerVersion); + return versions.getOrDefault(configServerURI, defaultVersion); } @Override @@ -207,10 +194,8 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS @Override public EndpointStatus getGlobalRotationStatus(DeploymentId deployment, String endpoint) { - EndpointStatus result = new EndpointStatus(EndpointStatus.Status.in, "", "", 1497618757l); - return endpoints.containsKey(endpoint) - ? endpoints.get(endpoint) - : result; + EndpointStatus result = new EndpointStatus(EndpointStatus.Status.in, "", "", 1497618757L); + return endpoints.getOrDefault(endpoint, result); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 6fc787d940e..b193fd7ef41 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -260,16 +260,19 @@ public class ControllerTest { Version systemVersion = controller.versionStatus().systemVersion().get().versionNumber(); Version newSystemVersion = new Version(systemVersion.getMajor(), systemVersion.getMinor()+1, 0); VespaVersion newSystemVespaVersion = new VespaVersion(DeploymentStatistics.empty(newSystemVersion), - "commit1", + "commit1", Instant.now(), true, Collections.emptyList(), - controller); + VespaVersion.Confidence.low + ); List<VespaVersion> versions = new ArrayList<>(controller.versionStatus().versions()); for (int i = 0; i < versions.size(); i++) { VespaVersion c = versions.get(i); if (c.isCurrentSystemVersion()) - versions.set(i, new VespaVersion(c.statistics(), c.releaseCommit(), c.releasedAt(), false, c.configServerHostnames(), controller)); + versions.set(i, new VespaVersion(c.statistics(), c.releaseCommit(), c.releasedAt(), + false, c.configServerHostnames(), + c.confidence())); } versions.add(newSystemVespaVersion); controller.updateVersionStatus(new VersionStatus(versions)); @@ -542,7 +545,7 @@ public class ControllerTest { // Current system version, matches version in test data Version version = Version.fromString("6.141.117"); - tester.configServer().setDefaultConfigServerVersion(version); + tester.configServer().setDefaultVersion(version); tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); @@ -572,7 +575,7 @@ public class ControllerTest { // New version is released version = Version.fromString("6.142.1"); - tester.configServer().setDefaultConfigServerVersion(version); + tester.configServer().setDefaultVersion(version); tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); tester.upgrader().maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index be14947de2b..e45166b1c16 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -90,12 +90,17 @@ public class DeploymentTester { .filter(c -> c instanceof Change.VersionChange) .map(Change.VersionChange.class::cast); } + + public void updateVersionStatus() { + controller().updateVersionStatus(VersionStatus.compute(controller(), tester.controller().systemVersion())); + } public void updateVersionStatus(Version currentVersion) { controller().updateVersionStatus(VersionStatus.compute(controller(), currentVersion)); } public void upgradeSystem(Version version) { + controllerTester().configServer().setDefaultVersion(version); updateVersionStatus(version); upgrader().maintain(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java index 286db864c22..4c935747ac6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java @@ -181,7 +181,7 @@ public class FailureRedeployerTest { // Current system version, matches version in test data Version version = Version.fromString("6.141.117"); - tester.configServer().setDefaultConfigServerVersion(version); + tester.configServer().setDefaultVersion(version); tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); @@ -200,7 +200,7 @@ public class FailureRedeployerTest { // New version is released version = Version.fromString("6.142.1"); - tester.configServer().setDefaultConfigServerVersion(version); + tester.configServer().setDefaultVersion(version); tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); tester.upgrader().maintain(); @@ -237,7 +237,7 @@ public class FailureRedeployerTest { // Current system version, matches version in test data Version version = Version.fromString("6.42.1"); - tester.configServer().setDefaultConfigServerVersion(version); + tester.configServer().setDefaultVersion(version); tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); @@ -262,7 +262,7 @@ public class FailureRedeployerTest { // Current system version, matches version in test data Version version = Version.fromString("6.42.1"); - tester.configServer().setDefaultConfigServerVersion(version); + tester.configServer().setDefaultVersion(version); tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java new file mode 100644 index 00000000000..feaf1289853 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java @@ -0,0 +1,53 @@ +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics; +import com.yahoo.vespa.hosted.controller.versions.VersionStatus; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; +import org.junit.Test; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class VersionStatusSerializerTest { + + @Test + public void testSerialization() throws Exception { + List<VespaVersion> vespaVersions = new ArrayList<>(); + DeploymentStatistics statistics = new DeploymentStatistics( + Version.fromString("5.0"), + Arrays.asList(ApplicationId.from("tenant1", "failing1", "default")), + Arrays.asList(ApplicationId.from("tenant2", "success1", "default"), + ApplicationId.from("tenant2", "success2", "default")) + ); + vespaVersions.add(new VespaVersion(statistics, "dead", Instant.now(), false, + Arrays.asList("cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); + vespaVersions.add(new VespaVersion(statistics, "cafe", Instant.now(), true, + Arrays.asList("cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); + VersionStatus status = new VersionStatus(vespaVersions); + VersionStatusSerializer serializer = new VersionStatusSerializer(); + VersionStatus deserialized = serializer.fromSlime(serializer.toSlime(status)); + + assertEquals(status.versions().size(), deserialized.versions().size()); + for (int i = 0; i < status.versions().size(); i++) { + VespaVersion a = status.versions().get(i); + VespaVersion b = deserialized.versions().get(i); + assertEquals(a.releaseCommit(), b.releaseCommit()); + assertEquals(a.releasedAt(), b.releasedAt()); + assertEquals(a.isCurrentSystemVersion(), b.isCurrentSystemVersion()); + assertEquals(a.statistics(), b.statistics()); + assertEquals(a.configServerHostnames(), b.configServerHostnames()); + assertEquals(a.confidence(), b.confidence()); + } + + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index c002c7fb24a..55a4b46f4a7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -85,7 +85,8 @@ public class DeploymentApiTest extends ControllerContainerTest { version.releasedAt(), version.isCurrentSystemVersion(), ImmutableSet.of("config1.test", "config2.test"), - controller); + VespaVersion.confidenceFrom(version.statistics(), controller, version.releasedAt()) + ); censored.add(version); } return new VersionStatus(censored); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json index c2e83373cf7..00bd1ed8208 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json @@ -2,7 +2,7 @@ "versions": [ { "version": "(ignore)", - "confidence": "normal", + "confidence": "high", "commit": "(ignore)", "date": 0, "controllerVersion": false, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index 7bbbf8f0499..17935906186 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -6,6 +6,7 @@ import com.yahoo.component.Vtag; import com.yahoo.config.provision.Environment; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; +import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; @@ -13,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; import org.junit.Test; import java.net.URI; @@ -46,7 +48,7 @@ public class VersionStatusTest { public void testSystemVersionIsControllerVersionIfConfigserversAreNewer() { ControllerTester tester = new ControllerTester(); Version largerThanCurrent = new Version(Vtag.currentVersion.getMajor() + 1); - tester.configServer().setDefaultConfigServerVersion(largerThanCurrent); + tester.configServer().setDefaultVersion(largerThanCurrent); VersionStatus versionStatus = VersionStatus.compute(tester.controller()); assertEquals(Vtag.currentVersion, versionStatus.systemVersion().get().versionNumber()); } @@ -55,7 +57,7 @@ public class VersionStatusTest { public void testSystemVersionIsVersionOfOldestConfigServer() throws URISyntaxException { ControllerTester tester = new ControllerTester(); Version oldest = new Version(5); - tester.configServer().configServerVersions().put(new URI("http://cfg.prod.corp-us-east-1.test"), oldest); + tester.configServer().versions().put(new URI("http://cfg.prod.corp-us-east-1.test"), oldest); VersionStatus versionStatus = VersionStatus.compute(tester.controller()); assertEquals(oldest, versionStatus.systemVersion().get().versionNumber()); } @@ -70,7 +72,6 @@ public class VersionStatusTest { .region("us-east-3") .build(); - // Application versions which are older than the current version Version version1 = new Version("5.1"); Version version2 = new Version("5.2"); tester.upgradeSystem(version1); @@ -90,14 +91,9 @@ public class VersionStatusTest { // - app3 is in production on version1, but then fails in staging test on version2 tester.completeUpgradeWithError(app3, version2, applicationPackage, stagingTest); - VersionStatus versionStatus = VersionStatus.compute(tester.controller()); - List<VespaVersion> versions = versionStatus.versions(); - assertEquals("The version of this controller, the default config server version, plus the two versions above exist", 4, versions.size()); - - VespaVersion v0 = versions.get(2); - assertEquals(tester.configServer().getDefaultConfigServerVersion(), v0.versionNumber()); - assertEquals(0, v0.statistics().failing().size()); - assertEquals(0, v0.statistics().production().size()); + tester.updateVersionStatus(); + List<VespaVersion> versions = tester.controller().versionStatus().versions(); + assertEquals("The two versions above exist", 2, versions.size()); VespaVersion v1 = versions.get(0); assertEquals(version1, v1.versionNumber()); @@ -116,11 +112,6 @@ public class VersionStatusTest { // Only one application is on v2 in at least one zone assertEquals(1, v2.statistics().production().size()); assertTrue(v2.statistics().production().contains(app2.id())); - - VespaVersion v3 = versions.get(3); - assertEquals(Vtag.currentVersion, v3.versionNumber()); - assertEquals(0, v3.statistics().failing().size()); - assertEquals(0, v3.statistics().production().size()); } @Test @@ -130,7 +121,7 @@ public class VersionStatusTest { Version version0 = new Version("5.0"); tester.upgradeSystem(version0); - // Setup applications + // Setup applications - all running on version0 Application canary0 = tester.createAndDeploy("canary0", 1, "canary"); Application canary1 = tester.createAndDeploy("canary1", 2, "canary"); Application canary2 = tester.createAndDeploy("canary2", 3, "canary"); @@ -146,8 +137,7 @@ public class VersionStatusTest { Application default9 = tester.createAndDeploy("default9", 13, "default"); Application conservative0 = tester.createAndDeploy("conservative1", 14, "conservative"); - - // The following applications should not affect confidence calculation: + // Applications that do not affect confidence calculation: // Application without deployment Application ignored0 = tester.createApplication("ignored0", "tenant1", 1000, 1000L); @@ -157,30 +147,40 @@ public class VersionStatusTest { "ignored1", "default-pr42", 1000); + assertEquals("All applications running on this version: High", + Confidence.high, confidence(tester.controller(), version0)); + + // New version is released Version version1 = new Version("5.1"); - Version version2 = new Version("5.2"); tester.upgradeSystem(version1); // Canaries upgrade to new versions and fail tester.completeUpgrade(canary0, version1, "canary"); tester.completeUpgradeWithError(canary1, version1, "canary", productionUsWest1); - tester.upgradeSystem(version2); - tester.completeUpgrade(canary2, version2, "canary"); - - VersionStatus versionStatus = VersionStatus.compute(tester.controller()); - List<VespaVersion> versions = versionStatus.versions(); - + tester.updateVersionStatus(); assertEquals("One canary failed: Broken", - VespaVersion.Confidence.broken, confidence(versions, version1)); - assertEquals("Nothing has failed but not all canaries has deployed: Low", - VespaVersion.Confidence.low, confidence(versions, version2)); - assertEquals("Current version of this - no deployments: Low", - VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion)); + Confidence.broken, confidence(tester.controller(), version1)); - // All canaries are upgraded to version2 which raises confidence to normal and more apps upgrade + // New version is released + Version version2 = new Version("5.2"); + tester.upgradeSystem(version2); + assertEquals("Confidence defaults to low for version with no applications", + Confidence.low, confidence(tester.controller(), version2)); + + // All canaries upgrade successfully tester.completeUpgrade(canary0, version2, "canary"); tester.completeUpgrade(canary1, version2, "canary"); + + assertEquals("Confidence for remains unchanged for version1: Broken", + Confidence.broken, confidence(tester.controller(), version1)); + assertEquals("Nothing has failed but not all canaries have upgraded: Low", + Confidence.low, confidence(tester.controller(), version2)); + + // Remaining canary upgrades to version2 which raises confidence to normal and more apps upgrade + tester.completeUpgrade(canary2, version2, "canary"); tester.upgradeSystem(version2); + assertEquals("Canaries have upgraded: Normal", + Confidence.normal, confidence(tester.controller(), version2)); tester.completeUpgrade(default0, version2, "default"); tester.completeUpgrade(default1, version2, "default"); tester.completeUpgrade(default2, version2, "default"); @@ -189,29 +189,27 @@ public class VersionStatusTest { tester.completeUpgrade(default5, version2, "default"); tester.completeUpgrade(default6, version2, "default"); tester.completeUpgrade(default7, version2, "default"); + tester.updateVersionStatus(); - versionStatus = VersionStatus.compute(tester.controller()); - versions = versionStatus.versions(); + // Remember confidence across restart + tester.restartController(); - assertEquals("No deployments: Low", - VespaVersion.Confidence.low, confidence(versions, version0)); + assertEquals("Confidence remains unchanged for version0: High", + Confidence.high, confidence(tester.controller(), version0)); assertEquals("All canaries deployed + < 90% of defaults: Normal", - VespaVersion.Confidence.normal, confidence(versions, version2)); - assertEquals("Current version of this - no deployments: Low", - VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion)); + Confidence.normal, confidence(tester.controller(), version2)); + assertTrue("Status for version without applications is removed", + tester.controller().versionStatus().versions().stream() + .noneMatch(vespaVersion -> vespaVersion.versionNumber().equals(version1))); // Another default application upgrades, raising confidence to high tester.completeUpgrade(default8, version2, "default"); + tester.updateVersionStatus(); - versionStatus = VersionStatus.compute(tester.controller()); - versions = versionStatus.versions(); - - assertEquals("No deployments: Low", - VespaVersion.Confidence.low, confidence(versions, version0)); + assertEquals("Confidence remains unchanged for version0: High", + Confidence.high, confidence(tester.controller(), version0)); assertEquals("90% of defaults deployed successfully: High", - VespaVersion.Confidence.high, confidence(versions, version2)); - assertEquals("Current version of this - no deployments: Low", - VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion)); + VespaVersion.Confidence.high, confidence(tester.controller(), version2)); // A new version is released, all canaries upgrade successfully, but enough "default" apps fail to mark version // as broken @@ -225,16 +223,14 @@ public class VersionStatusTest { tester.completeUpgradeWithError(default1, version3, "default", stagingTest); tester.completeUpgradeWithError(default2, version3, "default", stagingTest); tester.completeUpgradeWithError(default9, version3, "default", stagingTest); + tester.updateVersionStatus(); - versionStatus = VersionStatus.compute(tester.controller()); - versions = versionStatus.versions(); - - assertEquals("No deployments: Low", - VespaVersion.Confidence.low, confidence(versions, version0)); + assertEquals("Confidence remains unchanged for version0: High", + Confidence.high, confidence(tester.controller(), version0)); + assertEquals("Confidence remains unchanged for version2: High", + Confidence.high, confidence(tester.controller(), version2)); assertEquals("40% of defaults failed: Broken", - VespaVersion.Confidence.broken, confidence(versions, version3)); - assertEquals("Current version of this - no deployments: Low", - VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion)); + VespaVersion.Confidence.broken, confidence(tester.controller(), version3)); } @Test @@ -260,8 +256,8 @@ public class VersionStatusTest { vespaVersions.stream().noneMatch(v -> v.versionNumber().equals(versionWithUnknownTag))); } - private VespaVersion.Confidence confidence(List<VespaVersion> versions, Version version) { - return versions.stream() + private Confidence confidence(Controller controller, Version version) { + return controller.versionStatus().versions().stream() .filter(v -> v.statistics().version().equals(version)) .findFirst() .map(VespaVersion::confidence) |