diff options
Diffstat (limited to 'controller-server')
43 files changed, 2073 insertions, 402 deletions
diff --git a/controller-server/pom.xml b/controller-server/pom.xml index 93204845fbe..66fefd6f2fe 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -174,31 +174,6 @@ </exclusions> </dependency> - <dependency> - <groupId>com.yahoo.athenz</groupId> - <artifactId>athenz-zts-java-client</artifactId> - <scope>compile</scope> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - <!--Exclude all Jackson bundles provided by JDisc --> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - </exclusion> - </exclusions> - </dependency> - <!-- test --> <dependency> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 9208537dd98..677f2363c08 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -53,7 +53,7 @@ public class Application { /** Creates an empty application */ public Application(ApplicationId id) { this(id, DeploymentSpec.empty, ValidationOverrides.empty, Collections.emptyMap(), - new DeploymentJobs(OptionalLong.empty(), Collections.emptyList(), Optional.empty()), + new DeploymentJobs(OptionalLong.empty(), Collections.emptyList(), Optional.empty(), false), Change.empty(), Change.empty(), Optional.empty(), new ApplicationMetrics(0, 0), Optional.empty()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 57708bfc89c..7ae21e21f99 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -55,7 +55,6 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.time.Clock; -import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -141,6 +140,8 @@ public class ApplicationController { return sort(curator.readApplications(tenant)); } + public ArtifactRepository artifacts() { return artifactRepository; } + /** * Set the rotations marked as 'global' either 'in' or 'out of' service. * @@ -232,8 +233,8 @@ public class ApplicationController { * @throws IllegalArgumentException if the application already exists */ public Application createApplication(ApplicationId id, Optional<NToken> token) { - if ( ! (id.instance().isDefault() || id.instance().value().matches("\\d+"))) // TODO: Support instances properly - throw new UnsupportedOperationException("Only the instance names 'default' and names which are just the PR number are supported at the moment"); + if ( ! (id.instance().isDefault())) // TODO: Support instances properly + throw new UnsupportedOperationException("Only the instance name 'default' is supported at the moment"); try (Lock lock = lock(id)) { // Validate only application names which do not already exist. if (asList(id.tenant()).stream().noneMatch(application -> application.id().application().equals(id.application()))) @@ -354,6 +355,14 @@ public class ApplicationController { } } + /** Assembles and deploys a tester application to the given zone. */ + public ActivateResult deployTester(ApplicationId tester, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions options) { + if ( ! tester.instance().value().endsWith("-t")) + throw new IllegalArgumentException("'" + tester + "' is not a tester application!"); + + return deploy(tester, applicationPackage, zone, options, Collections.emptySet(), Collections.emptySet()); + } + private ActivateResult deploy(ApplicationId application, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions deployOptions, Set<String> rotationNames, Set<String> cnames) { @@ -599,7 +608,7 @@ public class ApplicationController { * and store the application, and finally release (close) the lock. */ Lock lock(ApplicationId application) { - return curator.lock(application, Duration.ofMinutes(10)); + return curator.lock(application); } /** Verify that each of the production zones listed in the deployment spec exist in this system. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index c90ab5d19ba..790d6d00035 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -133,6 +133,8 @@ public class Controller extends AbstractComponent { // Record the version of this controller curator().writeControllerVersion(this.hostname(), Vtag.currentVersion); + + jobController.updateStorage(); } /** Returns the instance controlling tenants */ @@ -141,6 +143,9 @@ public class Controller extends AbstractComponent { /** Returns the instance controlling applications */ public ApplicationController applications() { return applicationController; } + /** Returns the instance controlling deployment jobs. */ + public JobController jobController() { return jobController; } + public List<AthenzDomain> getDomainList(String prefix) { return athenzClientFactory.createZmsClientWithServicePrincipal().getDomainList(prefix); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index 79e7fa0295a..2209cdf3013 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -89,6 +89,12 @@ public class LockedApplication { outstandingChange, ownershipIssueId, metrics, rotation); } + public LockedApplication withBuiltInternally(boolean builtInternally) { + return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments, + deploymentJobs.withBuiltInternally(builtInternally), change, outstandingChange, + ownershipIssueId, metrics, rotation); + } + public LockedApplication withProjectId(OptionalLong projectId) { return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments, deploymentJobs.withProjectId(projectId), change, outstandingChange, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index 2e8fe795fb5..228ca01e764 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.athenz.api.NToken; +import com.yahoo.vespa.athenz.client.zts.ZtsClient; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; @@ -69,12 +70,13 @@ public class TenantController { /** Returns a list of all tenants accessible by the given user */ public List<Tenant> asList(UserId user) { AthenzUser athenzUser = AthenzUser.fromUserId(user.id()); - Set<AthenzDomain> userDomains = new HashSet<>(athenzClientFactory.createZtsClientWithServicePrincipal() - .getTenantDomainsForUser(athenzUser)); - return asList().stream() - .filter(tenant -> isUser(tenant, user) || - userDomains.stream().anyMatch(domain -> inDomain(tenant, domain))) - .collect(Collectors.toList()); + try (ZtsClient ztsClient = athenzClientFactory.createZtsClientWithServicePrincipal()) { + Set<AthenzDomain> userDomains = new HashSet<>(ztsClient.getTenantDomains(athenzClientFactory.getControllerIdentity(), athenzUser, "admin")); + return asList().stream() + .filter(tenant -> isUser(tenant, user) || + userDomains.stream().anyMatch(domain -> inDomain(tenant, domain))) + .collect(Collectors.toList()); + } } /** Create an user tenant with given username */ @@ -203,7 +205,7 @@ public class TenantController { * and store the tenant, and finally release (close) the lock. */ private Lock lock(TenantName tenant) { - return curator.lock(tenant, Duration.ofMinutes(10)); + return curator.lock(tenant); } private static boolean inDomain(Tenant tenant, AthenzDomain domain) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index 4968e161a35..65ba7e68d31 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -27,19 +27,22 @@ public class DeploymentJobs { private final OptionalLong projectId; private final ImmutableMap<JobType, JobStatus> status; private final Optional<IssueId> issueId; + private final boolean builtInternally; public DeploymentJobs(OptionalLong projectId, Collection<JobStatus> jobStatusEntries, - Optional<IssueId> issueId) { - this(projectId, asMap(jobStatusEntries), issueId); + Optional<IssueId> issueId, boolean builtInternally) { + this(projectId, asMap(jobStatusEntries), issueId, builtInternally); } - private DeploymentJobs(OptionalLong projectId, Map<JobType, JobStatus> status, Optional<IssueId> issueId) { + private DeploymentJobs(OptionalLong projectId, Map<JobType, JobStatus> status, Optional<IssueId> issueId, + boolean builtInternally) { requireId(projectId, "projectId must be a positive integer"); Objects.requireNonNull(status, "status cannot be null"); Objects.requireNonNull(issueId, "issueId cannot be null"); this.projectId = projectId; this.status = ImmutableMap.copyOf(status); this.issueId = issueId; + this.builtInternally = builtInternally; } private static Map<JobType, JobStatus> asMap(Collection<JobStatus> jobStatusEntries) { @@ -56,7 +59,7 @@ public class DeploymentJobs { if (job == null) job = JobStatus.initial(jobType); return job.withCompletion(completion, jobError); }); - return new DeploymentJobs(OptionalLong.of(projectId), status, issueId); + return new DeploymentJobs(OptionalLong.of(projectId), status, issueId, builtInternally); } public DeploymentJobs withTriggering(JobType jobType, JobStatus.JobRun jobRun) { @@ -65,21 +68,25 @@ public class DeploymentJobs { if (job == null) job = JobStatus.initial(jobType); return job.withTriggering(jobRun); }); - return new DeploymentJobs(projectId, status, issueId); + return new DeploymentJobs(projectId, status, issueId, builtInternally); } public DeploymentJobs withProjectId(OptionalLong projectId) { - return new DeploymentJobs(projectId, status, issueId); + return new DeploymentJobs(projectId, status, issueId, builtInternally); } public DeploymentJobs with(IssueId issueId) { - return new DeploymentJobs(projectId, status, Optional.ofNullable(issueId)); + return new DeploymentJobs(projectId, status, Optional.ofNullable(issueId), builtInternally); } public DeploymentJobs without(JobType job) { Map<JobType, JobStatus> status = new HashMap<>(this.status); status.remove(job); - return new DeploymentJobs(projectId, status, issueId); + return new DeploymentJobs(projectId, status, issueId, builtInternally); + } + + public DeploymentJobs withBuiltInternally(boolean builtInternally) { + return new DeploymentJobs(projectId, status, issueId, builtInternally); } /** Returns an immutable map of the status entries in this */ @@ -107,6 +114,8 @@ public class DeploymentJobs { public Optional<IssueId> issueId() { return issueId; } + public boolean builtInternally() { return builtInternally; } + private static OptionalLong requireId(OptionalLong id, String message) { Objects.requireNonNull(id, message); if ( ! id.isPresent()) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/config/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/config/package-info.java new file mode 100644 index 00000000000..f3f2acf1e01 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/config/package-info.java @@ -0,0 +1,10 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Required for using {@link com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig} outside controller-server module. + * + * @author bjorncs + */ +@ExportPackage +package com.yahoo.vespa.hosted.controller.athenz.config; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java index 159a4f11619..633c0470080 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java @@ -8,16 +8,19 @@ import com.yahoo.athenz.auth.impl.SimplePrincipal; import com.yahoo.athenz.auth.token.PrincipalToken; import com.yahoo.athenz.auth.util.Crypto; import com.yahoo.athenz.zms.ZMSClient; -import com.yahoo.athenz.zts.ZTSClient; import com.yahoo.container.jdisc.secretstore.SecretStore; +import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.NToken; +import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; +import com.yahoo.vespa.athenz.client.zts.ZtsClient; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.utils.AthenzIdentities; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZtsClient; import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; +import java.net.URI; import java.security.PrivateKey; /** @@ -38,6 +41,11 @@ public class AthenzClientFactoryImpl implements AthenzClientFactory { this.athenzPrincipalAuthority = new AthenzPrincipalAuthority(config.principalHeaderName()); } + @Override + public AthenzIdentity getControllerIdentity() { + return identityProvider.identity(); + } + /** * @return A ZMS client instance with the service identity as principal. */ @@ -51,7 +59,7 @@ public class AthenzClientFactoryImpl implements AthenzClientFactory { */ @Override public ZtsClient createZtsClientWithServicePrincipal() { - return new ZtsClientImpl(new ZTSClient(config.ztsUrl(), identityProvider.getIdentitySslContext()), config); + return new DefaultZtsClient(URI.create(config.ztsUrl()), identityProvider); } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZtsClientImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZtsClientImpl.java deleted file mode 100644 index c3ed8f0a99f..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZtsClientImpl.java +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz.impl; - -import com.yahoo.athenz.zts.TenantDomains; -import com.yahoo.athenz.zts.ZTSClient; -import com.yahoo.athenz.zts.ZTSClientException; -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZtsClient; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZtsException; -import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; - -import java.util.List; -import java.util.function.Supplier; -import java.util.logging.Logger; - -import static java.util.stream.Collectors.toList; - -/** - * @author bjorncs - */ -public class ZtsClientImpl implements ZtsClient { - - private static final Logger log = Logger.getLogger(ZtsClientImpl.class.getName()); - - private final ZTSClient ztsClient; - private final AthenzService service; - - public ZtsClientImpl(ZTSClient ztsClient, AthenzConfig config) { - this.ztsClient = ztsClient; - this.service = new AthenzService(config.domain(), config.service().name()); - } - - @Override - public List<AthenzDomain> getTenantDomainsForUser(AthenzIdentity identity) { - return getOrThrow(() -> { - log.log(LogLevel.DEBUG, String.format( - "getTenantDomains(domain=%s, identity=%s, rolename=admin, service=%s)", - service.getDomain().getName(), identity.getFullName(), service.getFullName())); - TenantDomains domains = ztsClient.getTenantDomains( - service.getDomain().getName(), identity.getFullName(), "admin", service.getName()); - return domains.getTenantDomainNames().stream() - .map(AthenzDomain::new) - .collect(toList()); - }); - } - - private static <T> T getOrThrow(Supplier<T> wrappedCode) { - try { - return wrappedCode.get(); - } catch (ZTSClientException e) { - log.warning("Error from Athenz: " + e.getMessage()); - throw new ZtsException(e.getCode(), e); - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzClientFactoryMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzClientFactoryMock.java index f7939422170..6f829113016 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzClientFactoryMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzClientFactoryMock.java @@ -3,10 +3,12 @@ package com.yahoo.vespa.hosted.controller.athenz.mock; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; +import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.NToken; +import com.yahoo.vespa.athenz.client.zts.ZtsClient; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZtsClient; import java.util.logging.Level; import java.util.logging.Logger; @@ -34,6 +36,11 @@ public class AthenzClientFactoryMock extends AbstractComponent implements Athenz } @Override + public AthenzIdentity getControllerIdentity() { + return new AthenzService("vespa.hosting"); + } + + @Override public ZmsClient createZmsClientWithServicePrincipal() { log("createZmsClientWithServicePrincipal()"); return new ZmsClientMock(athenz); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java index 4aa1c2b93a5..8b3fb3ca47e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java @@ -3,8 +3,17 @@ package com.yahoo.vespa.hosted.controller.athenz.mock; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZtsClient; +import com.yahoo.vespa.athenz.api.AthenzRole; +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.api.ZToken; +import com.yahoo.vespa.athenz.client.zts.Identity; +import com.yahoo.vespa.athenz.client.zts.InstanceIdentity; +import com.yahoo.vespa.athenz.client.zts.ZtsClient; +import com.yahoo.vespa.athenz.tls.Pkcs10Csr; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.time.Duration; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -24,12 +33,57 @@ public class ZtsClientMock implements ZtsClient { } @Override - public List<AthenzDomain> getTenantDomainsForUser(AthenzIdentity identity) { - log.log(Level.INFO, "getTenantDomainsForUser(principal='%s')", identity); + public List<AthenzDomain> getTenantDomains(AthenzIdentity providerIdentity, AthenzIdentity userIdentity, String roleName) { + log.log(Level.INFO, String.format("getTenantDomains(providerIdentity='%s', userIdentity='%s', roleName='%s')", + providerIdentity.getFullName(), userIdentity.getFullName(), roleName)); return athenz.domains.values().stream() - .filter(domain -> domain.tenantAdmins.contains(identity) || domain.admins.contains(identity)) + .filter(domain -> domain.tenantAdmins.contains(userIdentity) || domain.admins.contains(userIdentity)) .map(domain -> domain.name) .collect(toList()); } + @Override + public InstanceIdentity registerInstance(AthenzService providerIdentity, AthenzService instanceIdentity, String instanceId, String attestationData, boolean requestServiceToken, Pkcs10Csr csr) { + throw new UnsupportedOperationException(); + } + + @Override + public InstanceIdentity refreshInstance(AthenzService providerIdentity, AthenzService instanceIdentity, String instanceId, boolean requestServiceToken, Pkcs10Csr csr) { + throw new UnsupportedOperationException(); + } + + @Override + public Identity getServiceIdentity(AthenzService identity, String keyId, Pkcs10Csr csr) { + throw new UnsupportedOperationException(); + } + + @Override + public Identity getServiceIdentity(AthenzService identity, String keyId, KeyPair keyPair, String dnsSuffix) { + throw new UnsupportedOperationException(); + } + + @Override + public ZToken getRoleToken(AthenzDomain domain) { + throw new UnsupportedOperationException(); + } + + @Override + public ZToken getRoleToken(AthenzRole athenzRole) { + throw new UnsupportedOperationException(); + } + + @Override + public X509Certificate getRoleCertificate(AthenzRole role, Duration expiry, KeyPair keyPair, String cloud) { + throw new UnsupportedOperationException(); + } + + @Override + public X509Certificate getRoleCertificate(AthenzRole role, KeyPair keyPair, String cloud) { + throw new UnsupportedOperationException(); + } + + @Override + public void close() { + + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DummyStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DummyStepRunner.java new file mode 100644 index 00000000000..17b523c60bf --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DummyStepRunner.java @@ -0,0 +1,12 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; + +public class DummyStepRunner implements StepRunner { + + @Override + public Step.Status run(LockedStep step, RunId id) { + return Step.Status.succeeded; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java index 74dffc1c4fd..18b62f5ea0f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java @@ -1,7 +1,9 @@ package com.yahoo.vespa.hosted.controller.deployment; -import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; + +import java.util.Optional; /** * Wraps a JobController as a BuildService. @@ -20,17 +22,18 @@ public class InternalBuildService implements BuildService { @Override public void trigger(BuildJob buildJob) { - + jobs.start(buildJob.applicationId(), JobType.fromJobName(buildJob.jobName())); } @Override public JobState stateOf(BuildJob buildJob) { - return null; + Optional<RunStatus> run = jobs.last(buildJob.applicationId(), JobType.fromJobName(buildJob.jobName())); + return run.isPresent() && ! run.get().hasEnded() ? JobState.running : JobState.idle; } @Override public boolean builds(BuildJob buildJob) { - return false; + return jobs.builds(buildJob.applicationId()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java new file mode 100644 index 00000000000..77b6baa6bea --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -0,0 +1,511 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.SystemName; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.ActivateResult; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NoInstanceException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.Testers; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.deployment.Step.Status; +import com.yahoo.yolean.Exceptions; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.net.URI; +import java.text.SimpleDateFormat; +import java.time.Duration; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.TimeZone; +import java.util.function.Supplier; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import static com.yahoo.log.LogLevel.DEBUG; +import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.ACTIVATION_CONFLICT; +import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.APPLICATION_LOCK_FAILURE; +import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.OUT_OF_CAPACITY; +import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.active; +import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.dirty; +import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.reserved; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; +import static java.util.logging.Level.INFO; +import static java.util.logging.Level.WARNING; + +/** + * Runs steps of a deployment job against its provided controller. + * + * A dual-purpose logger is set up for each thread that runs a step here: + * 1. All messages are logged to a buffer which is stored in an external log storage at the end of execution, and + * 2. all messages are also logged through the usual logging framework; thus, by default, any messages of level + * {@code Level.INFO} or higher end up in the Vespa log, and all messages may be sent there by means of log-control. + * + * @author jonmv + */ +public class InternalStepRunner implements StepRunner { + + static final Duration endpointTimeout = Duration.ofMinutes(15); + static final Duration installationTimeout = Duration.ofMinutes(150); + + // TODO jvenstad: Move this tester logic to the application controller, perhaps? + public static ApplicationId testerOf(ApplicationId id) { + return ApplicationId.from(id.tenant().value(), + id.application().value(), + id.instance().value() + "-t"); + } + + private final Controller controller; + private final Testers testers; + private final ThreadLocal<ByteArrayLogger> logger = new ThreadLocal<>(); + + public InternalStepRunner(Controller controller, Testers testers) { + this.controller = controller; + this.testers = testers; + } + + @Override + public Status run(LockedStep step, RunId id) { + try { + logger.set(ByteArrayLogger.of(id.application(), id.type(), step.get())); + switch (step.get()) { + case deployInitialReal: return deployInitialReal(id); + case installInitialReal: return installInitialReal(id); + case deployReal: return deployReal(id); + case deployTester: return deployTester(id); + case installReal: return installReal(id); + case installTester: return installTester(id); + case startTests: return startTests(id); + case endTests: return endTests(id); + case deactivateReal: return deactivateReal(id); + case deactivateTester: return deactivateTester(id); + case report: return report(id); + default: throw new AssertionError("Unknown step '" + step + "'!"); + } + } + catch (RuntimeException e) { + logger.get().log(INFO, "Unexpected exception: " + Exceptions.toMessageString(e)); + return failed; + } + finally { + controller.jobController().log(id, step.get(), logger.get().getLog()); + logger.remove(); + } + } + + private Status deployInitialReal(RunId id) { + JobStatus.JobRun triggering = triggering(id.application(), id.type()); + logger.get().log("Deploying platform version " + + triggering.sourcePlatform().orElse(triggering.platform()) + + " and application version " + + triggering.sourceApplication().orElse(triggering.application()) + " ..."); + return deployReal(id, true); + } + + private Status deployReal(RunId id) { + JobStatus.JobRun triggering = triggering(id.application(), id.type()); + logger.get().log("Deploying platform version " + triggering.platform() + + " and application version " + triggering.application() + " ..."); + return deployReal(id, false); + } + + private Status deployReal(RunId id, boolean setTheStage) { + return deploy(id.application(), + id.type(), + () -> controller.applications().deploy(id.application(), + zone(id.type()), + Optional.empty(), + new DeployOptions(false, + Optional.empty(), + false, + setTheStage))); + } + + private Status deployTester(RunId id) { + // TODO jvenstad: Consider deploying old version of tester for initial staging feeding? + logger.get().log("Deploying the tester container ..."); + return deploy(testerOf(id.application()), + id.type(), + () -> controller.applications().deployTester(testerOf(id.application()), + testerPackage(id), + zone(id.type()), + new DeployOptions(true, + Optional.of(controller.systemVersion()), + false, + false))); + } + + private Status deploy(ApplicationId id, JobType type, Supplier<ActivateResult> deployment) { + try { + PrepareResponse prepareResponse = deployment.get().prepareResponse(); + if ( ! prepareResponse.configChangeActions.refeedActions.stream().allMatch(action -> action.allowed)) { + logger.get().log("Deploy failed due to non-compatible changes that require re-feed. " + + "Your options are: \n" + + "1. Revert the incompatible changes.\n" + + "2. If you think it is safe in your case, you can override this validation, see\n" + + " http://docs.vespa.ai/documentation/reference/validation-overrides.html\n" + + "3. Deploy as a new application under a different name.\n" + + "Illegal actions:\n" + + prepareResponse.configChangeActions.refeedActions.stream() + .filter(action -> ! action.allowed) + .flatMap(action -> action.messages.stream()) + .collect(Collectors.joining("\n")) + "\n" + + "Details:\n" + + prepareResponse.log.stream() + .map(entry -> entry.message) + .collect(Collectors.joining("\n"))); + return failed; + } + + if (prepareResponse.configChangeActions.restartActions.isEmpty()) + logger.get().log("No services requiring restart."); + else + prepareResponse.configChangeActions.restartActions.stream() + .flatMap(action -> action.services.stream()) + .map(service -> service.hostName) + .sorted().distinct() + .map(Hostname::new) + .forEach(hostname -> { + controller.applications().restart(new DeploymentId(id, zone(type)), Optional.of(hostname)); + logger.get().log("Restarting services on host " + hostname.id() + "."); + }); + logger.get().log("Deployment successful."); + return succeeded; + } + catch (ConfigServerException e) { + if ( e.getErrorCode() == OUT_OF_CAPACITY && type.isTest() + || e.getErrorCode() == ACTIVATION_CONFLICT + || e.getErrorCode() == APPLICATION_LOCK_FAILURE) { + logger.get().log("Will retry, because of '" + e.getErrorCode() + "' deploying:\n" + e.getMessage()); + return unfinished; + } + throw e; + } + } + + private Status installInitialReal(RunId id) { + return installReal(id.application(), id.type(), true); + } + + private Status installReal(RunId id) { + return installReal(id.application(), id.type(), false); + } + + private Status installReal(ApplicationId id, JobType type, boolean setTheStage) { + JobStatus.JobRun triggering = triggering(id, type); + Version platform = setTheStage ? triggering.sourcePlatform().orElse(triggering.platform()) : triggering.platform(); + ApplicationVersion application = setTheStage ? triggering.sourceApplication().orElse(triggering.application()) : triggering.application(); + logger.get().log("Checking installation of " + platform + " and " + application + " ..."); + + if (nodesConverged(id, type, platform) && servicesConverged(id, type)) { + logger.get().log("Installation succeeded!"); + return succeeded; + } + + if (timedOut(id, type, installationTimeout)) { + logger.get().log(INFO, "Installation failed to complete within " + installationTimeout.toMinutes() + " minutes!"); + return failed; + } + + logger.get().log("Installation not yet complete."); + return unfinished; + } + + private Status installTester(RunId id) { + logger.get().log("Checking installation of tester container ..."); + + if (servicesConverged(testerOf(id.application()), id.type())) { + logger.get().log("Tester container successfully installed!"); + return succeeded; + } + + if (timedOut(id.application(), id.type(), installationTimeout)) { + logger.get().log(WARNING, "Installation of tester failed to complete within " + installationTimeout.toMinutes() + " minutes of real deployment!"); + return failed; + } + + logger.get().log("Installation of tester not yet complete."); + return unfinished; + } + + private boolean nodesConverged(ApplicationId id, JobType type, Version target) { + List<Node> nodes = controller.configServer().nodeRepository().list(zone(type), id, Arrays.asList(active, reserved)); + for (Node node : nodes) + // TODO jvenstad: Add ALLOWED_TO_BE_DOWN and reboot and restart generation information as well. + logger.get().log(String.format("%70s: %-12s%-25s%-32s%s", + node.hostname(), + node.serviceState(), + node.wantedVersion() + (node.currentVersion().equals(node.wantedVersion()) ? "" : " <-- " + node.currentVersion()), + node.restartGeneration() == node.wantedRestartGeneration() ? "" + : "restart pending (" + node.wantedRestartGeneration() + " <-- " + node.restartGeneration() + ")", + node.rebootGeneration() == node.wantedRebootGeneration() ? "" + : "reboot pending (" + node.wantedRebootGeneration() + " <-- " + node.rebootGeneration() + ")")); + + return nodes.stream().allMatch(node -> node.currentVersion().equals(target) + && node.restartGeneration() == node.wantedRestartGeneration() + && node.rebootGeneration() == node.wantedRebootGeneration()); + } + + private boolean servicesConverged(ApplicationId id, JobType type) { + // TODO jvenstad: Print information for each host. + return controller.configServer().serviceConvergence(new DeploymentId(id, zone(type))).map(ServiceConvergence::converged).orElse(false); + } + + private Status startTests(RunId id) { + logger.get().log("Attempting to find endpoints ..."); + Map<ZoneId, List<URI>> endpoints = deploymentEndpoints(id.application()); + logger.get().log("Found endpoints:\n" + + endpoints.entrySet().stream() + .map(zoneEndpoints -> "- " + zoneEndpoints.getKey() + ":\n" + + zoneEndpoints.getValue().stream() + .map(uri -> " |-- " + uri) + .collect(Collectors.joining("\n")))); + if ( ! endpoints.containsKey(zone(id.type()))) { + if (timedOut(id.application(), id.type(), endpointTimeout)) { + logger.get().log(WARNING, "Endpoints failed to show up within " + endpointTimeout.toMinutes() + " minutes!"); + return failed; + } + + logger.get().log("Endpoints for the deployment to test are not yet ready."); + return unfinished; + } + + Optional<URI> testerEndpoint = testerEndpoint(id); + if (testerEndpoint.isPresent()) { + logger.get().log("Starting tests ..."); + testers.startTests(testerEndpoint.get(), + Testers.Suite.of(id.type()), + testConfig(id.application(), zone(id.type()), controller.system(), endpoints)); + return succeeded; + } + + if (timedOut(id.application(), id.type(), installationTimeout)) { + logger.get().log(WARNING, "Endpoint for tester failed to show up within " + installationTimeout.toMinutes() + " minutes of real deployment!"); + return failed; + } + + logger.get().log("Endpoints of tester container not yet available."); + return unfinished; + } + + private Status endTests(RunId id) { + URI testerEndpoint = testerEndpoint(id) + .orElseThrow(() -> new NoSuchElementException("Endpoint for tester vanished again before tests were complete!")); + + Status status; + switch (testers.getStatus(testerEndpoint)) { + case NOT_STARTED: + throw new IllegalStateException("Tester reports tests not started, even though they should have!"); + case RUNNING: + logger.get().log("Tests still running ..."); + return unfinished; + case FAILURE: + logger.get().log("Tests failed."); + status = failed; break; + case ERROR: + logger.get().log(INFO, "Tester failed running its tests!"); + status = failed; break; + case SUCCESS: + logger.get().log("Tests completed successfully."); + status = succeeded; break; + default: + throw new AssertionError("Unknown status!"); + } + logger.get().log(new String(testers.getLogs(testerEndpoint))); // TODO jvenstad: Replace with something less hopeless! + return status; + } + + private Status deactivateReal(RunId id) { + logger.get().log("Deactivating deployment of " + id.application() + " in " + zone(id.type()) + " ..."); + return deactivate(id.application(), id.type()); + } + + private Status deactivateTester(RunId id) { + logger.get().log("Deactivating tester of " + id.application() + " in " + zone(id.type()) + " ..."); + return deactivate(testerOf(id.application()), id.type()); + } + + private Status deactivate(ApplicationId id, JobType type) { + try { + controller.configServer().deactivate(new DeploymentId(id, zone(type))); + } + catch (NoInstanceException e) { } + return succeeded; + } + + private Status report(RunId id) { + controller.jobController().active(id).ifPresent(run -> controller.applications().deploymentTrigger().notifyOfCompletion(report(run))); + return succeeded; + } + + /** Returns the real application with the given id. */ + private Application application(ApplicationId id) { + return controller.applications().require(id); + } + + /** Returns the zone of the given job type. */ + private ZoneId zone(JobType type) { + return type.zone(controller.system()).get(); + } + + /** Returns the triggering of the currently running job, i.e., this job. */ + private JobStatus.JobRun triggering(ApplicationId id, JobType type) { + return application(id).deploymentJobs().statusOf(type).get().lastTriggered().get(); + } + + /** Returns whether the time elapsed since the last real deployment in the given zone is more than the given timeout. */ + private boolean timedOut(ApplicationId id, JobType type, Duration timeout) { + return application(id).deployments().get(zone(type)).at().isBefore(controller.clock().instant().minus(timeout)); + } + + /** Returns a generated job report for the given run. */ + private DeploymentJobs.JobReport report(RunStatus run) { + return new DeploymentJobs.JobReport(run.id().application(), + run.id().type(), + Long.MAX_VALUE, + run.id().number(), + Optional.empty(), + run.hasFailed() ? Optional.of(DeploymentJobs.JobError.unknown) : Optional.empty()); + } + + /** Returns the application package for the tester application, assembled from a generated config, fat-jar and services.xml. */ + private ApplicationPackage testerPackage(RunId id) { + ApplicationVersion version = application(id.application()).deploymentJobs() + .statusOf(id.type()).get() + .lastTriggered().get() + .application(); + + byte[] testJar = controller.applications().artifacts().getTesterJar(testerOf(id.application()), version.id()); + byte[] servicesXml = servicesXml(); + + // TODO hakonhall: Assemble! + + throw new AssertionError(); + } + + /** Returns all endpoints for all current deployments of the given real application. */ + private Map<ZoneId, List<URI>> deploymentEndpoints(ApplicationId id) { + ImmutableMap.Builder<ZoneId, List<URI>> deployments = ImmutableMap.builder(); + application(id).deployments().keySet() + .forEach(zone -> controller.applications().getDeploymentEndpoints(new DeploymentId(id, zone)) + .ifPresent(endpoints -> deployments.put(zone, endpoints))); + return deployments.build(); + } + + /** Returns a URI of the tester endpoint retrieved from the routing generator, provided it matches an expected form. */ + private Optional<URI> testerEndpoint(RunId id) { + ApplicationId tester = testerOf(id.application()); + return controller.applications().getDeploymentEndpoints(new DeploymentId(tester, zone(id.type()))) + .flatMap(uris -> uris.stream() + .filter(uri -> uri.getHost().contains(String.format("%s--%s--%s.", + tester.instance().value(), + tester.application().value(), + tester.tenant().value()))) + .findAny()); + } + + /** Returns the generated services.xml content for the tester application. */ + private byte[] servicesXml() { + //TODO hakonhall: Create! + return "".getBytes(); + } + + /** Returns the config for the tests to run for the given job. */ + private static byte[] testConfig(ApplicationId id, ZoneId testerZone, SystemName system, Map<ZoneId, List<URI>> deployments) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("application", id.serializedForm()); + root.setString("zone", testerZone.value()); + root.setString("system", system.name()); + Cursor endpointsObject = root.setObject("endpoints"); + deployments.forEach((zone, endpoints) -> { + Cursor endpointArray = endpointsObject.setArray(zone.value()); + for (URI endpoint : endpoints) + endpointArray.addString(endpoint.toString()); + }); + try { + return SlimeUtils.toJsonBytes(slime); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** Logger which logs all records to a private byte array, as well as to its parent. */ + static class ByteArrayLogger extends Logger { + + private static final Logger parent = Logger.getLogger(InternalStepRunner.class.getName()); + private static final SimpleDateFormat timestampFormat = new SimpleDateFormat("[HH:mm:ss.SSS] "); + static { timestampFormat.setTimeZone(TimeZone.getTimeZone("UTC")); } + + private final ByteArrayOutputStream bytes; + private final PrintStream out; + + private ByteArrayLogger(Logger parent, String suffix) { + super(parent.getName() + suffix, null); + setParent(parent); + + bytes = new ByteArrayOutputStream(); + out = new PrintStream(bytes); + } + + static ByteArrayLogger of(ApplicationId id, JobType type, Step step) { + return new ByteArrayLogger(parent, String.format(".%s.%s.%s", id.serializedForm(), type.jobName(), step)); + } + + @Override + public void log(LogRecord record) { + // TODO jvenstad: Store log records in a serialised format. + String timestamp = timestampFormat.format(new Date(record.getMillis())); + for (String line : record.getMessage().split("\n")) + out.println(timestamp + ": " + line); + + getParent().log(record); + } + + public void log(String message) { + log(DEBUG, message); + } + + @Override + public boolean isLoggable(Level __) { + return true; + } + + public byte[] getLog() { + out.flush(); + return bytes.toByteArray(); + } + + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 4da13632eef..9ced75303ff 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -1,93 +1,303 @@ package com.yahoo.vespa.hosted.controller.deployment; +import com.google.common.collect.ImmutableMap; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogStore; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NoInstanceException; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.UnaryOperator; +import java.util.stream.Stream; + +import static com.google.common.collect.ImmutableList.copyOf; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; +import static com.yahoo.vespa.hosted.controller.deployment.InternalStepRunner.testerOf; /** * A singleton owned by the controller, which contains the state and methods for controlling deployment jobs. * * Keys are the {@link ApplicationId} of the real application, for which the deployment job is run, and the - * {@link ZoneId} of the real deployment to test. - * + * {@link JobType} of the real deployment to test. * Although the deployment jobs are themselves applications, their IDs are not to be referenced. * + * Jobs consist of sets of {@link Step}s, defined in {@link JobProfile}s. + * Each run is represented by a {@link RunStatus}, which holds the status of each step of the run, as well as + * some other meta data. + * * @author jonmv */ public class JobController { private final Controller controller; + private final CuratorDb curator; private final LogStore logs; public JobController(Controller controller, LogStore logStore) { this.controller = controller; + this.curator = controller.curator(); this.logs = logStore; + + } + + /** Rewrite all job data with the newest format. */ + public void updateStorage() { + for (ApplicationId id : applications()) + for (JobType type : jobs(id)) { + locked(id, type, runs -> { + curator.readLastRun(id, type).ifPresent(curator::writeLastRun); + }); + } + } + + /** Returns the details currently logged for the given run, if known. */ + public Optional<RunDetails> details(RunId id) { + RunStatus run = runs(id.application(), id.type()).get(id); + if (run == null) + return Optional.empty(); + + Map<Step, byte[]> details = new HashMap<>(); + for (Step step : run.steps().keySet()) { + byte[] log = logs.get(id, step.name()); + if (log.length > 0) + details.put(step, log); + } + return Optional.of(new RunDetails(details)); } + /** Appends the given log bytes to the currently stored bytes for the given run and step. */ + public void log(RunId id, Step step, byte[] log) { + try (Lock __ = curator.lock(id.application(), id.type())) { + logs.append(id, step.name(), log); + } + } -// GET: + // TODO jvenstad: Remove this, and let the DeploymentTrigger trigger directly with the correct BuildService. /** Returns whether the given application has registered with this build service. */ - boolean builds(ApplicationId application) { - return false; + public boolean builds(ApplicationId id) { + return controller.applications().get(id) + .map(application -> application.deploymentJobs().builtInternally()) + .orElse(false); } /** Returns a list of all application which have registered. */ - List<ApplicationId> applications() { - return null; + public List<ApplicationId> applications() { + return copyOf(controller.applications().asList().stream() + .filter(application -> application.deploymentJobs().builtInternally()) + .map(Application::id) + .iterator()); } /** Returns all job types which have been run for the given application. */ - List<ZoneId> jobs(ApplicationId application) { - return null; + public List<JobType> jobs(ApplicationId id) { + return copyOf(Stream.of(JobType.values()) + .filter(type -> last(id, type).isPresent()) + .iterator()); } - /** Returns a list of meta information about all known runs of the given job type. */ - List<JobMeta> runs(ApplicationId application, ZoneId zone) { - return null; + /** Returns an immutable map of all known runs for the given application and job type. */ + public Map<RunId, RunStatus> runs(ApplicationId id, JobType type) { + Map<RunId, RunStatus> runs = curator.readHistoricRuns(id, type); + last(id, type).ifPresent(run -> runs.putIfAbsent(run.id(), run)); + return ImmutableMap.copyOf(runs); } - /** Returns the current status of the given job. */ - JobMeta status(JobId job) { - return null; + /** Returns the run with the given id, if it exists. */ + public Optional<RunStatus> run(RunId id) { + return runs(id.application(), id.type()).values().stream() + .filter(run -> run.id().equals(id)) + .findAny(); } - /** Returns the details for the given job. */ - JobDetails details(JobId job) { - return null; + /** Returns the last run of the given type, for the given application, if one has been run. */ + public Optional<RunStatus> last(ApplicationId id, JobType type) { + return curator.readLastRun(id, type); } + /** Returns the run with the given id, provided it is still active. */ + public Optional<RunStatus> active(RunId id) { + return last(id.application(), id.type()) + .filter(run -> ! run.hasEnded()) + .filter(run -> run.id().equals(id)); + } + + /** Returns a list of all active runs. */ + public List<RunStatus> active() { + return copyOf(applications().stream() + .flatMap(id -> Stream.of(JobType.values()) + .map(type -> last(id, type)) + .filter(Optional::isPresent).map(Optional::get) + .filter(run -> ! run.hasEnded())) + .iterator()); + } + + /** Changes the status of the given step, for the given run, provided it is still active. */ + public void update(RunId id, Step.Status status, LockedStep step) { + locked(id, run -> run.with(status, step)); + } + + /** Changes the status of the given run to inactive, and stores it as a historic run. */ + public void finish(RunId id) { + locked(id, run -> { // Store the modified run after it has been written to the collection, in case the latter fails. + RunStatus finishedRun = run.finished(controller.clock().instant()); + locked(id.application(), id.type(), runs -> runs.put(run.id(), finishedRun)); + return finishedRun; + }); + } + + /** Marks the given run as aborted; no further normal steps will run, but run-always steps will try to succeed. */ + public void abort(RunId id) { + locked(id, run -> run.aborted()); + } -// POST: /** Registers the given application, such that it may have deployment jobs run here. */ - void register(ApplicationId application) { - ; + void register(ApplicationId id) { + controller.applications().lockIfPresent(id, application -> + controller.applications().store(application.withBuiltInternally(true))); } - /** Orders a run of the given type, and returns the id of the created job. */ - JobId run(ApplicationId application, ZoneId zone) { - return null; + /** Accepts and stores a new application package and test jar pair under a generated application version key. */ + public ApplicationVersion submit(ApplicationId id, SourceRevision revision, + byte[] applicationPackage, byte[] applicationTestJar) { + AtomicReference<ApplicationVersion> version = new AtomicReference<>(); + controller.applications().lockOrThrow(id, application -> { + controller.applications().store(application.withBuiltInternally(true)); + + long run = nextBuild(id); + version.set(ApplicationVersion.from(revision, run)); + + // TODO smorgrav: Store the pair. + + notifyOfNewSubmission(id, revision, run); + }); + return version.get(); } + /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ + public void start(ApplicationId id, JobType type) { + controller.applications().lockIfPresent(id, application -> { + if ( ! application.get().deploymentJobs().builtInternally()) + throw new IllegalArgumentException(id + " is not built here!"); + + locked(id, type, __ -> { + Optional<RunStatus> last = last(id, type); + if (last.flatMap(run -> active(run.id())).isPresent()) + throw new IllegalStateException("Can not start " + type + " for " + id + "; it is already running!"); -// PUT: - /** Stores the given details for the given job. */ - void store(JobDetails details, JobId job) { - ; + RunId newId = new RunId(id, type, last.map(run -> run.id().number()).orElse(0L) + 1); + curator.writeLastRun(RunStatus.initial(newId, controller.clock().instant())); + }); + }); } + /** Unregisters the given application and makes all associated data eligible for garbage collection. */ + public void unregister(ApplicationId id) { + controller.applications().lockIfPresent(id, application -> { + controller.applications().store(application.withBuiltInternally(false)); + jobs(id).forEach(type -> { + try (Lock __ = curator.lock(id, type)) { + last(id, type).ifPresent(last -> active(last.id()).ifPresent(active -> abort(active.id()))); + } + }); + }); + } + + /** Deletes stale data and tester deployments for applications which are unknown, or no longer built internally. */ + public void collectGarbage() { + Set<ApplicationId> applicationsToBuild = new HashSet<>(applications()); + curator.applicationsWithJobs().stream() + .filter(id -> ! applicationsToBuild.contains(id)) + .forEach(id -> { + try { + for (JobType type : jobs(id)) + locked(id, type, deactivateTester, __ -> { + try (Lock ___ = curator.lock(id, type)) { + deactivateTester(id, type); + curator.deleteJobData(id, type); + } + }); + } + catch (TimeoutException e) { + return; // Don't remove the data if we couldn't deactivate all testers. + } + curator.deleteJobData(id); + }); + } -// DELETE: - /** Unregisters the given application, and deletes all associated data. */ - void unregister(ApplicationId application) { - ; + // TODO jvenstad: Urgh, clean this up somehow? + public void deactivateTester(ApplicationId id, JobType type) { + try { + controller.configServer().deactivate(new DeploymentId(testerOf(id), type.zone(controller.system()).get())); + } + catch (NoInstanceException ignored) { + // ok; already gone + } } - /** Aborts the given job. */ - void abort(JobId job) { - ; + // TODO jvenstad: Find a more appropriate way of doing this, at least when this is the only build service. + private long nextBuild(ApplicationId id) { + return 1 + controller.applications().require(id).deploymentJobs() + .statusOf(JobType.component) + .flatMap(JobStatus::lastCompleted) + .map(JobStatus.JobRun::id) + .orElse(0L); + } + + // TODO jvenstad: Find a more appropriate way of doing this when this is the only build service. + private void notifyOfNewSubmission(ApplicationId id, SourceRevision revision, long number) { + DeploymentJobs.JobReport report = new DeploymentJobs.JobReport(id, + JobType.component, + Long.MAX_VALUE, // TODO jvenstad: Clean up this! + number, + Optional.of(revision), + Optional.empty()); + controller.applications().deploymentTrigger().notifyOfCompletion(report); + } + + /** Locks and modifies the list of historic runs for the given application and job type. */ + private void locked(ApplicationId id, JobType type, Consumer<Map<RunId, RunStatus>> modifications) { + try (Lock __ = curator.lock(id, type)) { + Map<RunId, RunStatus> runs = curator.readHistoricRuns(id, type); + modifications.accept(runs); + curator.writeHistoricRuns(id, type, runs.values()); + } + } + + /** Locks and modifies the run with the given id, provided it is still active. */ + private void locked(RunId id, UnaryOperator<RunStatus> modifications) { + try (Lock __ = curator.lock(id.application(), id.type())) { + RunStatus run = active(id).orElseThrow(() -> new IllegalArgumentException(id + " is not an active run!")); + run = modifications.apply(run); + curator.writeLastRun(run); + } + } + + /** Locks the given step and checks none of its prerequisites are running, then performs the given actions. */ + public void locked(ApplicationId id, JobType type, Step step, Consumer<LockedStep> action) throws TimeoutException { + try (Lock lock = curator.lock(id, type, step)) { + for (Step prerequisite : step.prerequisites()) // Check that no prerequisite is still running. + try (Lock __ = curator.lock(id, type, prerequisite)) { ; } + + action.accept(new LockedStep(lock, step)); + } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobDetails.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobDetails.java deleted file mode 100644 index 3c787c8314f..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobDetails.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.vespa.hosted.controller.api.ActivateResult; - -/** - * Contains details about a deployment job run. - * - * @author jonmv - */ -public class JobDetails { - - private final ActivateResult deploymentResult; - private final String convergenceLog; - private final String testLog; - - public JobDetails(ActivateResult deploymentResult, String convergenceLog, String testLog) { - this.deploymentResult = deploymentResult; - this.convergenceLog = convergenceLog; - this.testLog = testLog; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobId.java deleted file mode 100644 index 541494a23fc..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobId.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; - -import java.util.Objects; - -/** - * Immutable ID of a job run by an {@link InternalBuildService}. - * - * @author jonmv - */ -public class JobId { - - private final ApplicationId application; - private final ZoneId zone; - private final long number; - - public JobId(ApplicationId application, ZoneId zone, long number) { - this.application = Objects.requireNonNull(application, "ApplicationId cannot be null!"); - this.zone = Objects.requireNonNull(zone, "ZoneId cannot be null!"); - if (number <= 0) throw new IllegalArgumentException("Build number must be a positive integer!"); - this.number = number; - } - - public ApplicationId application() { return application; } - public ZoneId zone() { return zone; } - public long number() { return number; } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if ( ! (o instanceof JobId)) return false; - - JobId id = (JobId) o; - - if (number != id.number) return false; - if ( ! application.equals(id.application)) return false; - return zone == id.zone; - } - - @Override - public int hashCode() { - int result = application.hashCode(); - result = 31 * result + zone.hashCode(); - result = 31 * result + (int) (number ^ (number >>> 32)); - return result; - } - - @Override - public String toString() { - return "Run " + number + " in " + zone + " for " + application; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMeta.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMeta.java deleted file mode 100644 index dde675402ce..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMeta.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.yahoo.vespa.hosted.controller.deployment; - -import java.time.Instant; -import java.util.Optional; - -/** - * Contains state information for a deployment job run by an {@link InternalBuildService}. - * - * @author jonmv - */ -public class JobMeta { - - private final JobId id; - private final JobState state; - private final JobOutcome outcome; - private final Instant start; - private final Instant end; - - public JobMeta(JobId id, JobState state, JobOutcome outcome, Instant start, Instant end) { - this.id = id; - this.state = state; - this.outcome = outcome; - this.start = start; - this.end = end; - } - - public JobId id() { - return id; - } - - public JobState state() { - return state; - } - - public JobOutcome outcome() { - return outcome; - } - - public Instant start() { - return start; - } - - public Optional<Instant> end() { - return Optional.ofNullable(end); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java new file mode 100644 index 00000000000..0cad9e98d5d --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java @@ -0,0 +1,75 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; + +import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; + +import static com.yahoo.vespa.hosted.controller.deployment.Step.*; + +/** + * Static profiles defining the {@link Step}s of a deployment job. + * + * @author jonmv + */ +public enum JobProfile { + + // TODO jvenstad: runTests is not a run-always step, as it really means: check if tests are done, and store whatever is ready. + systemTest(EnumSet.of(deployReal, + installReal, + deployTester, + installTester, + startTests, + endTests), + EnumSet.of(deactivateTester, + deactivateReal, + report)), + + stagingTest(EnumSet.of(deployInitialReal, + installInitialReal, + deployReal, + installReal, + deployTester, + installTester, + startTests, + endTests), + EnumSet.of(deactivateTester, + deactivateReal, + report)), + + production(EnumSet.of(deployReal, + installReal, + deployTester, + installTester, + startTests, + endTests), + EnumSet.of(deactivateTester, + report)); + + + private final Set<Step> steps; + private final Set<Step> alwaysRun; + + JobProfile(Set<Step> runWhileSuccess, Set<Step> alwaysRun) { + runWhileSuccess.addAll(alwaysRun); + this.steps = Collections.unmodifiableSet(runWhileSuccess); + this.alwaysRun = Collections.unmodifiableSet(alwaysRun); + } + + public static JobProfile of(JobType type) { + switch (type.environment()) { + case test: return systemTest; + case staging: return stagingTest; + case prod: return production; + default: throw new AssertionError("Unexpected environment '" + type.environment() + "'!"); + } + } + + /** Returns all steps in this profile, the default for which is to run only when all prerequisites are successes. */ + public Set<Step> steps() { return steps; } + + /** Returns the set of steps that should always be run, regardless of outcome. */ + public Set<Step> alwaysRun() { return alwaysRun; } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobState.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobState.java deleted file mode 100644 index 19e575efaf8..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobState.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.yahoo.vespa.hosted.controller.deployment; - -/** - * Status of jobs run by an {@link InternalBuildService}. - * - * @author jonmv - */ -public enum JobState { - - /** Job is not currently running, and may be started. */ - idle, - - /** Real application is deploying. */ - deploying, - - /** Real application is converging. */ - converging, - - /** Tester is starting up, but is not yet ready to serve its status. */ - initializing, - - /** Job is up and running normally. */ - running, - - /** Tests are complete, and results may be fetched. */ - finished - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java new file mode 100644 index 00000000000..1a35169488a --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java @@ -0,0 +1,11 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.vespa.curator.Lock; + +public class LockedStep { + + private final Step step; + LockedStep(Lock lock, Step step) { this.step = step; } + public Step get() { return step; } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunDetails.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunDetails.java new file mode 100644 index 00000000000..60d7a6a8f04 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunDetails.java @@ -0,0 +1,26 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.google.common.collect.ImmutableMap; + +import java.util.Map; +import java.util.Optional; + +/** + * Contains details about a deployment job run. + * + * @author jonmv + */ +public class RunDetails { + + // TODO jvenstad: Store a serialised structure, rather than a flat text. + private final Map<Step, byte[]> logs; + + public RunDetails(Map<Step, byte[]> logs) { + this.logs = ImmutableMap.copyOf(logs); + } + + public Optional<byte[]> get(Step step) { + return Optional.ofNullable(logs.get(step)); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobOutcome.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java index caecdcffb9b..aaf43097908 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobOutcome.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java @@ -5,7 +5,7 @@ package com.yahoo.vespa.hosted.controller.deployment; * * @author jonmv */ -public enum JobOutcome { +public enum RunResult { /** Deployment of the real application was rejected due to missing capacity. */ outOfCapacity, @@ -13,8 +13,8 @@ public enum JobOutcome { /** Deployment of the real application was rejected. */ deploymentFailed, - /** Convergence of the real application timed out. */ - convergenceFailed, + /** Installation of the real application timed out. */ + installationFailed, /** Real application was deployed, but the tester application was not. */ testError, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java new file mode 100644 index 00000000000..a91cc905add --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java @@ -0,0 +1,166 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.google.common.collect.ImmutableList; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; + +import java.time.Instant; +import java.util.Collections; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; +import static java.util.Objects.requireNonNull; + +/** + * Immutable class containing status information for a deployment job run by an {@link InternalBuildService}. + * + * @author jonmv + */ +public class RunStatus { + + private final RunId id; + private final Map<Step, Step.Status> steps; + private final Instant start; + private final Optional<Instant> end; + private final boolean aborted; + // TODO jvenstad: Add a Versions object and a reason String. Requires shortcutting of triggering of these runs. + + // For deserialisation only -- do not use! + public RunStatus(RunId id, Map<Step, Step.Status> steps, Instant start, Optional<Instant> end, boolean aborted) { + this.id = id; + this.steps = Collections.unmodifiableMap(new EnumMap<>(steps)); + this.start = start; + this.end = end; + this.aborted = aborted; + } + + public static RunStatus initial(RunId id, Instant now) { + EnumMap<Step, Step.Status> steps = new EnumMap<>(Step.class); + JobProfile.of(id.type()).steps().forEach(step -> steps.put(step, unfinished)); + return new RunStatus(id, steps, requireNonNull(now), Optional.empty(), false); + } + + public RunStatus with(Step.Status status, LockedStep step) { + if (hasEnded()) + throw new AssertionError("This step ended at " + end.get() + " -- it can't be further modified!"); + + EnumMap<Step, Step.Status> steps = new EnumMap<>(this.steps); + steps.put(step.get(), requireNonNull(status)); + return new RunStatus(id, steps, start, end, aborted); + } + + public RunStatus finished(Instant now) { + if (hasEnded()) + throw new AssertionError("This step ended at " + end.get() + " -- it can't be ended again!"); + + return new RunStatus(id, new EnumMap<>(steps), start, Optional.of(now), aborted); + } + + public RunStatus aborted() { + if (hasEnded()) + throw new AssertionError("This step ended at " + end.get() + " -- it can't be aborted now!"); + + return new RunStatus(id, new EnumMap<>(steps), start, end, true); + } + + /** Returns the id of this run. */ + public RunId id() { + return id; + } + + /** Returns an unmodifiable view of the status of all steps in this run. */ + public Map<Step, Step.Status> steps() { + return steps; + } + + /** Returns the final result of this run, if it has ended. */ + public Optional<RunResult> result() { + // TODO jvenstad: To implement, or not ... If so, base on status. + throw new AssertionError(); + } + + /** Returns the instant at which this run began. */ + public Instant start() { + return start; + } + + /** Returns the instant at which this run ended, if it has. */ + public Optional<Instant> end() { + return end; + } + + /** Returns whether the run has failed, and should switch to its run-always steps. */ + public boolean hasFailed() { + return aborted || steps.values().contains(failed); + } + + /** Returns whether the run has been forcefully aborted. */ + public boolean isAborted() { + return aborted; + } + + /** Returns whether the run has ended, i.e., has become inactive, and can no longer be updated. */ + public boolean hasEnded() { + return end.isPresent(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if ( ! (o instanceof RunStatus)) return false; + + RunStatus status = (RunStatus) o; + + return id.equals(status.id); + } + + @Override + public int hashCode() { + return id.hashCode(); + } + + @Override + public String toString() { + return "RunStatus{" + + "id=" + id + + ", start=" + start + + ", end=" + end + + ", aborted=" + aborted + + ", steps=" + steps + + '}'; + } + + /** Returns the list of steps to run for this job right now, depending on whether the job has failed. */ + public List<Step> readySteps() { + return hasFailed() ? forcedSteps() : normalSteps(); + } + + /** Returns the list of unfinished steps whose prerequisites have all succeeded. */ + private List<Step> normalSteps() { + return ImmutableList.copyOf(steps.entrySet().stream() + .filter(entry -> entry.getValue() == unfinished + && entry.getKey().prerequisites().stream() + .allMatch(step -> steps.get(step) == null + || steps.get(step) == succeeded)) + .map(Map.Entry::getKey) + .iterator()); + } + + /** Returns the list of not-yet-succeeded run-always steps whose run-always prerequisites have all succeeded. */ + private List<Step> forcedSteps() { + return ImmutableList.copyOf(steps.entrySet().stream() + .filter(entry -> entry.getValue() != succeeded + && JobProfile.of(id.type()).alwaysRun().contains(entry.getKey()) + && entry.getKey().prerequisites().stream() + .filter(JobProfile.of(id.type()).alwaysRun()::contains) + .allMatch(step -> steps.get(step) == null + || steps.get(step) == succeeded)) + .map(Map.Entry::getKey) + .iterator()); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java new file mode 100644 index 00000000000..eeb85dc2506 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -0,0 +1,85 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.google.common.collect.ImmutableList; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * Steps that make up a deployment job. See {@link JobProfile} for preset profiles. + * + * Each step lists its prerequisites; this serves two purposes: + * + * 1. A step may only run after its prerequisites, so these define a topological order in which + * the steps can be run. Since a job profile may list only a subset of the existing steps, + * only the prerequisites of a step which are included in a run's profile will be considered. + * Under normal circumstances, a step will run only after each of its prerequisites have succeeded. + * When a run has failed, however, each of the always-run steps of the run's profile will be run, + * again in a topological order, and again requiring success of all their always-run prerequisites. + * + * 2. A step will never run concurrently with its prerequisites. This is to ensure, e.g., that relevant + * information from a failed run is stored, and that deployment does not occur after deactivation. + * + * @see JobController + * @author jonmv + */ +public enum Step { + + /** Download and deploy the initial real application, for staging tests. */ + deployInitialReal, + + /** See that the real application has had its nodes converge to the initial state. */ + installInitialReal(deployInitialReal), + + /** Download and deploy real application, restarting services if required. */ + deployReal(installInitialReal), + + /** See that real application has had its nodes converge to the wanted version and generation. */ + installReal(deployReal), + + /** Download test-jar and assemble and deploy tester application. */ + deployTester, + + /** See that tester is done deploying, and is ready to serve. */ + installTester(deployTester), + + /** Ask the tester to run its tests. */ + startTests(installReal, installTester), + + /** See that the tests are done running. */ + endTests(startTests), + + /** Delete the real application -- used for test deployments. */ + deactivateReal(deployInitialReal, deployReal, endTests), + + /** Deactivate the tester. */ + deactivateTester(deployTester, endTests), + + /** Report completion to the deployment orchestration machinery. */ + report(deactivateReal, deactivateTester); + + + private final List<Step> prerequisites; + + Step(Step... prerequisites) { + this.prerequisites = ImmutableList.copyOf(prerequisites); + } + + public List<Step> prerequisites() { return prerequisites; } + + + public enum Status { + + /** Step still has unsatisfied finish criteria -- it may not even have started. */ + unfinished, + + /** Step failed and subsequent steps may not start. */ + failed, + + /** Step succeeded and subsequent steps may now start. */ + succeeded; + + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java new file mode 100644 index 00000000000..cf024064cc4 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java @@ -0,0 +1,25 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.deployment.LockedStep; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import com.yahoo.vespa.hosted.controller.deployment.Step; + +/** + * Advances a given job run by running the appropriate {@link Step}s, based on their current status. + * + * When an attempt is made to advance a given job, a lock for that job (application and type) is + * taken, and released again only when the attempt finishes. Multiple other attempts may be made in + * the meantime, but they should give up unless the lock is promptly acquired. + * + * @author jonmv + */ +public interface StepRunner { + + /** Attempts to run the given locked step in the given run, and returns its new status. */ + Step.Status run(LockedStep step, RunId id); + +} + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/package-info.java new file mode 100644 index 00000000000..e8fb638bc34 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 105435f0346..7fa16a02649 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -4,11 +4,14 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.Testers; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues; import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; +import com.yahoo.vespa.hosted.controller.deployment.DummyStepRunner; +import com.yahoo.vespa.hosted.controller.deployment.InternalStepRunner; import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -38,10 +41,11 @@ public class ControllerMaintenance extends AbstractComponent { private final ApplicationOwnershipConfirmer applicationOwnershipConfirmer; private final DnsMaintainer dnsMaintainer; private final SystemUpgrader systemUpgrader; + private final JobRunner jobRunner; @SuppressWarnings("unused") // instantiated by Dependency Injection public ControllerMaintenance(MaintainerConfig maintainerConfig, Controller controller, CuratorDb curator, - JobControl jobControl, Metric metric, Chef chefClient, + JobControl jobControl, Metric metric, Chef chefClient, Testers testers, DeploymentIssues deploymentIssues, OwnershipIssues ownershipIssues, NameService nameService, NodeRepositoryClientInterface nodeRepositoryClient) { Duration maintenanceInterval = Duration.ofMinutes(maintainerConfig.intervalMinutes()); @@ -59,6 +63,7 @@ public class ControllerMaintenance extends AbstractComponent { applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, ownershipIssues); dnsMaintainer = new DnsMaintainer(controller, Duration.ofHours(12), jobControl, nameService); systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl); + jobRunner = new JobRunner(controller, Duration.ofSeconds(30), jobControl, new InternalStepRunner(controller, testers)); } public Upgrader upgrader() { return upgrader; } @@ -81,6 +86,7 @@ public class ControllerMaintenance extends AbstractComponent { applicationOwnershipConfirmer.deconstruct(); dnsMaintainer.deconstruct(); systemUpgrader.deconstruct(); + jobRunner.deconstruct(); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java new file mode 100644 index 00000000000..7dbf1a2c05e --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -0,0 +1,101 @@ +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.deployment.InternalBuildService; +import com.yahoo.vespa.hosted.controller.deployment.JobController; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import com.yahoo.vespa.hosted.controller.deployment.Step; +import com.yahoo.vespa.hosted.controller.deployment.StepRunner; +import org.jetbrains.annotations.TestOnly; + +import java.time.Duration; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Logger; + +/** + * Advances the set of {@link RunStatus}es for an {@link InternalBuildService}. + * + * @see JobController + * @author jonmv + */ +public class JobRunner extends Maintainer { + + private static final Logger log = Logger.getLogger(JobRunner.class.getName()); + + private final JobController jobs; + private final ExecutorService executors; + private final StepRunner runner; + + public JobRunner(Controller controller, Duration duration, JobControl jobControl, StepRunner runner) { + this(controller, duration, jobControl, Executors.newFixedThreadPool(32), runner); + } + + @TestOnly + JobRunner(Controller controller, Duration duration, JobControl jobControl, ExecutorService executors, StepRunner runner) { + super(controller, duration, jobControl); + this.jobs = controller.jobController(); + this.executors = executors; + this.runner = runner; + } + + @Override + protected void maintain() { + jobs.active().forEach(this::advance); + jobs.collectGarbage(); + } + + @Override + public void deconstruct() { + super.deconstruct(); + executors.shutdown(); + try { + executors.awaitTermination(50, TimeUnit.SECONDS); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + /** Advances each of the ready steps for the given run, or marks it as finished, and stashes it. */ + void advance(RunStatus run) { + List<Step> steps = run.readySteps(); + steps.forEach(step -> executors.execute(() -> advance(run.id(), step))); + if (steps.isEmpty()) + jobs.finish(run.id()); + } + + /** Attempts to advance the status of the given step, for the given run. */ + void advance(RunId id, Step step) { + try { + AtomicBoolean changed = new AtomicBoolean(false); + jobs.locked(id.application(), id.type(), step, lockedStep -> { + jobs.active(id).ifPresent(run -> { // The run may have become inactive, so we bail out. + if ( ! run.readySteps().contains(step)) + return; // Someone may have updated the run status, making this step obsolete, so we bail out. + + Step.Status status = runner.run(lockedStep, run.id()); + if (run.steps().get(step) != status) { + jobs.update(run.id(), status, lockedStep); + changed.set(true); + } + }); + }); + if (changed.get()) + jobs.active(id).ifPresent(this::advance); + } + catch (TimeoutException e) { + // Something else is already advancing this step, or a prerequisite -- try again later! + } + catch (RuntimeException e) { + log.log(LogLevel.WARNING, "Exception attempting to advance " + step + " of " + id, e); + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java index 40563c4cf95..f6ccbf6aa4e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java @@ -12,6 +12,7 @@ import java.time.Duration; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.logging.Level; import java.util.logging.Logger; @@ -50,7 +51,7 @@ public abstract class Maintainer extends AbstractComponent implements Runnable { } } } - catch (UncheckedTimeoutException e) { + catch (TimeoutException e) { // another controller instance is running this job at the moment; ok } catch (Throwable t) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index d804afdf98e..763d26834e6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -80,6 +80,7 @@ public class ApplicationSerializer { private final String projectIdField = "projectId"; private final String jobStatusField = "jobStatus"; private final String issueIdField = "jiraIssueId"; + private final String builtInternallyField = "builtInternally"; // JobStatus field private final String jobTypeField = "jobType"; @@ -227,6 +228,7 @@ public class ApplicationSerializer { deploymentJobs.projectId().ifPresent(projectId -> cursor.setLong(projectIdField, projectId)); jobStatusToSlime(deploymentJobs.jobStatus().values(), cursor.setArray(jobStatusField)); deploymentJobs.issueId().ifPresent(jiraIssueId -> cursor.setString(issueIdField, jiraIssueId.value())); + cursor.setBool(builtInternallyField, deploymentJobs.builtInternally()); } private void jobStatusToSlime(Collection<JobStatus> jobStatuses, Cursor jobStatusArray) { @@ -374,8 +376,9 @@ public class ApplicationSerializer { OptionalLong projectId = optionalLong(object.field(projectIdField)); List<JobStatus> jobStatusList = jobStatusListFromSlime(object.field(jobStatusField)); Optional<IssueId> issueId = optionalString(object.field(issueIdField)).map(IssueId::from); + boolean builtInternally = object.field(builtInternallyField).asBool(); - return new DeploymentJobs(projectId, jobStatusList, issueId); + return new DeploymentJobs(projectId, jobStatusList, issueId, builtInternally); } private Change changeFromSlime(Inspector object) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 184ac90691a..49e8d7498ab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; +import com.google.common.util.concurrent.UncheckedTimeoutException; import com.google.inject.Inject; import com.yahoo.component.Version; import com.yahoo.component.Vtag; @@ -13,6 +14,10 @@ import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.tenant.UserTenant; @@ -26,11 +31,13 @@ import java.time.Duration; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeoutException; import java.util.function.Function; import java.util.function.Predicate; import java.util.logging.Level; @@ -53,6 +60,7 @@ public class CuratorDb { private static final Path lockRoot = root.append("locks"); private static final Path tenantRoot = root.append("tenants"); private static final Path applicationRoot = root.append("applications"); + private static final Path jobRoot = root.append("jobs"); private static final Path controllerRoot = root.append("controllers"); private final StringSetSerializer stringSetSerializer = new StringSetSerializer(); @@ -61,6 +69,7 @@ public class CuratorDb { private final ConfidenceOverrideSerializer confidenceOverrideSerializer = new ConfidenceOverrideSerializer(); private final TenantSerializer tenantSerializer = new TenantSerializer(); private final ApplicationSerializer applicationSerializer = new ApplicationSerializer(); + private final RunSerializer runSerializer = new RunSerializer(); private final Curator curator; @@ -93,12 +102,20 @@ public class CuratorDb { return lock; } - public Lock lock(TenantName name, Duration timeout) { - return lock(lockPath(name), timeout); + public Lock lock(TenantName name) { + return lock(lockPath(name), defaultLockTimeout.multipliedBy(2)); } - public Lock lock(ApplicationId id, Duration timeout) { - return lock(lockPath(id), timeout); + public Lock lock(ApplicationId id) { + return lock(lockPath(id), defaultLockTimeout.multipliedBy(2)); + } + + public Lock lock(ApplicationId id, JobType type) { + return lock(lockPath(id, type), defaultLockTimeout); + } + + public Lock lock(ApplicationId id, JobType type, Step step) throws TimeoutException { + return tryLock(lockPath(id, type, step)); } public Lock lockRotations() { @@ -113,11 +130,8 @@ public class CuratorDb { return lock(lockRoot.append("inactiveJobsLock"), defaultLockTimeout); } - public Lock lockMaintenanceJob(String jobName) { - // Use a short timeout such that if maintenance jobs are started at about the same time on different nodes - // and the maintenance job takes a long time to complete, only one of the nodes will run the job - // in each maintenance interval - return lock(lockRoot.append("maintenanceJobLocks").append(jobName), Duration.ofSeconds(1)); + public Lock lockMaintenanceJob(String jobName) throws TimeoutException { + return tryLock(lockRoot.append("maintenanceJobLocks").append(jobName)); } @SuppressWarnings("unused") // Called by internal code @@ -137,6 +151,19 @@ public class CuratorDb { // -------------- Helpers ------------------------------------------ + /** Try locking with a low timeout, meaning it is OK to fail lock acquisition. + * + * Useful for maintenance jobs, where there is no point in running the jobs back to back. + */ + private Lock tryLock(Path path) throws TimeoutException { + try { + return lock(path, Duration.ofSeconds(1)); + } + catch (UncheckedTimeoutException e) { + throw new TimeoutException(e.getMessage()); + } + } + private <T> Optional<T> read(Path path, Function<byte[], T> mapper) { return curator.getData(path).filter(data -> data.length > 0).map(mapper); } @@ -278,6 +305,40 @@ public class CuratorDb { curator.delete(applicationPath(application)); } + // -------------- Job Runs ------------------------------------------------ + + public void writeLastRun(RunStatus run) { + curator.set(lastRunPath(run.id().application(), run.id().type()), asJson(runSerializer.toSlime(run))); + } + + public void writeHistoricRuns(ApplicationId id, JobType type, Iterable<RunStatus> runs) { + curator.set(jobPath(id, type), asJson(runSerializer.toSlime(runs))); + } + + public Optional<RunStatus> readLastRun(ApplicationId id, JobType type) { + return readSlime(lastRunPath(id, type)).map(runSerializer::runFromSlime); + } + + public Map<RunId, RunStatus> readHistoricRuns(ApplicationId id, JobType type) { + // TODO jvenstad: Add, somewhere, a retention filter based on age or count. + return readSlime(jobPath(id, type)).map(runSerializer::runsFromSlime).orElse(new LinkedHashMap<>()); + } + + public void deleteJobData(ApplicationId id, JobType type) { + curator.delete(jobPath(id, type)); + curator.delete(lastRunPath(id, type)); + } + + public void deleteJobData(ApplicationId id) { + curator.delete(jobRoot.append(id.serializedForm())); + } + + public List<ApplicationId> applicationsWithJobs() { + return curator.getChildren(jobRoot).stream() + .map(ApplicationId::fromSerializedForm) + .collect(Collectors.toList()); + } + // -------------- Provisioning (called by internal code) ------------------ @SuppressWarnings("unused") @@ -333,6 +394,27 @@ public class CuratorDb { return lockPath; } + private Path lockPath(ApplicationId application, JobType type) { + Path lockPath = lockRoot + .append(application.tenant().value()) + .append(application.application().value()) + .append(application.instance().value()) + .append(type.jobName()); + curator.create(lockPath); + return lockPath; + } + + private Path lockPath(ApplicationId application, JobType type, Step step) { + Path lockPath = lockRoot + .append(application.tenant().value()) + .append(application.application().value()) + .append(application.instance().value()) + .append(type.jobName()) + .append(step.name()); + curator.create(lockPath); + return lockPath; + } + private Path lockPath(String provisionId) { Path lockPath = lockRoot .append(provisionStatePath()) @@ -381,6 +463,14 @@ public class CuratorDb { return applicationRoot.append(application.serializedForm()); } + private static Path jobPath(ApplicationId id, JobType type) { + return jobRoot.append(id.serializedForm()).append(type.jobName()); + } + + private static Path lastRunPath(ApplicationId id, JobType type) { + return jobPath(id, type).append("last"); + } + private static Path controllerPath(String hostname) { return controllerRoot.append(hostname); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java new file mode 100644 index 00000000000..7df60278390 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -0,0 +1,157 @@ +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.ObjectTraverser; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import com.yahoo.vespa.hosted.controller.deployment.Step; +import com.yahoo.vespa.hosted.controller.deployment.Step.Status; + +import java.time.Instant; +import java.util.EnumMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.report; +import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; +import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; + +/** + * Serialises and deserialises RunStatus objects for persistent storage. + * + * @author jonmv + */ +public class RunSerializer { + + private static final String stepsField = "steps"; + private static final String applicationField = "id"; + private static final String jobTypeField = "type"; + private static final String numberField = "number"; + private static final String startField = "start"; + private static final String endField = "end"; + private static final String abortedField = "aborted"; + + RunStatus runFromSlime(Slime slime) { + return runFromSlime(slime.get()); + } + + Map<RunId, RunStatus> runsFromSlime(Slime slime) { + Map<RunId, RunStatus> runs = new LinkedHashMap<>(); + Inspector runArray = slime.get(); + runArray.traverse((ArrayTraverser) (__, runObject) -> { + RunStatus run = runFromSlime(runObject); + runs.put(run.id(), run); + }); + return runs; + } + + private RunStatus runFromSlime(Inspector runObject) { + EnumMap<Step, Status> steps = new EnumMap<>(Step.class); + runObject.field(stepsField).traverse((ObjectTraverser) (step, status) -> { + steps.put(stepOf(step), statusOf(status.asString())); + }); + return new RunStatus(new RunId(ApplicationId.fromSerializedForm(runObject.field(applicationField).asString()), + JobType.fromJobName(runObject.field(jobTypeField).asString()), + runObject.field(numberField).asLong()), + steps, + Instant.ofEpochMilli(runObject.field(startField).asLong()), + Optional.of(runObject.field(endField)) + .filter(Inspector::valid) + .map(end -> Instant.ofEpochMilli(end.asLong())), + runObject.field(abortedField).asBool()); + } + + Slime toSlime(Iterable<RunStatus> runs) { + Slime slime = new Slime(); + Cursor runArray = slime.setArray(); + runs.forEach(run -> toSlime(run, runArray.addObject())); + return slime; + } + + Slime toSlime(RunStatus run) { + Slime slime = new Slime(); + toSlime(run, slime.setObject()); + return slime; + } + + private void toSlime(RunStatus run, Cursor runObject) { + runObject.setString(applicationField, run.id().application().serializedForm()); + runObject.setString(jobTypeField, run.id().type().jobName()); + runObject.setLong(numberField, run.id().number()); + runObject.setLong(startField, run.start().toEpochMilli()); + run.end().ifPresent(end -> runObject.setLong(endField, end.toEpochMilli())); + if (run.isAborted()) runObject.setBool(abortedField, true); + Cursor stepsObject = runObject.setObject(stepsField); + run.steps().forEach((step, status) -> stepsObject.setString(valueOf(step), valueOf(status))); + } + + static String valueOf(Step step) { + switch (step) { + case deployInitialReal : return "DIR"; + case installInitialReal : return "IIR"; + case deployReal : return "DR" ; + case installReal : return "IR" ; + case deactivateReal : return "DAR"; + case deployTester : return "DT" ; + case installTester : return "IT" ; + case deactivateTester : return "DAT"; + case startTests : return "ST" ; + case endTests : return "ET" ; + case report : return "R" ; + default : throw new AssertionError("No value defined for '" + step + "'!"); + } + } + + static Step stepOf(String step) { + switch (step) { + case "DIR" : return deployInitialReal ; + case "IIR" : return installInitialReal; + case "DR" : return deployReal ; + case "IR" : return installReal ; + case "DAR" : return deactivateReal ; + case "DT" : return deployTester ; + case "IT" : return installTester ; + case "DAT" : return deactivateTester ; + case "ST" : return startTests ; + case "ET" : return endTests ; + case "R" : return report ; + default : throw new IllegalArgumentException("No step defined by '" + step + "'!"); + } + } + + static String valueOf(Status status) { + switch (status) { + case unfinished : return "U"; + case failed : return "F"; + case succeeded : return "S"; + default : throw new AssertionError("No value defined for '" + status + "'!"); + } + } + + static Status statusOf(String status) { + switch (status) { + case "U" : return unfinished; + case "F" : return failed ; + case "S" : return succeeded ; + default : throw new IllegalArgumentException("No status defined by '" + status + "'!"); + } + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 5b818288b06..6e953dcdd57 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -29,7 +29,6 @@ import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.BuildJob; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; -import com.yahoo.vespa.hosted.controller.persistence.ApplicationSerializer; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import com.yahoo.vespa.hosted.controller.rotation.RotationLock; import org.junit.Test; @@ -211,52 +210,6 @@ public class ControllerTest { } @Test - public void testPullRequestDeployment() { - // Setup system - ControllerTester tester = new ControllerTester(); - ApplicationController applications = tester.controller().applications(); - - // staging deployment - long app1ProjectId = 22; - ApplicationId app1 = tester.createAndDeploy("tenant1", "domain1", - "application1", Environment.staging, - app1ProjectId).id(); - - // pull-request deployment - uses different instance id - ApplicationId app1pr = tester.createAndDeploy("tenant1", "domain1", - "application1", "1", - Environment.staging, app1ProjectId, null).id(); - - assertTrue(applications.get(app1).isPresent()); - assertEquals(app1, applications.get(app1).get().id()); - assertTrue(applications.get(app1pr).isPresent()); - assertEquals(app1pr, applications.get(app1pr).get().id()); - - // Simulate restart - tester.createNewController(); - applications = tester.controller().applications(); - - assertTrue(applications.get(app1).isPresent()); - assertEquals(app1, applications.get(app1).get().id()); - assertTrue(applications.get(app1pr).isPresent()); - assertEquals(app1pr, applications.get(app1pr).get().id()); - - // Deleting application also removes PR instance - ApplicationId app2 = tester.createAndDeploy("tenant1", "domain1", - "application2", Environment.staging, - 33).id(); - tester.controller().applications().deleteApplication(app1, Optional.of(new NToken("ntoken"))); - assertEquals("All instances deleted", 0, - tester.controller().applications().asList(app1.tenant()).stream() - .filter(app -> app.id().application().equals(app1.application())) - .count()); - assertEquals("Other application survives", 1, - tester.controller().applications().asList(app1.tenant()).stream() - .filter(app -> app.id().application().equals(app2.application())) - .count()); - } - - @Test public void testGlobalRotations() throws IOException { // Setup tester and app def ControllerTester tester = new ControllerTester(); @@ -539,35 +492,6 @@ public class ControllerTest { tester.applications().require(app.id()).deploymentJobs().jobStatus().isEmpty()); } - @Test - public void testDeploymentOfNewInstanceWithIllegalApplicationName() { - ControllerTester tester = new ControllerTester(); - String application = "this_application_name_is_far_too_long_and_has_underscores"; - ZoneId zone = ZoneId.from("test", "us-east-1"); - DeployOptions options = new DeployOptions(false, - Optional.empty(), - false, - false); - - tester.createTenant("tenant", "domain", null); - - // Deploy an application which doesn't yet exist, and which has an illegal application name. - try { - tester.controller().applications().deploy(ApplicationId.from("tenant", application, "123"), zone, Optional.empty(), options); - fail("Illegal application name should cause validation exception."); - } - catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("Invalid id")); - } - - // Sneak an illegal application in the back door. - tester.createApplication(new ApplicationSerializer().toSlime(new Application(ApplicationId.from("tenant", application, "default")))); - - // Deploy a PR instance for the application, with no NToken. - tester.controller().applications().deploy(ApplicationId.from("tenant", application, "456"), zone, Optional.empty(), options); - assertTrue(tester.controller().applications().get(ApplicationId.from("tenant", application, "456")).isPresent()); - } - private void runUpgrade(DeploymentTester tester, ApplicationId application, ApplicationVersion version) { Version next = Version.fromString("6.2"); tester.upgradeSystem(next); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java index 04c670cf136..c722d30c885 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java @@ -47,6 +47,21 @@ public class ArtifactRepositoryMock extends AbstractComponent implements Artifac } @Override + public void putApplicationPackage(ApplicationId application, String applicationVersion, byte[] applicationPackage) { + throw new AssertionError(); + } + + @Override + public byte[] getTesterJar(ApplicationId tester, String applicationVersion) { + throw new AssertionError(); + } + + @Override + public void putTesterJar(ApplicationId tester, String applicationVersion, byte[] fatTestJar) { + throw new AssertionError(); + } + + @Override public byte[] getSystemApplicationPackage(ApplicationId application, ZoneId zone, Version version) { return new byte[0]; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java new file mode 100644 index 00000000000..e084e9aa46d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -0,0 +1,262 @@ +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.TestIdentities; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.deployment.JobController; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import com.yahoo.vespa.hosted.controller.deployment.Step; +import com.yahoo.vespa.hosted.controller.deployment.Step.Status; +import com.yahoo.vespa.hosted.controller.deployment.StepRunner; +import org.junit.Test; + +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.report; +import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; +import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author jonmv + */ +public class JobRunnerTest { + + @Test + public void multiThreadedExecutionFinishes() throws InterruptedException { + DeploymentTester tester = new DeploymentTester(); + JobController jobs = tester.controller().jobController(); + // Fail the installation of the initial version of the real application in staging tests, and succeed everything else. + StepRunner stepRunner = (step, id) -> id.type() == stagingTest && step.get() == startTests? failed : succeeded; + CountDownLatch latch = new CountDownLatch(19); // Number of steps that will run, below: all but endTests in staging and all 9 in system. + JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), + Executors.newFixedThreadPool(32), notifying(stepRunner, latch)); + + ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); + jobs.submit(id, new SourceRevision("repo", "branch", "bada55"), new byte[0], new byte[0]); + + jobs.start(id, systemTest); + try { + jobs.start(id, systemTest); + fail("Job is already running, so this should not be allowed!"); + } + catch (IllegalStateException e) { } + jobs.start(id, stagingTest); + + assertTrue(jobs.last(id, systemTest).get().steps().values().stream().allMatch(unfinished::equals)); + runner.maintain(); + assertFalse(jobs.last(id, systemTest).get().hasEnded()); + assertFalse(jobs.last(id, stagingTest).get().hasEnded()); + + latch.await(1, TimeUnit.SECONDS); + assertEquals(0, latch.getCount()); + + runner.deconstruct(); // Ensures all workers have finished writing to the curator. + assertTrue(jobs.last(id, systemTest).get().steps().values().stream().allMatch(succeeded::equals)); + assertTrue(jobs.last(id, stagingTest).get().hasEnded()); + assertTrue(jobs.last(id, stagingTest).get().hasFailed()); + } + + @Test + public void stepLogic() { + DeploymentTester tester = new DeploymentTester(); + JobController jobs = tester.controller().jobController(); + Map<Step, Status> outcomes = new EnumMap<>(Step.class); + JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), + inThreadExecutor(), mappedRunner(outcomes)); + + ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); + jobs.submit(id, new SourceRevision("repo", "branch", "bada55"), new byte[0], new byte[0]); + Supplier<RunStatus> run = () -> jobs.last(id, systemTest).get(); + + jobs.start(id, systemTest); + RunId first = run.get().id(); + + Map<Step, Status> steps = run.get().steps(); + runner.maintain(); + assertEquals(steps, run.get().steps()); + assertEquals(Arrays.asList(deployReal, deployTester), run.get().readySteps()); + + outcomes.put(deployReal, succeeded); + runner.maintain(); + assertEquals(Arrays.asList(installReal, deployTester), run.get().readySteps()); + + outcomes.put(installReal, succeeded); + runner.maintain(); + assertEquals(Arrays.asList(deployTester), run.get().readySteps()); + + outcomes.put(deployTester, succeeded); + runner.maintain(); + assertEquals(Arrays.asList(installTester), run.get().readySteps()); + + outcomes.put(installTester, succeeded); + runner.maintain(); + assertEquals(Arrays.asList(startTests), run.get().readySteps()); + + outcomes.put(startTests, succeeded); + runner.maintain(); + assertEquals(Arrays.asList(endTests), run.get().readySteps()); + + outcomes.put(endTests, succeeded); + runner.maintain(); + assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps()); + + // Failure deactivating real fails the run, but run-always steps continue. + outcomes.put(deactivateReal, failed); + runner.maintain(); + assertTrue(run.get().hasFailed()); + assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps()); + + // Abortion does nothing, as the run has already failed. + jobs.abort(run.get().id()); + runner.maintain(); + assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps()); + + outcomes.put(deactivateReal, succeeded); + outcomes.put(deactivateTester, succeeded); + outcomes.put(report, succeeded); + runner.maintain(); + assertTrue(run.get().hasFailed()); + assertTrue(run.get().hasEnded()); + assertTrue(run.get().isAborted()); + + // A new run is attempted. + jobs.start(id, systemTest); + assertEquals(first.number() + 1, run.get().id().number()); + + // Run fails on tester deployment -- remaining run-always steps succeed, and the run finishes. + outcomes.put(deployTester, failed); + runner.maintain(); + assertTrue(run.get().hasEnded()); + assertTrue(run.get().hasFailed()); + assertFalse(run.get().isAborted()); + assertEquals(failed, run.get().steps().get(deployTester)); + assertEquals(unfinished, run.get().steps().get(installTester)); + assertEquals(succeeded, run.get().steps().get(report)); + + assertEquals(2, jobs.runs(id, systemTest).size()); + + // Start a third run, then unregister and wait for data to be deleted. + jobs.start(id, systemTest); + jobs.unregister(id); + runner.maintain(); + assertFalse(jobs.last(id, systemTest).isPresent()); + assertTrue(jobs.runs(id, systemTest).isEmpty()); + } + + @Test + public void locksAndGarbage() throws InterruptedException, BrokenBarrierException { + DeploymentTester tester = new DeploymentTester(); + JobController jobs = tester.controller().jobController(); + // Hang during tester deployment, until notified. + CyclicBarrier barrier = new CyclicBarrier(2); + JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), + Executors.newFixedThreadPool(32), waitingRunner(barrier)); + + ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); + jobs.submit(id, new SourceRevision("repo", "branch", "bada55"), new byte[0], new byte[0]); + + RunId runId = new RunId(id, systemTest, 1); + jobs.start(id, systemTest); + runner.maintain(); + barrier.await(); + try { + jobs.locked(id, systemTest, deactivateTester, step -> { }); + fail("deployTester step should still be locked!"); + } + catch (TimeoutException e) { } + + // Thread is still trying to deploy tester -- delete application, and see all data is garbage collected. + assertEquals(Collections.singletonList(runId), jobs.active().stream().map(run -> run.id()).collect(Collectors.toList())); + tester.controller().applications().deleteApplication(id, Optional.of(TestIdentities.userNToken)); + assertEquals(Collections.emptyList(), jobs.active()); + assertEquals(runId, jobs.last(id, systemTest).get().id()); + + // Deployment still ongoing, so garbage is not yet collected. + runner.maintain(); + assertEquals(runId, jobs.last(id, systemTest).get().id()); + + // Deployment lets go, deactivation may now run, and trash is thrown out. + barrier.await(); + runner.maintain(); + assertEquals(Optional.empty(), jobs.last(id, systemTest)); + } + + private static ExecutorService inThreadExecutor() { + return new AbstractExecutorService() { + AtomicBoolean shutDown = new AtomicBoolean(false); + @Override public void shutdown() { shutDown.set(true); } + @Override public List<Runnable> shutdownNow() { shutDown.set(true); return Collections.emptyList(); } + @Override public boolean isShutdown() { return shutDown.get(); } + @Override public boolean isTerminated() { return shutDown.get(); } + @Override public boolean awaitTermination(long timeout, TimeUnit unit) { return true; } + @Override public void execute(Runnable command) { command.run(); } + }; + } + + private static StepRunner notifying(StepRunner runner, CountDownLatch latch) { + return (step, id) -> { + Status status = runner.run(step, id); + synchronized (latch) { + assertTrue(latch.getCount() > 0); + latch.countDown(); + } + return status; + }; + } + + private static StepRunner mappedRunner(Map<Step, Status> outcomes) { + return (step, id) -> outcomes.getOrDefault(step.get(), Status.unfinished); + } + + private static StepRunner waitingRunner(CyclicBarrier barrier) { + return (step, id) -> { + try { + if (step.get() == deployTester) { + barrier.await(); // Wake up the main thread, which waits for this step to be locked. + barrier.reset(); + barrier.await(); // Then wait while holding the lock for this step, until the main thread wakes us up. + } + } + catch (InterruptedException | BrokenBarrierException e) { + throw new AssertionError(e); + } + return succeeded; + }; + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index e96d41f5b44..a17584f9bfa 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -87,7 +87,7 @@ public class ApplicationSerializerTest { .withTriggering(Version.fromString("5.6.6"), ApplicationVersion.unknown, deployments.stream().findFirst(), "Test 3", Instant.ofEpochMilli(6)) .withCompletion(11, empty(), Instant.ofEpochMilli(7))); - DeploymentJobs deploymentJobs = new DeploymentJobs(projectId, statusList, empty()); + DeploymentJobs deploymentJobs = new DeploymentJobs(projectId, statusList, empty(), true); Application original = new Application(ApplicationId.from("t1", "a1", "i1"), deploymentSpec, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java new file mode 100644 index 00000000000..12640a5e8fa --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -0,0 +1,90 @@ +package com.yahoo.vespa.hosted.controller.persistence; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import com.yahoo.vespa.hosted.controller.deployment.Step; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.util.Collections; + +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.report; +import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; +import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class RunSerializerTest { + + private static final RunSerializer serializer = new RunSerializer(); + private static final Path runFile = Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json"); + private static final RunId id = new RunId(ApplicationId.from("tenant", "application", "default"), + JobType.productionUsEast3, + (long) 112358); + private static final Instant start = Instant.parse("2007-12-03T10:15:30.00Z"); + + @Test + public void testSerialization() throws IOException { + for (Step step : Step.values()) + assertEquals(step, RunSerializer.stepOf(RunSerializer.valueOf(step))); + + for (Step.Status status : Step.Status.values()) + assertEquals(status, RunSerializer.statusOf(RunSerializer.valueOf(status))); + + // The purpose of this serialised data is to ensure a new format does not break everything, so keep it up to date! + RunStatus run = serializer.runsFromSlime(SlimeUtils.jsonToSlime(Files.readAllBytes(runFile))).get(id); + for (Step step : Step.values()) + assertTrue(run.steps().containsKey(step)); + + assertEquals(id, run.id()); + assertEquals(start, run.start()); + assertFalse(run.hasEnded()); + assertFalse(run.isAborted()); + assertEquals(ImmutableMap.<Step, Step.Status>builder() + .put(deployInitialReal, unfinished) + .put(installInitialReal, failed) + .put(deployReal, succeeded) + .put(installReal, unfinished) + .put(deactivateReal, failed) + .put(deployTester, succeeded) + .put(installTester, unfinished) + .put(deactivateTester, failed) + .put(startTests, succeeded) + .put(endTests, unfinished) + .put(report, failed) + .build(), + run.steps()); + + run = run.aborted().finished(Instant.now()); + assertTrue(run.isAborted()); + assertTrue(run.hasEnded()); + + RunStatus phoenix = serializer.runsFromSlime(serializer.toSlime(Collections.singleton(run))).get(id); + assertEquals(run.id(), phoenix.id()); + assertEquals(run.start(), phoenix.start()); + assertEquals(run.end(), phoenix.end()); + assertEquals(run.isAborted(), phoenix.isAborted()); + assertEquals(run.steps(), phoenix.steps()); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json new file mode 100644 index 00000000000..d659bd9fff0 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json @@ -0,0 +1,21 @@ +[ + { + "id": "tenant:application:default", + "type": "production-us-east-3", + "number": 112358, + "start": 1196676930000, + "steps": { + "DIR": "U", + "IIR": "F", + "DR": "S", + "IR": "U", + "DAR": "F", + "DT": "S", + "IT": "U", + "DAT": "F", + "ST": "S", + "ET": "U", + "R": "F" + } + } +]
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index 280ce582029..de645cff96c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -81,6 +81,7 @@ public class ControllerContainerTest { " <component id='com.yahoo.vespa.hosted.controller.maintenance.JobControl'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.integration.RoutingGeneratorMock'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.integration.ArtifactRepositoryMock'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesters'/>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiHandler'>\n" + " <binding>http://*/application/v4/*</binding>\n" + " </handler>\n" + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 0ad82df8db1..e3d060ee806 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -22,6 +22,9 @@ "name": "DnsMaintainer" }, { + "name": "JobRunner" + }, + { "name": "MetricsReporter" }, { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index 09216eec3c7..291e6899a7a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -170,11 +170,6 @@ public class VersionStatusTest { // Application without deployment Application ignored0 = tester.createApplication("ignored0", "tenant1", 1000, 1000L); - // Pull request builds - tester.controllerTester().createApplication(TenantName.from("tenant1"), - "ignored1", - "43", 1000); - assertEquals("All applications running on this version: High", Confidence.high, confidence(tester.controller(), version0)); |