From fa2ae0775dea47fcbbc4c17e52fa7298d7ae16b7 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 7 Nov 2019 15:12:47 +0100 Subject: All applications are now deployed on internal pipeline --- .../yahoo/vespa/hosted/controller/Application.java | 10 +-- .../hosted/controller/ApplicationController.java | 35 +++------- .../vespa/hosted/controller/LockedApplication.java | 46 ++++++------- .../controller/deployment/DeploymentTrigger.java | 20 ++---- .../controller/deployment/JobController.java | 43 ++---------- .../persistence/ApplicationSerializer.java | 4 +- .../hosted/controller/persistence/CuratorDb.java | 9 ++- .../restapi/application/ApplicationApiHandler.java | 5 -- .../application/JobControllerApiHandlerHelper.java | 23 +++---- .../controller/deployment/DeploymentContext.java | 1 - .../controller/maintenance/JobRunnerTest.java | 2 +- .../persistence/ApplicationSerializerTest.java | 2 - .../restapi/application/ApplicationApiTest.java | 6 -- .../responses/instance-with-routing-policy.json | 1 - ...stance-without-change-multiple-deployments.json | 1 - .../restapi/application/responses/instance.json | 1 - .../application/responses/instance1-recursive.json | 1 - .../responses/jobs-direct-deployment.json | 76 +--------------------- 18 files changed, 58 insertions(+), 228 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 0a8b5ca8c3d..681c1b4283a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -45,7 +45,6 @@ public class Application { private final ValidationOverrides validationOverrides; private final Optional latestVersion; private final OptionalLong projectId; - private final boolean internal; private final Change change; private final Change outstandingChange; private final Optional deploymentIssueId; @@ -60,14 +59,14 @@ public class Application { public Application(TenantAndApplicationId id, Instant now) { this(id, now, DeploymentSpec.empty, ValidationOverrides.empty, Change.empty(), Change.empty(), Optional.empty(), Optional.empty(), Optional.empty(), OptionalInt.empty(), - new ApplicationMetrics(0, 0), Set.of(), OptionalLong.empty(), false, Optional.empty(), List.of()); + new ApplicationMetrics(0, 0), Set.of(), OptionalLong.empty(), Optional.empty(), List.of()); } // DO NOT USE! For serialization purposes, only. public Application(TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, Change change, Change outstandingChange, Optional deploymentIssueId, Optional ownershipIssueId, Optional owner, OptionalInt majorVersion, ApplicationMetrics metrics, Set deployKeys, OptionalLong projectId, - boolean internal, Optional latestVersion, Collection instances) { + Optional latestVersion, Collection instances) { this.id = Objects.requireNonNull(id, "id cannot be null"); this.createdAt = Objects.requireNonNull(createdAt, "instant of creation cannot be null"); this.deploymentSpec = Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null"); @@ -81,7 +80,6 @@ public class Application { this.metrics = Objects.requireNonNull(metrics, "metrics cannot be null"); this.deployKeys = Objects.requireNonNull(deployKeys, "deployKeys cannot be null"); this.projectId = Objects.requireNonNull(projectId, "projectId cannot be null"); - this.internal = internal; this.latestVersion = requireNotUnknown(latestVersion); this.instances = ImmutableSortedMap.copyOf(instances.stream().collect(Collectors.toMap(Instance::name, Function.identity()))); } @@ -102,10 +100,6 @@ public class Application { /** Returns the last submitted version of this application. */ public Optional latestVersion() { return latestVersion; } - /** Returns whether this application is run on the internal deployment pipeline. */ - // TODO jonmv: Remove, as will be always true. - public boolean internal() { return internal; } - /** * Returns the last deployed validation overrides of this application, * or the empty validation overrides if it has never been deployed diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 881e793b0b3..90c3fabf2d6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -207,6 +207,11 @@ public class ApplicationController { return curator.readApplications(); } + /** Returns the ID of all known applications. */ + public List idList() { + return curator.readApplicationIds(); + } + /** Returns a snapshot of all applications of a tenant */ public List asList(TenantName tenant) { return curator.readApplications(tenant); @@ -385,7 +390,7 @@ public class ApplicationController { applicationVersion = preferOldestVersion ? triggered.sourceApplication().orElse(triggered.application()) : triggered.application(); - applicationPackage = getApplicationPackage(instanceId, application.get().internal(), applicationVersion); + applicationPackage = getApplicationPackage(instanceId, applicationVersion); applicationPackage = withTesterCertificate(applicationPackage, instanceId, jobType); validateRun(application.get(), instance, zone, platformVersion, applicationVersion); } @@ -397,13 +402,6 @@ public class ApplicationController { applicationCertificate = Optional.empty(); } - // TODO jonmv: REMOVE! This is now irrelevant for non-CD-test deployments and non-unit tests. - if ( ! preferOldestVersion - && ! application.get().internal() - && ! zone.environment().isManuallyDeployed()) { - application = storeWithUpdatedConfig(application, applicationPackage); - } - endpoints = registerEndpointsInDns(applicationPackage.deploymentSpec(), application.get().require(instanceId.instance()), zone); } // Release application lock while doing the deployment, which is a lengthy task. @@ -434,25 +432,8 @@ public class ApplicationController { } /** Fetches the requested application package from the artifact store(s). */ - public ApplicationPackage getApplicationPackage(ApplicationId id, boolean internal, ApplicationVersion version) { - try { - return internal - ? new ApplicationPackage(applicationStore.get(id.tenant(), id.application(), version)) - : new ApplicationPackage(artifactRepository.getApplicationPackage(id, version.id())); - } - catch (RuntimeException e) { // If application has switched deployment pipeline, artifacts stored prior to the switch are in the other artifact store. - try { - log.info("Fetching application package for " + id + " from alternate repository; it is now deployed " - + (internal ? "internally" : "externally") + "\nException was: " + Exceptions.toMessageString(e)); - return internal - ? new ApplicationPackage(artifactRepository.getApplicationPackage(id, version.id())) - : new ApplicationPackage(applicationStore.get(id.tenant(), id.application(), version)); - } - catch (RuntimeException s) { // If this fails, too, the first failure is most likely the relevant one. - e.addSuppressed(s); - throw e; - } - } + public ApplicationPackage getApplicationPackage(ApplicationId id, ApplicationVersion version) { + return new ApplicationPackage(applicationStore.get(id.tenant(), id.application(), version)); } /** Stores the deployment spec and validation overrides from the application package, and runs cleanup. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index 46d1d436521..fa81a990c70 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -45,7 +45,6 @@ public class LockedApplication { private final ApplicationMetrics metrics; private final Set deployKeys; private final OptionalLong projectId; - private final boolean internal; private final Optional latestVersion; private final Map instances; @@ -60,14 +59,14 @@ public class LockedApplication { application.deploymentSpec(), application.validationOverrides(), application.change(), application.outstandingChange(), application.deploymentIssueId(), application.ownershipIssueId(), application.owner(), application.majorVersion(), application.metrics(), application.deployKeys(), - application.projectId(), application.internal(), application.latestVersion(), application.instances()); + application.projectId(), application.latestVersion(), application.instances()); } private LockedApplication(Lock lock, TenantAndApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, Change change, Change outstandingChange, Optional deploymentIssueId, Optional ownershipIssueId, Optional owner, OptionalInt majorVersion, ApplicationMetrics metrics, Set deployKeys, - OptionalLong projectId, boolean internal, Optional latestVersion, + OptionalLong projectId, Optional latestVersion, Map instances) { this.lock = lock; this.id = id; @@ -83,7 +82,6 @@ public class LockedApplication { this.metrics = metrics; this.deployKeys = deployKeys; this.projectId = projectId; - this.internal = internal; this.latestVersion = latestVersion; this.instances = Map.copyOf(instances); } @@ -92,7 +90,7 @@ public class LockedApplication { public Application get() { return new Application(id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, internal, latestVersion, instances.values()); + projectId, latestVersion, instances.values()); } public LockedApplication withNewInstance(InstanceName instance) { @@ -100,7 +98,7 @@ public class LockedApplication { instances.put(instance, new Instance(id.instance(instance))); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, internal, latestVersion, instances); + projectId, latestVersion, instances); } public LockedApplication with(InstanceName instance, UnaryOperator modification) { @@ -108,7 +106,7 @@ public class LockedApplication { instances.put(instance, modification.apply(instances.get(instance))); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, internal, latestVersion, instances); + projectId, latestVersion, instances); } public LockedApplication without(InstanceName instance) { @@ -116,67 +114,61 @@ public class LockedApplication { instances.remove(instance); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, internal, latestVersion, instances); + projectId, latestVersion, instances); } public LockedApplication withNewSubmission(ApplicationVersion latestVersion) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, internal, Optional.of(latestVersion), instances); - } - - public LockedApplication withBuiltInternally(boolean builtInternally) { - return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, - deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, builtInternally, latestVersion, instances); + projectId, Optional.of(latestVersion), instances); } public LockedApplication withProjectId(OptionalLong projectId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, internal, latestVersion, instances); + projectId, latestVersion, instances); } public LockedApplication withDeploymentIssueId(IssueId issueId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, Optional.ofNullable(issueId), ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, internal, latestVersion, instances); + projectId, latestVersion, instances); } public LockedApplication with(DeploymentSpec deploymentSpec) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, internal, latestVersion, instances); + projectId, latestVersion, instances); } public LockedApplication with(ValidationOverrides validationOverrides) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, internal, latestVersion, instances); + projectId, latestVersion, instances); } public LockedApplication withChange(Change change) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, internal, latestVersion, instances); + projectId, latestVersion, instances); } public LockedApplication withOutstandingChange(Change outstandingChange) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, internal, latestVersion, instances); + projectId, latestVersion, instances); } public LockedApplication withOwnershipIssueId(IssueId issueId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, deploymentIssueId, Optional.of(issueId), owner, majorVersion, metrics, deployKeys, - projectId, internal, latestVersion, instances); + projectId, latestVersion, instances); } public LockedApplication withOwner(User owner) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, deploymentIssueId, ownershipIssueId, Optional.of(owner), majorVersion, metrics, deployKeys, - projectId, internal, latestVersion, instances); + projectId, latestVersion, instances); } /** Set a major version for this, or set to null to remove any major version override */ @@ -184,13 +176,13 @@ public class LockedApplication { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, deploymentIssueId, ownershipIssueId, owner, majorVersion == null ? OptionalInt.empty() : OptionalInt.of(majorVersion), - metrics, deployKeys, projectId, internal, latestVersion, instances); + metrics, deployKeys, projectId, latestVersion, instances); } public LockedApplication with(ApplicationMetrics metrics) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, - projectId, internal, latestVersion, instances); + projectId, latestVersion, instances); } public LockedApplication withDeployKey(PublicKey pemDeployKey) { @@ -198,7 +190,7 @@ public class LockedApplication { keys.add(pemDeployKey); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, keys, - projectId, internal, latestVersion, instances); + projectId, latestVersion, instances); } public LockedApplication withoutDeployKey(PublicKey pemDeployKey) { @@ -206,7 +198,7 @@ public class LockedApplication { keys.remove(pemDeployKey); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, keys, - projectId, internal, latestVersion, instances); + projectId, latestVersion, instances); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 45bf8a43f07..8b38ed13b28 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -78,13 +78,11 @@ public class DeploymentTrigger { private final Controller controller; private final Clock clock; - private final BuildService buildService; private final JobController jobs; public DeploymentTrigger(Controller controller, BuildService buildService, Clock clock) { this.controller = Objects.requireNonNull(controller, "controller cannot be null"); this.clock = Objects.requireNonNull(clock, "clock cannot be null"); - this.buildService = Objects.requireNonNull(buildService, "buildService cannot be null"); this.jobs = controller.jobController(); } @@ -103,10 +101,9 @@ public class DeploymentTrigger { if (acceptNewApplicationVersion(application.get())) { application = application.withChange(application.get().change().with(version)) .withOutstandingChange(Change.empty()); - if (application.get().internal()) - for (Run run : jobs.active(id)) - if ( ! run.id().type().environment().isManuallyDeployed()) - jobs.abort(run.id()); + for (Run run : jobs.active(id)) + if ( ! run.id().type().environment().isManuallyDeployed()) + jobs.abort(run.id()); } else application = application.withOutstandingChange(Change.of(version)); @@ -197,13 +194,10 @@ public class DeploymentTrigger { log.log(LogLevel.DEBUG, String.format("Triggering %s: %s", job, job.triggering)); try { applications().lockApplicationOrThrow(TenantAndApplicationId.from(job.applicationId()), application -> { - if (application.get().internal()) - jobs.start(job.applicationId(), job.jobType, new Versions(job.triggering.platform(), - job.triggering.application(), - job.triggering.sourcePlatform(), - job.triggering.sourceApplication())); - else - buildService.trigger(job); + jobs.start(job.applicationId(), job.jobType, new Versions(job.triggering.platform(), + job.triggering.application(), + job.triggering.sourcePlatform(), + job.triggering.sourceApplication())); applications().store(application.with(job.applicationId().instance(), instance -> instance.withJobTriggering(job.jobType, job.triggering))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 84f14109c4b..e22b70050ab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -59,6 +59,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.copyVespaLogs; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toUnmodifiableList; /** * A singleton owned by the controller, which contains the state and methods for controlling deployment jobs. @@ -198,7 +199,6 @@ public class JobController { /** Returns a list of all applications which have registered. */ public List applications() { return copyOf(controller.applications().asList().stream() - .filter(Application::internal) .map(Application::id) .iterator()); } @@ -206,7 +206,6 @@ public class JobController { /** Returns a list of all instances of applications which have registered. */ public List instances() { return copyOf(controller.applications().asList().stream() - .filter(Application::internal) .flatMap(application -> application.instances().values().stream()) .map(Instance::id) .iterator()); @@ -272,9 +271,9 @@ public class JobController { /** Returns a list of all active runs. */ public List active() { - return copyOf(applications().stream() - .flatMap(id -> active(id).stream()) - .iterator()); + return controller.applications().idList().stream() + .flatMap(id -> active(id).stream()) + .collect(toUnmodifiableList()); } /** Returns a list of all active runs for the given instance. */ @@ -347,9 +346,6 @@ public class JobController { ApplicationPackage applicationPackage, byte[] testPackageBytes) { AtomicReference version = new AtomicReference<>(); controller.applications().lockApplicationOrThrow(id, application -> { - if ( ! application.get().internal()) - application = registered(application); - long run = 1 + application.get().latestVersion() .map(latestVersion -> latestVersion.buildNumber().getAsLong()) .orElse(0L); @@ -377,31 +373,12 @@ public class JobController { return version.get(); } - /** Registers the given application, copying necessary application packages, and returns the modified version. */ - private LockedApplication registered(LockedApplication application) { - for (Instance instance : application.get().instances().values()) { - // TODO jvenstad: Remove when everyone has migrated off SDv3 pipelines. Real soon now! - // Copy all current packages to the new application store - instance.productionDeployments().values().stream() - .map(Deployment::applicationVersion) - .distinct() - .forEach(appVersion -> { - byte[] content = controller.applications().artifacts().getApplicationPackage(instance.id(), appVersion.id()); - controller.applications().applicationStore().put(instance.id().tenant(), instance.id().application(), appVersion, content); - }); - } - return application.withBuiltInternally(true); - } - /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ public void start(ApplicationId id, JobType type, Versions versions) { if ( ! type.environment().isManuallyDeployed() && versions.targetApplication().isUnknown()) throw new IllegalArgumentException("Target application must be a valid reference."); controller.applications().lockApplicationIfPresent(TenantAndApplicationId.from(id), application -> { - if ( ! application.get().internal()) - throw new IllegalArgumentException(id + " is not built here!"); - locked(id, type, __ -> { Optional last = last(id, type); if (last.flatMap(run -> active(run.id())).isPresent()) @@ -423,9 +400,6 @@ public class JobController { controller.applications().createApplication(TenantAndApplicationId.from(id), Optional.empty()); controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { - if ( ! application.get().internal()) - application = registered(application); - if ( ! application.get().instances().containsKey(id.instance())) application = application.withNewInstance(id.instance()); @@ -460,15 +434,6 @@ public class JobController { } } - /** Unregisters the given application and makes all associated data eligible for garbage collection. */ - public void unregister(TenantAndApplicationId id) { - controller.applications().lockApplicationIfPresent(id, application -> { - controller.applications().store(application.withBuiltInternally(false)); - for (InstanceName instance : application.get().instances().keySet()) - jobs(id.instance(instance)).forEach(type -> last(id.instance(instance), type).ifPresent(last -> abort(last.id()))); - }); - } - /** Deletes run data and tester deployments for applications which are unknown, or no longer built internally. */ public void collectGarbage() { Set applicationsToBuild = new HashSet<>(instances()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 75f489d3345..487413442b7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -179,7 +179,6 @@ public class ApplicationSerializer { application.projectId().ifPresent(projectId -> root.setLong(projectIdField, projectId)); application.deploymentIssueId().ifPresent(jiraIssueId -> root.setString(deploymentIssueField, jiraIssueId.value())); application.ownershipIssueId().ifPresent(issueId -> root.setString(ownershipIssueIdField, issueId.value())); - root.setBool(builtInternallyField, application.internal()); toSlime(application.change(), root, deployingField); toSlime(application.outstandingChange(), root, outstandingChangeField); application.owner().ifPresent(owner -> root.setString(ownerField, owner.username())); @@ -369,11 +368,10 @@ public class ApplicationSerializer { List instances = instancesFromSlime(id, deploymentSpec, root.field(instancesField)); OptionalLong projectId = Serializers.optionalLong(root.field(projectIdField)); Optional latestVersion = latestVersionFromSlime(root.field(latestVersionField)); - boolean builtInternally = root.field(builtInternallyField).asBool(); return new Application(id, createdAt, deploymentSpec, validationOverrides, deploying, outstandingChange, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, - deployKeys, projectId, builtInternally, latestVersion, instances); + deployKeys, projectId, latestVersion, instances); } private Optional latestVersionFromSlime(Inspector latestVersionObject) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 037fb4da987..637da842d2f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -56,6 +56,7 @@ import java.util.stream.Stream; import static java.util.Comparator.comparing; import static java.util.stream.Collectors.collectingAndThen; +import static java.util.stream.Collectors.toUnmodifiableList; /** * Curator backed database for storing the persistence state of controllers. This maps controller specific operations @@ -352,16 +353,18 @@ public class CuratorDb { } private List readApplications(Predicate applicationFilter) { - return readApplicationIds().filter(applicationFilter) + return readApplicationIds().stream() + .filter(applicationFilter) .sorted() .map(this::readApplication) .flatMap(Optional::stream) .collect(Collectors.toUnmodifiableList()); } - private Stream readApplicationIds() { + public List readApplicationIds() { return curator.getChildren(applicationRoot).stream() - .map(TenantAndApplicationId::fromSerialized); + .map(TenantAndApplicationId::fromSerialized) + .collect(toUnmodifiableList()); } public void removeApplication(TenantAndApplicationId id) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 5cb7d7a6065..c8f5720327a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -290,11 +290,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), "all"); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("choice")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/key")) return removeDeployKey(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return JobControllerApiHandlerHelper.unregisterResponse(controller.jobController(), path.get("tenant"), path.get("application")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return deleteInstance(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), "all"); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("choice")); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/submit")) return JobControllerApiHandlerHelper.unregisterResponse(controller.jobController(), path.get("tenant"), path.get("application")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.abortJobResponse(controller.jobController(), appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deactivate(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true, request); @@ -833,7 +831,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { .steps(application.deploymentSpec().requireInstance(instance.name())) .sortedJobs(instance.deploymentJobs().jobStatus().values()); - object.setBool("deployedInternally", application.internal()); Cursor deploymentsArray = object.setArray("deploymentJobs"); for (JobStatus job : jobStatus) { Cursor jobObject = deploymentsArray.addObject(); @@ -1479,7 +1476,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { applicationVersion = Optional.of(ApplicationVersion.from(toSourceRevision(sourceRevision), buildNumber.asLong())); applicationPackage = Optional.of(controller.applications().getApplicationPackage(applicationId, - application.get().internal(), applicationVersion.get())); } @@ -1503,7 +1499,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { applicationVersion = Optional.of(version); vespaVersion = Optional.of(deployment.get().version()); applicationPackage = Optional.of(controller.applications().getApplicationPackage(applicationId, - application.get().internal(), applicationVersion.get())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index cb4c1e71c43..899eb546bca 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -41,10 +41,12 @@ import java.util.ArrayDeque; import java.util.Comparator; import java.util.Deque; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy.conservative; @@ -128,7 +130,7 @@ class JobControllerApiHandlerHelper { pendingProduction, running, type, - Optional.ofNullable(status.get(type)), + status.get(type), deployment); }); }); @@ -220,15 +222,15 @@ class JobControllerApiHandlerHelper { private static void deploymentToSlime(Cursor deploymentObject, Instance instance, Change change, Map pendingProduction, Map running, - JobType type, Optional jobStatus, Deployment deployment) { + JobType type, JobStatus jobStatus, Deployment deployment) { deploymentObject.setLong("at", deployment.at().toEpochMilli()); deploymentObject.setString("platform", deployment.version().toString()); applicationVersionToSlime(deploymentObject.setObject("application"), deployment.applicationVersion()); - deploymentObject.setBool("verified", jobStatus.flatMap(job -> job.lastSuccess()) + deploymentObject.setBool("verified", jobStatus.lastSuccess() .map(Run::versions) - .filter(run -> run.targetPlatform().equals(deployment.version()) - && run.targetApplication().equals(deployment.applicationVersion())) - .isPresent()); + .filter(run -> run.targetPlatform().equals(deployment.version()) + && run.targetApplication().equals(deployment.applicationVersion())) + .isPresent()); if (running.containsKey(type)) deploymentObject.setString("status", running.get(type).steps().get(deployReal) == unfinished ? "deploying" : "verifying"); else if (change.hasTargets()) @@ -467,15 +469,6 @@ class JobControllerApiHandlerHelper { return new SlimeJsonResponse(slime); } - /** Unregisters the application from the internal deployment pipeline. */ - static HttpResponse unregisterResponse(JobController jobs, String tenantName, String applicationName) { - TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName); - jobs.unregister(id); - Slime slime = new Slime(); - slime.setObject().setString("message", "Unregistered '" + id + "' from internal deployment pipeline."); - return new SlimeJsonResponse(slime); - } - private static String nameOf(RunStatus status) { switch (status) { case running: return "running"; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index d0eb86c3ba1..d181ab8d38f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -357,7 +357,6 @@ public class DeploymentContext { /** Deploy default application package, start a run for that change and return its ID */ public RunId newRun(JobType type) { - assertFalse(application().internal()); // Use this only once per test. submit(); readyJobsTrigger.maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index d3752fcaff4..f7d451ab931 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -190,7 +190,7 @@ public class JobRunnerTest { // Start a third run, then unregister and wait for data to be deleted. jobs.start(id, systemTest, versions); - jobs.unregister(appId); + tester.applications().deleteInstance(id); runner.maintain(); assertFalse(jobs.last(id, systemTest).isPresent()); assertTrue(jobs.runs(id, systemTest).isEmpty()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 9ac73604da5..7536272d6d9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -151,7 +151,6 @@ public class ApplicationSerializerTest { new ApplicationMetrics(0.5, 0.9), Set.of(publicKey, otherPublicKey), projectId, - true, Optional.of(applicationVersion1), instances); @@ -165,7 +164,6 @@ public class ApplicationSerializerTest { assertEquals(original.validationOverrides().xmlForm(), serialized.validationOverrides().xmlForm()); assertEquals(original.projectId(), serialized.projectId()); - assertEquals(original.internal(), serialized.internal()); assertEquals(original.deploymentIssueId(), serialized.deploymentIssueId()); assertEquals(0, serialized.require(id3.instance()).deployments().size()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 5deedaa2fb1..b5b7c1d0ad0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -696,12 +696,6 @@ public class ApplicationApiTest extends ControllerContainerTest { .userIdentity(USER_ID), "{\"message\":\"Aborting run 2 of staging-test for tenant1.application1.instance1\"}"); - // TODO jonmv: Remove, as this is the only option now. - // DELETE submission to unsubscribe from continuous deployment. - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/submit", DELETE) - .userIdentity(HOSTED_VESPA_OPERATOR), - "{\"message\":\"Unregistered 'tenant1.application1' from internal deployment pipeline.\"}"); - // PUT (create) the authenticated user byte[] data = new byte[0]; tester.assertResponse(request("/application/v4/user?user=new_user&domain=by", PUT) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json index bb81125e54d..0f759eaed55 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json @@ -9,7 +9,6 @@ "gitCommit": "commit1" }, "projectId": 1000, - "deployedInternally": true, "deploymentJobs": [ { "type": "system-test", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-without-change-multiple-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-without-change-multiple-deployments.json index 75493a30155..e8adfd579d5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-without-change-multiple-deployments.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-without-change-multiple-deployments.json @@ -9,7 +9,6 @@ "gitCommit": "commit1" }, "projectId": 1000, - "deployedInternally": true, "deploymentJobs": [ { "type": "system-test", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json index 061931336ed..3664af7116c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json @@ -20,7 +20,6 @@ } } }, - "deployedInternally": true, "deploymentJobs": [ { "type": "system-test", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json index 2dacacedf51..ddce3921518 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json @@ -20,7 +20,6 @@ } } }, - "deployedInternally": true, "deploymentJobs": [ { "type": "system-test", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json index 5535e286dcd..85a3245c308 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json @@ -1,79 +1,7 @@ { "devJobs": {}, - "deployments": [ - { - "us-west-1": { - "at": 0, - "application": { - "hash": "unknown" - }, - "verified": false, - "platform": "6.1" - } - } - ], + "deployments": [], "lastVersions": {}, "deploying": {}, - "jobs": { - "staging-test": { - "runs": [ - { - "reason": "Testing for productionUsWest1", - "wantedPlatform": "6.1", - "currentPlatform": "6.1", - "wantedApplication": { - "hash": "unknown" - }, - "currentApplication": { - "hash": "unknown" - }, - "tasks": { - "capacity": "running" - }, - "status": "pending" - } - ], - "url": "https://some.url:43/root/staging-test" - }, - "system-test": { - "runs": [ - { - "reason": "Testing for productionUsWest1", - "wantedPlatform": "6.1", - "currentPlatform": "6.1", - "wantedApplication": { - "hash": "unknown" - }, - "currentApplication": { - "hash": "unknown" - }, - "tasks": { - "capacity": "running" - }, - "status": "pending" - } - ], - "url": "https://some.url:43/root/system-test" - }, - "us-west-1": { - "runs": [ - { - "wantedPlatform": "6.1", - "currentPlatform": "6.1", - "wantedApplication": { - "hash": "unknown" - }, - "currentApplication": { - "hash": "unknown" - }, - "tasks": { - "staging-test": "pending", - "system-test": "pending" - }, - "status": "pending" - } - ], - "url": "https://some.url:43/root/production-us-west-1" - } - } + "jobs": {} } -- cgit v1.2.3