From 48417d7a59599c7c8149a19421a35f1d74ca8290 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Wed, 27 Jun 2018 16:45:16 +0200 Subject: Persistent (de)serialisation, and some minors --- .../controller/deployment/JobController.java | 9 ++- .../vespa/hosted/controller/deployment/Step.java | 5 +- .../controller/maintenance/RealStepRunner.java | 65 ++++++++++++++---- .../controller/persistence/JobSerializer.java | 77 ++++++++++++++++++++-- 4 files changed, 134 insertions(+), 22 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 40d20e1e100..466a658b49c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -47,6 +47,10 @@ public class JobController { this.logs = logStore; } + public LogStore logs() { + return logs; + } + // TODO jvenstad: Remove this, and let the DeploymentTrigger trigger directly with the correct BuildService. /** Returns whether the given application has registered with this build service. */ public boolean builds(ApplicationId id) { @@ -113,11 +117,6 @@ public class JobController { }); } - /** Returns the details for the given job. */ - public RunDetails details(RunId id) { - return new RunDetails(logs.getPrepareResponse(id), logs.getConvergenceLog(id), logs.getTestLog(id)); - } - /** Registers the given application, such that it may have deployment jobs run here. */ void register(ApplicationId id) { controller.applications().lockIfPresent(id, application -> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index 240f2ada242..a1d07001331 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -51,7 +51,10 @@ public enum Step { deactivateReal(deployInitialReal, deployReal, startTests), /** Deactivate the tester. */ - deactivateTester(deployTester, storeData); + deactivateTester(deployTester, storeData), + + /** Report completion to deployment orchestration machinery. */ + report; private final List prerequisites; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java index da80df59655..9f539e854e8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RealStepRunner.java @@ -1,20 +1,37 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.ApplicationController; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.deployment.LockedStep; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; + +/** + * Runs steps of a deployment job against its provided controller. + * + * @author jonmv + */ public class RealStepRunner implements StepRunner { - private static final String prefix = "-test-"; + private static ApplicationId testerOf(ApplicationId id) { + return ApplicationId.from(id.tenant().value(), + id.application().value(), + "-test-" + id.instance().value()); + } - private final ApplicationController applications; + private final Controller controller; - public RealStepRunner(ApplicationController applications) { - this.applications = applications; + public RealStepRunner(Controller controller) { + this.controller = controller; } @Override @@ -31,54 +48,80 @@ public class RealStepRunner implements StepRunner { case storeData: return storeData(id); case deactivateReal: return deactivateReal(id); case deactivateTester: return deactivateTester(id); + case report: return report(id); default: throw new AssertionError("Unknown step '" + step + "'!"); } } private Step.Status deployInitialReal(RunId id) { - throw new AssertionError(); + try { + // TODO jvenstad: Do whatever is required based on the result, and log all of this. + controller.applications().deploy(id.application(), + id.type().zone(controller.system()).get(), + Optional.empty(), + new DeployOptions(false, Optional.empty(), false, true)); + return succeeded; + } + catch (ConfigServerException e) { + // TODO jvenstad: Consider retrying different things as well. + // TODO jvenstad: Log error information. + if (id.type().isTest() && e.getErrorCode() == ConfigServerException.ErrorCode.OUT_OF_CAPACITY) + return unfinished; + } + return failed; } private Step.Status installInitialReal(RunId id) { - throw new AssertionError(); + // If converged and serviceconverged: succeeded + // If timeout, failed + return unfinished; } private Step.Status deployReal(RunId id) { + // Separate out deploy logic from above, and reuse. throw new AssertionError(); } private Step.Status deployTester(RunId id) { + // Find endpoints of real application. This will move down at a later time. + // See above. throw new AssertionError(); } private Step.Status installReal(RunId id) { + // See three above. throw new AssertionError(); } private Step.Status installTester(RunId id) { + // See above. throw new AssertionError(); } private Step.Status startTests(RunId id) { + // Empty for now, but will be: find endpoints and post them. throw new AssertionError(); } private Step.Status storeData(RunId id) { + // Update test logs. + // If tests are done, return test results. throw new AssertionError(); } private Step.Status deactivateReal(RunId id) { + // Try to deactivate, and if deactivated, finished. throw new AssertionError(); } private Step.Status deactivateTester(RunId id) { + // See above. throw new AssertionError(); } - private static ApplicationId testerOf(ApplicationId id) { - return ApplicationId.from(id.tenant().value(), - id.application().value(), - prefix + id.instance().value()); + private Step.Status report(RunId id) { + // Easy squeezy. + throw new AssertionError(); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java index 6813ed2969d..ba66bb67636 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobSerializer.java @@ -10,15 +10,30 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; +import com.yahoo.vespa.hosted.controller.deployment.Step.Status; import java.time.Instant; -import java.util.ArrayList; import java.util.EnumMap; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.Optional; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.report; +import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; +import static com.yahoo.vespa.hosted.controller.deployment.Step.storeData; +import static java.util.Objects.requireNonNull; + public class JobSerializer { private static final String stepsField = "steps"; @@ -44,9 +59,9 @@ public class JobSerializer { } private RunStatus runFromSlime(Inspector runObject) { - EnumMap steps = new EnumMap<>(Step.class); + EnumMap steps = new EnumMap<>(Step.class); runObject.field(stepsField).traverse((ObjectTraverser) (step, status) -> { - steps.put(Step.valueOf(step), Step.Status.valueOf(status.asString())); + steps.put(stepOf(step), statusOf(status.asString())); }); return new RunStatus(new RunId(ApplicationId.fromSerializedForm(runObject.field(applicationField).asString()), JobType.fromJobName(runObject.field(jobTypeField).asString()), @@ -78,7 +93,59 @@ public class JobSerializer { runObject.setLong(startField, run.start().toEpochMilli()); run.end().ifPresent(end -> runObject.setLong(endField, end.toEpochMilli())); Cursor stepsObject = runObject.setObject(stepsField); - run.steps().forEach((step, status) -> stepsObject.setString(step.name(), status.name())); + run.steps().forEach((step, status) -> stepsObject.setString(valueOf(step), valueOf(status))); + } + + private static String valueOf(Step step) { + switch (step) { + case deployInitialReal : return "DIR"; + case installInitialReal : return "IIR"; + case deployReal : return "DR" ; + case installReal : return "ID" ; + case deactivateReal : return "DAR"; + case deployTester : return "DT" ; + case installTester : return "IR" ; + case deactivateTester : return "DAT"; + case startTests : return "ST" ; + case storeData : return "SD" ; + case report : return "R" ; + default : throw new AssertionError("No value defined for '" + step + "'!"); + } + } + + private static Step stepOf(String step) { + switch (step) { + case "DIR" : return deployInitialReal ; + case "IIR" : return installInitialReal; + case "DR" : return deployReal ; + case "ID" : return installReal ; + case "DAR" : return deactivateReal ; + case "DT" : return deployTester ; + case "IR" : return installTester ; + case "DAT" : return deactivateTester ; + case "ST" : return startTests ; + case "SD" : return storeData ; + case "R" : return report ; + default : throw new IllegalArgumentException("No step defined by '" + step + "'!"); + } + } + + private static String valueOf(Status status) { + switch (status) { + case unfinished : return "U"; + case failed : return "F"; + case succeeded : return "S"; + default : throw new AssertionError("No value defined for '" + status + "'!"); + } + } + + private static Status statusOf(String status) { + switch (status) { + case "U" : return unfinished; + case "F" : return failed ; + case "S" : return succeeded ; + default : throw new IllegalArgumentException("No status defined by '" + status + "'!"); + } } } -- cgit v1.2.3