Re-add files that were lost by rename + reset

3 files changed, 427 insertions, 0 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InternalStepRunner.java
new file mode 100644
index 00000000000..41d719f37ed
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InternalStepRunner.java
@@ -0,0 +1,180 @@
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.ActivateResult;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.ApplicationVersion;
+import com.yahoo.vespa.hosted.controller.deployment.LockedStep;
+import com.yahoo.vespa.hosted.controller.deployment.Step.Status;
+
+import java.util.Optional;
+import java.util.function.Supplier;
+
+import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.ACTIVATION_CONFLICT;
+import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.APPLICATION_LOCK_FAILURE;
+import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.OUT_OF_CAPACITY;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished;
+
+/**
+ * Runs steps of a deployment job against its provided controller.
+ *
+ * @author jonmv
+ */
+public class InternalStepRunner implements StepRunner {
+
+    private static ApplicationId testerOf(ApplicationId id) {
+        return ApplicationId.from(id.tenant().value(),
+                                  id.application().value(),
+                                  id.instance().value() + "-t");
+    }
+
+    private final Controller controller;
+
+    public InternalStepRunner(Controller controller) {
+        this.controller = controller;
+    }
+
+    @Override
+    public Status run(LockedStep step, RunId id) {
+        switch (step.get()) {
+            case deployInitialReal: return deployInitialReal(id);
+            case installInitialReal: return installInitialReal(id);
+            case deployReal: return deployReal(id);
+            case deployTester: return deployTester(id);
+            case installReal: return installReal(id);
+            case installTester: return installTester(id);
+            case startTests: return startTests(id);
+            case runTests: return storeData(id);
+            case deactivateReal: return deactivateReal(id);
+            case deactivateTester: return deactivateTester(id);
+            case report: return report(id);
+            default: throw new AssertionError("Unknown step '" + step + "'!");
+        }
+    }
+
+    private Status deployInitialReal(RunId id) {
+        return deployReal(id, true);
+    }
+
+    private Status deployReal(RunId id) {
+        // Separate out deploy logic from above, and reuse.
+        return deployReal(id, false);
+    }
+
+    private Status deployReal(RunId id, boolean setTheStage) {
+        return deploy(id.application(),
+                      id.type(),
+                      () -> controller.applications().deploy(id.application(),
+                                                             id.type().zone(controller.system()).get(),
+                                                             Optional.empty(),
+                                                             new DeployOptions(false,
+                                                                               Optional.empty(),
+                                                                               false,
+                                                                               setTheStage)));
+    }
+
+    private Status deployTester(RunId id) {
+        // Find endpoints of real application. This will move down at a later time.
+        // See above.
+        return deploy(testerOf(id.application()),
+                      id.type(),
+                      () -> controller.applications().deployTester(testerOf(id.application()),
+                                                                   testerPackage(id),
+                                                                   id.type().zone(controller.system()).get(),
+                                                                   new DeployOptions(true,
+                                                                                     Optional.of(controller.systemVersion()),
+                                                                                     false,
+                                                                                     false)));
+    }
+
+    private Status deploy(ApplicationId id, JobType type, Supplier<ActivateResult> deploy) {
+        try {
+            // TODO jvenstad: Do whatever is required based on the result, and log all of this.
+            ActivateResult result = deploy.get();
+
+            return succeeded;
+        }
+        catch (ConfigServerException e) {
+            // TODO jvenstad: Consider retrying different things as well.
+            // TODO jvenstad: Log error information.
+            if (   e.getErrorCode() == OUT_OF_CAPACITY && type.isTest()
+                || e.getErrorCode() == ACTIVATION_CONFLICT
+                || e.getErrorCode() == APPLICATION_LOCK_FAILURE) {
+
+                return unfinished;
+            }
+        }
+        return failed;
+    }
+
+    private Status installInitialReal(RunId id) {
+        return install(id.application(), id.type());
+    }
+
+    private Status installReal(RunId id) {
+        return install(id.application(), id.type());
+    }
+
+    private Status installTester(RunId id) {
+        return install(testerOf(id.application()), id.type());
+    }
+
+    private Status install(ApplicationId id, JobType type) {
+        // If converged and serviceconverged: succeeded
+        // If timeout, failed
+        return unfinished;
+    }
+
+    private Status startTests(RunId id) {
+        // Empty for now, but will be: find endpoints and post them.
+        throw new AssertionError();
+    }
+
+    private Status storeData(RunId id) {
+        // Update test logs.
+        // If tests are done, return test results.
+        throw new AssertionError();
+    }
+
+    private Status deactivateReal(RunId id) {
+        return deactivate(id.application(), id.type());
+    }
+
+    private Status deactivateTester(RunId id) {
+        return deactivate(testerOf(id.application()), id.type());
+    }
+
+    private Status deactivate(ApplicationId id, JobType type) {
+        // Try to deactivate, and if deactivated, finished.
+        throw new AssertionError();
+    }
+
+    private Status report(RunId id) {
+        // Easy squeezy.
+        throw new AssertionError();
+    }
+
+    private Application application(ApplicationId id) {
+        return controller.applications().require(id);
+    }
+
+    private ApplicationPackage testerPackage(RunId id) {
+        ApplicationVersion version = application(id.application()).deploymentJobs()
+                                                                  .statusOf(id.type()).get()
+                                                                  .lastTriggered().get()
+                                                                  .application();
+
+
+        // TODO hakonhall: Fetch!
+        throw new AssertionError();
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
new file mode 100644
index 00000000000..c7d782c8302
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
@@ -0,0 +1,157 @@
+package com.yahoo.vespa.hosted.controller.persistence;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.slime.ArrayTraverser;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.ObjectTraverser;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
+import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
+import com.yahoo.vespa.hosted.controller.deployment.Step;
+import com.yahoo.vespa.hosted.controller.deployment.Step.Status;
+
+import java.time.Instant;
+import java.util.EnumMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Optional;
+
+import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.report;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.runTests;
+
+/**
+ * Serialises and deserialises RunStatus objects for persistent storage.
+ *
+ * @author jonmv
+ */
+public class RunSerializer {
+
+    private static final String stepsField = "steps";
+    private static final String applicationField = "id";
+    private static final String jobTypeField = "type";
+    private static final String numberField = "number";
+    private static final String startField = "start";
+    private static final String endField = "end";
+    private static final String abortedField = "aborted";
+
+    RunStatus runFromSlime(Slime slime) {
+        return runFromSlime(slime.get());
+    }
+
+    Map<RunId, RunStatus> runsFromSlime(Slime slime) {
+        Map<RunId, RunStatus> runs = new LinkedHashMap<>();
+        Inspector runArray = slime.get();
+        runArray.traverse((ArrayTraverser) (__, runObject) -> {
+            RunStatus run = runFromSlime(runObject);
+            runs.put(run.id(), run);
+        });
+        return runs;
+    }
+
+    private RunStatus runFromSlime(Inspector runObject) {
+        EnumMap<Step, Status> steps = new EnumMap<>(Step.class);
+        runObject.field(stepsField).traverse((ObjectTraverser) (step, status) -> {
+            steps.put(stepOf(step), statusOf(status.asString()));
+        });
+        return new RunStatus(new RunId(ApplicationId.fromSerializedForm(runObject.field(applicationField).asString()),
+                                       JobType.fromJobName(runObject.field(jobTypeField).asString()),
+                                       runObject.field(numberField).asLong()),
+                             steps,
+                             Instant.ofEpochMilli(runObject.field(startField).asLong()),
+                             Optional.of(runObject.field(endField))
+                                     .filter(Inspector::valid)
+                                     .map(end -> Instant.ofEpochMilli(end.asLong())),
+                             runObject.field(abortedField).asBool());
+    }
+
+    Slime toSlime(Iterable<RunStatus> runs) {
+        Slime slime = new Slime();
+        Cursor runArray = slime.setArray();
+        runs.forEach(run -> toSlime(run, runArray.addObject()));
+        return slime;
+    }
+
+    Slime toSlime(RunStatus run) {
+        Slime slime = new Slime();
+        toSlime(run, slime.setObject());
+        return slime;
+    }
+
+    private void toSlime(RunStatus run, Cursor runObject) {
+        runObject.setString(applicationField, run.id().application().serializedForm());
+        runObject.setString(jobTypeField, run.id().type().jobName());
+        runObject.setLong(numberField, run.id().number());
+        runObject.setLong(startField, run.start().toEpochMilli());
+        run.end().ifPresent(end -> runObject.setLong(endField, end.toEpochMilli()));
+        if (run.isAborted()) runObject.setBool(abortedField, true);
+        Cursor stepsObject = runObject.setObject(stepsField);
+        run.steps().forEach((step, status) -> stepsObject.setString(valueOf(step), valueOf(status)));
+    }
+
+    static String valueOf(Step step) {
+        switch (step) {
+            case deployInitialReal  : return "DIR";
+            case installInitialReal : return "IIR";
+            case deployReal         : return "DR" ;
+            case installReal        : return "IR" ;
+            case deactivateReal     : return "DAR";
+            case deployTester       : return "DT" ;
+            case installTester      : return "IT" ;
+            case deactivateTester   : return "DAT";
+            case startTests         : return "ST" ;
+            case runTests           : return "RT" ;
+            case report             : return "R"  ;
+            default                 : throw new AssertionError("No value defined for '" + step + "'!");
+        }
+    }
+
+    static Step stepOf(String step) {
+        switch (step) {
+            case "DIR" : return deployInitialReal ;
+            case "IIR" : return installInitialReal;
+            case "DR"  : return deployReal        ;
+            case "IR"  : return installReal       ;
+            case "DAR" : return deactivateReal    ;
+            case "DT"  : return deployTester      ;
+            case "IT"  : return installTester     ;
+            case "DAT" : return deactivateTester  ;
+            case "ST"  : return startTests        ;
+            case "RT"  : return runTests;
+            case "R"   : return report            ;
+            default    : throw new IllegalArgumentException("No step defined by '" + step + "'!");
+        }
+    }
+
+    static String valueOf(Status status) {
+        switch (status) {
+            case unfinished : return "U";
+            case failed     : return "F";
+            case succeeded  : return "S";
+            default  : throw new AssertionError("No value defined for '" + status + "'!");
+        }
+    }
+
+    static Status statusOf(String status) {
+        switch (status) {
+            case "U" : return unfinished;
+            case "F" : return failed    ;
+            case "S" : return succeeded ;
+            default  : throw new IllegalArgumentException("No status defined by '" + status + "'!");
+        }
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
new file mode 100644
index 00000000000..01acc401a1d
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
@@ -0,0 +1,90 @@
+package com.yahoo.vespa.hosted.controller.persistence;
+
+import com.google.common.collect.ImmutableMap;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
+import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
+import com.yahoo.vespa.hosted.controller.deployment.Step;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Instant;
+import java.util.Collections;
+
+import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.report;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.runTests;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class RunSerializerTest {
+
+    private static final RunSerializer serializer = new RunSerializer();
+    private static final Path runFile = Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json");
+    private static final RunId id = new RunId(ApplicationId.from("tenant", "application", "default"),
+                                               JobType.productionUsEast3,
+                                               (long) 112358);
+    private static final Instant start = Instant.parse("2007-12-03T10:15:30.00Z");
+
+    @Test
+    public void testSerialization() throws IOException {
+        for (Step step : Step.values())
+            assertEquals(step, RunSerializer.stepOf(RunSerializer.valueOf(step)));
+
+        for (Step.Status status : Step.Status.values())
+            assertEquals(status, RunSerializer.statusOf(RunSerializer.valueOf(status)));
+
+        // The purpose of this serialised data is to ensure a new format does not break everything, so keep it up to date!
+        RunStatus run = serializer.runsFromSlime(SlimeUtils.jsonToSlime(Files.readAllBytes(runFile))).get(id);
+        for (Step step : Step.values())
+            assertTrue(run.steps().containsKey(step));
+
+        assertEquals(id, run.id());
+        assertEquals(start, run.start());
+        assertFalse(run.hasEnded());
+        assertFalse(run.isAborted());
+        assertEquals(ImmutableMap.<Step, Step.Status>builder()
+                    .put(deployInitialReal, unfinished)
+                    .put(installInitialReal, failed)
+                    .put(deployReal, succeeded)
+                    .put(installReal, unfinished)
+                    .put(deactivateReal, failed)
+                    .put(deployTester, succeeded)
+                    .put(installTester, unfinished)
+                    .put(deactivateTester, failed)
+                    .put(startTests, succeeded)
+                    .put(runTests, unfinished)
+                    .put(report, failed)
+                    .build(),
+                     run.steps());
+
+        run = run.aborted().finished(Instant.now());
+        assertTrue(run.isAborted());
+        assertTrue(run.hasEnded());
+
+        RunStatus phoenix = serializer.runsFromSlime(serializer.toSlime(Collections.singleton(run))).get(id);
+        assertEquals(run.id(), phoenix.id());
+        assertEquals(run.start(), phoenix.start());
+        assertEquals(run.end(), phoenix.end());
+        assertEquals(run.isAborted(), phoenix.isAborted());
+        assertEquals(run.steps(), phoenix.steps());
+    }
+
+}