Merge pull request #21600 from vespa-engine/jonmv/deployment-playground-2v7.555.50

Jonmv/deployment playground 2

6 files changed, 356 insertions, 21 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
index 2b52143f574..189192d8d9a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
@@ -266,10 +266,12 @@ class JobControllerApiHandlerHelper {
             stepObject.setBool("declared", stepStatus.isDeclared());
             stepObject.setString("instance", stepStatus.instance().value());
 
-            stepStatus.readyAt(change).ifPresent(ready -> stepObject.setLong("readyAt", ready.toEpochMilli()));
-            stepStatus.readyAt(change)
-                      .filter(controller.clock().instant()::isBefore)
-                      .ifPresent(until -> stepObject.setLong("delayedUntil", until.toEpochMilli()));
+            // TODO: recursively search dependents for what is the relevant partial change when this is a delay step ...
+            Optional<Instant> readyAt = stepStatus.job().map(jobsToRun::get).map(jobs -> jobs.get(0).readyAt())
+                                                  .orElse(stepStatus.readyAt(change));
+            readyAt.ifPresent(ready -> stepObject.setLong("readyAt", ready.toEpochMilli()));
+            readyAt.filter(controller.clock().instant()::isBefore)
+                   .ifPresent(until -> stepObject.setLong("delayedUntil", until.toEpochMilli()));
             stepStatus.pausedUntil().ifPresent(until -> stepObject.setLong("pausedUntil", until.toEpochMilli()));
             stepStatus.coolingDownUntil(change).ifPresent(until -> stepObject.setLong("coolingDownUntil", until.toEpochMilli()));
             stepStatus.blockedUntil(Change.of(controller.systemVersion(versionStatus))) // Dummy version — just anything with a platform.
@@ -304,17 +306,17 @@ class JobControllerApiHandlerHelper {
                     for (VespaVersion available : availablePlatforms) {
                         if (   deployments.stream().anyMatch(deployment -> deployment.version().isAfter(available.versionNumber()))
                             || deployments.stream().noneMatch(deployment -> deployment.version().isBefore(available.versionNumber())) && ! deployments.isEmpty()
-                            || change.platform().map(available.versionNumber()::compareTo).orElse(1) < 0)
+                            || status.hasCompleted(stepStatus.instance(), Change.of(available.versionNumber()))
+                            || change.platform().map(available.versionNumber()::compareTo).orElse(1) <= 0)
                             break;
 
-                        Cursor availableObject = availableArray.addObject();
-                        availableObject.setString("platform", available.versionNumber().toFullString());
+                        availableArray.addObject().setString("platform", available.versionNumber().toFullString());
                     }
+                    change.platform().ifPresent(version -> availableArray.addObject().setString("platform", version.toFullString()));
                     toSlime(latestPlatformObject.setArray("blockers"), blockers.stream().filter(ChangeBlocker::blocksVersions));
                 }
-                List<ApplicationVersion> availableApplications = new ArrayList<>(application.versions());
+                List<ApplicationVersion> availableApplications = new ArrayList<>(application.deployableVersions(false));
                 if ( ! availableApplications.isEmpty()) {
-                    Collections.reverse(availableApplications);
                     var latestApplication = availableApplications.get(0);
                     Cursor latestApplicationObject = latestVersionsObject.setObject("application");
                     toSlime(latestApplicationObject.setObject("application"), latestApplication);
@@ -326,12 +328,13 @@ class JobControllerApiHandlerHelper {
                     for (ApplicationVersion available : availableApplications) {
                         if (   deployments.stream().anyMatch(deployment -> deployment.applicationVersion().compareTo(available) > 0)
                             || deployments.stream().noneMatch(deployment -> deployment.applicationVersion().compareTo(available) < 0) && ! deployments.isEmpty()
-                            || change.application().map(available::compareTo).orElse(1) < 0)
+                            || status.hasCompleted(stepStatus.instance(), Change.of(available))
+                            || change.application().map(available::compareTo).orElse(1) <= 0)
                             break;
 
-                        Cursor availableObject = availableArray.addObject();
-                        toSlime(availableObject.setObject("application"), available);
+                        toSlime(availableArray.addObject().setObject("application"), available);
                     }
+                    change.application().ifPresent(version -> toSlime(availableArray.addObject().setObject("application"), version));
                     toSlime(latestApplicationObject.setArray("blockers"), blockers.stream().filter(ChangeBlocker::blocksRevisions));
                 }
             }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
index 6b13e6c951a..87fe8dd17fe 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
@@ -35,10 +35,12 @@ public class ControllerContainerTest {
 
     @Before
     public void startContainer() {
-        container = JDisc.fromServicesXml(controllerServicesXml(), Networking.disable);
+        container = JDisc.fromServicesXml(controllerServicesXml(), networking());
         addUserToHostedOperatorRole(hostedOperator);
     }
 
+    protected Networking networking() { return Networking.disable; }
+
     @After
     public void stopContainer() { container.close(); }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json
index 8677b621f0d..c0a6829b026 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json
@@ -579,14 +579,6 @@
             },
             {
               "application": {
-                "build": 3,
-                "compileVersion": "6.1.0",
-                "sourceUrl": "repository1/tree/commit1",
-                "commit": "commit1"
-              }
-            },
-            {
-              "application": {
                 "build": 2,
                 "compileVersion": "6.1.0",
                 "sourceUrl": "repository1/tree/commit1",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/AllowingFilter.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/AllowingFilter.java
new file mode 100644
index 00000000000..de7f2515979
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/AllowingFilter.java
@@ -0,0 +1,41 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.playground;
+
+import com.yahoo.jdisc.http.filter.DiscFilterRequest;
+import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase;
+import com.yahoo.vespa.athenz.api.AthenzDomain;
+import com.yahoo.vespa.athenz.api.AthenzPrincipal;
+import com.yahoo.vespa.athenz.api.AthenzUser;
+import com.yahoo.vespa.hosted.controller.api.integration.user.User;
+import com.yahoo.vespa.hosted.controller.api.role.Role;
+import com.yahoo.vespa.hosted.controller.api.role.SecurityContext;
+import com.yahoo.yolean.Exceptions;
+
+import java.time.Instant;
+import java.util.Optional;
+import java.util.Set;
+
+public class AllowingFilter extends JsonSecurityRequestFilterBase {
+
+    static final AthenzPrincipal user = new AthenzPrincipal(new AthenzUser("demo"));
+    static final AthenzDomain domain = new AthenzDomain("domain");
+
+    @Override
+    protected Optional<ErrorResponse> filter(DiscFilterRequest request) {
+        try {
+            request.setUserPrincipal(user);
+            request.setAttribute(User.ATTRIBUTE_NAME, new User("mail@mail", user.getName(), "demo", null, true, -1, User.NO_DATE));
+            request.setAttribute("okta.identity-token", "okta-it");
+            request.setAttribute("okta.access-token", "okta-at");
+            request.setAttribute(SecurityContext.ATTRIBUTE_NAME,
+                                 new SecurityContext(user,
+                                                     Set.of(Role.hostedOperator()),
+                                                     Instant.now().minusSeconds(3600)));
+            return Optional.empty();
+        }
+        catch (Throwable t) {
+            return Optional.of(new ErrorResponse(500, Exceptions.toMessageString(t)));
+        }
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlayground.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlayground.java
new file mode 100644
index 00000000000..2c91aceb8b0
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlayground.java
@@ -0,0 +1,226 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.playground;
+
+import com.yahoo.application.Networking;
+import com.yahoo.component.Version;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.deployment.Run;
+import com.yahoo.vespa.hosted.controller.restapi.ContainerTester;
+import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.time.Duration;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.BiConsumer;
+
+public class DeploymentPlayground extends ControllerContainerTest {
+
+    private final Object monitor = new Object();
+    private DeploymentTester deploymentTester;
+
+    @Override
+    protected Networking networking() { return Networking.enable; }
+
+    public static void main(String[] args) throws IOException, InterruptedException {
+        DeploymentPlayground test = null;
+        try {
+            test = new DeploymentPlayground();
+            test.startContainer();
+            test.run();
+        }
+        catch (Throwable t) {
+            t.printStackTrace();
+        }
+        if (test != null && test.container != null) {
+            test.stopContainer();
+            test.deploymentTester.runner().shutdown();
+            test.deploymentTester.upgrader().shutdown();
+            test.deploymentTester.readyJobsTrigger().shutdown();
+            test.deploymentTester.outstandingChangeDeployer().shutdown();
+        }
+    }
+
+    public void run() throws IOException {
+        ContainerTester tester = new ContainerTester(container, "");
+        deploymentTester = new DeploymentTester(new ControllerTester(tester));
+        deploymentTester.controllerTester().computeVersionStatus();
+
+        AthenzDbMock.Domain domainMock = tester.athenzClientFactory().getSetup().getOrCreateDomain(AllowingFilter.domain);
+        domainMock.markAsVespaTenant();
+        domainMock.admin(AllowingFilter.user.getIdentity());
+
+        Map<String, DeploymentContext> instances = new LinkedHashMap<>();
+        for (String name : List.of("alpha", "beta", "prod5", "prod25", "prod100"))
+              instances.put(name, deploymentTester.newDeploymentContext("gemini", "core", name));
+
+        instances.values().iterator().next().submit(ApplicationPackageBuilder.fromDeploymentXml(readDeploymentXml())).deploy();
+
+        repl(instances);
+    }
+
+    static String readDeploymentXml() throws IOException {
+        return Files.readString(Paths.get(System.getProperty("user.home") + "/git/" +
+                                          "vespa/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml"));
+    }
+
+    void repl(Map<String, DeploymentContext> instances) throws IOException {
+        String[] command;
+        BufferedReader in = new BufferedReader(new InputStreamReader(System.in));
+        AtomicBoolean on = new AtomicBoolean();
+        Thread auto = new Thread(() -> auto(instances, on));
+        auto.setDaemon(true);
+        auto.start();
+        while (true) {
+            try {
+                command = in.readLine().trim().split("\\s+");
+                if (command.length == 0 || command[0].isEmpty()) continue;
+                synchronized (monitor) {
+                    switch (command[0]) {
+                        case "exit":
+                            auto.interrupt();
+                            return;
+                        case "tick":
+                            deploymentTester.controllerTester().computeVersionStatus();
+                            deploymentTester.outstandingChangeDeployer().run();
+                            deploymentTester.upgrader().run();
+                            deploymentTester.triggerJobs();
+                            deploymentTester.runner().run();
+                            break;
+                        case "run":
+                            run(instances.get(command[1]), DeploymentContext::runJob, List.of(command).subList(2, command.length));
+                            break;
+                        case "fail":
+                            run(instances.get(command[1]), DeploymentContext::failDeployment, List.of(command).subList(2, command.length));
+                            break;
+                        case "upgrade":
+                            deploymentTester.controllerTester().upgradeSystem(new Version(command[1]));
+                            deploymentTester.controllerTester().computeVersionStatus();
+                            break;
+                        case "submit":
+                            instances.values().iterator().next().submit(ApplicationPackageBuilder.fromDeploymentXml(readDeploymentXml()));
+                            break;
+                        case "resubmit":
+                            instances.values().iterator().next().resubmit(ApplicationPackageBuilder.fromDeploymentXml(readDeploymentXml()));
+                            break;
+                        case "advance":
+                            deploymentTester.clock().advance(Duration.ofMinutes(Long.parseLong(command[1])));
+                            break;
+                        case "auto":
+                            switch (command[1]) {
+                                case "on":  on.set(true); break;
+                                case "off": on.set(false); break;
+                                default: System.err.println("Argument to 'auto' must be 'on' or 'off'"); break;
+                            }
+                            break;
+                        default:
+                            System.err.println("Cannot run '" + String.join(" ", command) + "'");
+                    }
+                }
+            }
+            catch (Throwable t) {
+                t.printStackTrace();
+            }
+        }
+    }
+
+    void auto(Map<String, DeploymentContext> instances, AtomicBoolean on) {
+        while ( ! Thread.currentThread().isInterrupted()) {
+            try {
+                synchronized (monitor) {
+                    monitor.wait(6000);
+                    if ( ! on.get())
+                        continue;
+
+                    System.err.println("auto running");
+                    deploymentTester.clock().advance(Duration.ofSeconds(60));
+                    deploymentTester.controllerTester().computeVersionStatus();
+                    deploymentTester.outstandingChangeDeployer().run();
+                    deploymentTester.upgrader().run();
+                    deploymentTester.triggerJobs();
+                    deploymentTester.runner().run();
+                    for (Run run : deploymentTester.jobs().active())
+                        if (run.versions().sourcePlatform().map(run.versions().targetPlatform()::equals).orElse(true) || Math.random() < 0.4)
+                            instances.get(run.id().application().instance().value()).runJob(run.id().type());
+                }
+            }
+            catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+            catch (Throwable t) {
+                t.printStackTrace();
+            }
+        }
+    }
+
+    void run(DeploymentContext instance, BiConsumer<DeploymentContext, JobType> action, List<String> jobs) {
+        for (boolean triggered = true; triggered; ) {
+            triggered = false;
+            for (Run run : deploymentTester.jobs().active(instance.instanceId()))
+                if (jobs.isEmpty() || jobs.contains(run.id().type().jobName().replace("production-", ""))) {
+                    action.accept(instance, run.id().type());
+                    triggered = true;
+                }
+        }
+    }
+
+    @Override
+    protected String variablePartXml() {
+        return "  <component id='com.yahoo.vespa.hosted.controller.security.AthenzAccessControlRequests'/>\n" +
+               "  <component id='com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade'/>\n" +
+
+               "  <handler id='com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiHandler'>\n" +
+               "    <binding>http://*/application/v4/*</binding>\n" +
+               "  </handler>\n" +
+               "  <handler id='com.yahoo.vespa.hosted.controller.restapi.athenz.AthenzApiHandler'>\n" +
+               "    <binding>http://*/athenz/v1/*</binding>\n" +
+               "  </handler>\n" +
+               "  <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v1.ZoneApiHandler'>\n" +
+               "    <binding>http://*/zone/v1</binding>\n" +
+               "    <binding>http://*/zone/v1/*</binding>\n" +
+               "  </handler>\n" +
+
+               "  <http>\n" +
+               "    <server id='default' port='8080' />\n" +
+               "    <filtering>\n" +
+               "      <request-chain id='default'>\n" +
+               "        <filter id='com.yahoo.jdisc.http.filter.security.cors.CorsPreflightRequestFilter'>\n" +
+               "          <config name=\"jdisc.http.filter.security.cors.cors-filter\">" +
+               "            <allowedUrls>\n" +
+               "              <item>http://localhost:3000</item>\n" +
+               "              <item>http://localhost:8080</item>\n" +
+               "            </allowedUrls>\n" +
+               "          </config>\n" +
+               "        </filter>\n" +
+               "        <filter id='com.yahoo.vespa.hosted.controller.restapi.playground.AllowingFilter'/>\n" +
+               "        <filter id='com.yahoo.vespa.hosted.controller.restapi.filter.ControllerAuthorizationFilter'/>\n" +
+               "        <binding>http://*/*</binding>\n" +
+               "      </request-chain>\n" +
+               "      <response-chain id='responses'>\n" +
+               "        <filter id='com.yahoo.jdisc.http.filter.security.cors.CorsResponseFilter'>\n" +
+               "          <config name=\"jdisc.http.filter.security.cors.cors-filter\">" +
+               "            <allowedUrls>\n" +
+               "              <item>http://localhost:3000</item>\n" +
+               "              <item>http://localhost:8080</item>\n" +
+               "            </allowedUrls>\n" +
+               "          </config>\n" +
+               "        </filter>\n" +
+               "        <binding>http://*/*</binding>\n" +
+               "      </response-chain>\n" +
+               "    </filtering>\n" +
+               "  </http>\n";
+    }
+
+}
+
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml
new file mode 100644
index 00000000000..a3642acb21a
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml
@@ -0,0 +1,71 @@
+<deployment>
+    <instance id='alpha'>
+        <upgrade rollout="simultaneous" revision-change="when-failing" revision-target="next" />
+        <test />
+        <staging />
+    </instance>
+    <instance id='beta'>
+        <upgrade rollout="simultaneous" revision-change="when-clear" revision-target="latest" />
+        <prod>
+            <region>us-east-3</region>
+            <test>us-east-3</test>
+        </prod>
+    </instance>
+    <instance id='prod5'>
+        <upgrade rollout="simultaneous" revision-change="when-clear" revision-target="next" />
+        <prod>
+            <parallel>
+                <steps>
+                    <region>us-east-3</region>
+                    <test>us-east-3</test>
+                </steps>
+                <steps>
+                    <region>us-central-1</region>
+                    <test>us-central-1</test>
+                </steps>
+                <steps>
+                    <region>us-west-1</region>
+                    <test>us-west-1</test>
+                </steps>
+            </parallel>
+        </prod>
+    </instance>
+    <instance id='prod25'>
+        <upgrade rollout="simultaneous" revision-change="when-failing" revision-target="next" />
+        <prod>
+            <parallel>
+                <steps>
+                    <region>us-east-3</region>
+                    <test>us-east-3</test>
+                </steps>
+                <steps>
+                    <region>us-central-1</region>
+                    <test>us-central-1</test>
+                </steps>
+                <steps>
+                    <region>us-west-1</region>
+                    <test>us-west-1</test>
+                </steps>
+            </parallel>
+        </prod>
+    </instance>
+    <instance id='prod100'>
+        <upgrade rollout="simultaneous" revision-change="when-clear" revision-target="next" />
+        <prod>
+            <parallel>
+                <steps>
+                    <region>us-east-3</region>
+                    <test>us-east-3</test>
+                </steps>
+                <steps>
+                    <region>us-central-1</region>
+                    <test>us-central-1</test>
+                </steps>
+                <steps>
+                    <region>us-west-1</region>
+                    <test>us-west-1</test>
+                </steps>
+            </parallel>
+        </prod>
+    </instance>
+</deployment>