From 5d0f1f6e54a5d4bb13cbebb8055f887019d0c62c Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 8 Mar 2022 09:38:11 +0100 Subject: Add deployment playground --- .../restapi/ControllerContainerTest.java | 4 +- .../restapi/playground/AllowingFilter.java | 41 ++++ .../playground/DeploymentPlaygroundTest.java | 229 +++++++++++++++++++++ .../controller/restapi/playground/deployment.xml | 71 +++++++ 4 files changed, 344 insertions(+), 1 deletion(-) create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/AllowingFilter.java create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlaygroundTest.java create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index 6b13e6c951a..87fe8dd17fe 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -35,10 +35,12 @@ public class ControllerContainerTest { @Before public void startContainer() { - container = JDisc.fromServicesXml(controllerServicesXml(), Networking.disable); + container = JDisc.fromServicesXml(controllerServicesXml(), networking()); addUserToHostedOperatorRole(hostedOperator); } + protected Networking networking() { return Networking.disable; } + @After public void stopContainer() { container.close(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/AllowingFilter.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/AllowingFilter.java new file mode 100644 index 00000000000..de7f2515979 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/AllowingFilter.java @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.playground; + +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; +import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.AthenzPrincipal; +import com.yahoo.vespa.athenz.api.AthenzUser; +import com.yahoo.vespa.hosted.controller.api.integration.user.User; +import com.yahoo.vespa.hosted.controller.api.role.Role; +import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; +import com.yahoo.yolean.Exceptions; + +import java.time.Instant; +import java.util.Optional; +import java.util.Set; + +public class AllowingFilter extends JsonSecurityRequestFilterBase { + + static final AthenzPrincipal user = new AthenzPrincipal(new AthenzUser("demo")); + static final AthenzDomain domain = new AthenzDomain("domain"); + + @Override + protected Optional filter(DiscFilterRequest request) { + try { + request.setUserPrincipal(user); + request.setAttribute(User.ATTRIBUTE_NAME, new User("mail@mail", user.getName(), "demo", null, true, -1, User.NO_DATE)); + request.setAttribute("okta.identity-token", "okta-it"); + request.setAttribute("okta.access-token", "okta-at"); + request.setAttribute(SecurityContext.ATTRIBUTE_NAME, + new SecurityContext(user, + Set.of(Role.hostedOperator()), + Instant.now().minusSeconds(3600))); + return Optional.empty(); + } + catch (Throwable t) { + return Optional.of(new ErrorResponse(500, Exceptions.toMessageString(t))); + } + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlaygroundTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlaygroundTest.java new file mode 100644 index 00000000000..f6fce6f7403 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlaygroundTest.java @@ -0,0 +1,229 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.playground; + +import com.yahoo.application.Networking; +import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.deployment.Run; +import com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance; +import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; +import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; +import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiConsumer; + +public class DeploymentPlaygroundTest extends ControllerContainerTest { + + private final Object monitor = new Object(); + private ContainerTester tester; + private DeploymentTester deploymentTester; + + @Override + protected Networking networking() { return Networking.enable; } + + public static void main(String[] args) throws IOException, InterruptedException { + DeploymentPlaygroundTest test = null; + try { + test = new DeploymentPlaygroundTest(); + test.startContainer(); + test.run(); + } + catch (Throwable t) { + t.printStackTrace(); + } + if (test != null && test.container != null) { + test.stopContainer(); + test.deploymentTester.runner().shutdown(); + test.deploymentTester.upgrader().shutdown(); + test.deploymentTester.readyJobsTrigger().shutdown(); + test.deploymentTester.outstandingChangeDeployer().shutdown(); + } + } + + public void run() throws IOException { + tester = new ContainerTester(container, ""); + deploymentTester = new DeploymentTester(new ControllerTester(tester)); + deploymentTester.controllerTester().computeVersionStatus(); + + AthenzDbMock.Domain domainMock = tester.athenzClientFactory().getSetup().getOrCreateDomain(AllowingFilter.domain); + domainMock.markAsVespaTenant(); + domainMock.admin(AllowingFilter.user.getIdentity()); + + + Map instances = new LinkedHashMap<>(); + for (String name : List.of("alpha", "beta", "prod5", "prod25", "prod100")) + instances.put(name, deploymentTester.newDeploymentContext("gemini", "core", name)); + + instances.values().iterator().next().submit(ApplicationPackageBuilder.fromDeploymentXml(readDeploymentXml())).deploy(); + + repl(instances); + } + + static String readDeploymentXml() throws IOException { + return Files.readString(Paths.get(System.getProperty("user.home") + "/git/vespa/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml")); + } + + void repl(Map instances) throws IOException { + String[] command; + BufferedReader in = new BufferedReader(new InputStreamReader(System.in)); + AtomicBoolean on = new AtomicBoolean(); + Thread auto = new Thread(() -> auto(instances, on)); + auto.setDaemon(true); + auto.start(); + while (true) { + try { + command = in.readLine().trim().split("\\s+"); + if (command.length == 0 || command[0].isEmpty()) continue; + synchronized (monitor) { + switch (command[0]) { + case "exit": + auto.interrupt(); + return; + case "tick": + deploymentTester.controllerTester().computeVersionStatus(); + deploymentTester.outstandingChangeDeployer().run(); + deploymentTester.upgrader().run(); + deploymentTester.triggerJobs(); + deploymentTester.runner().run(); + break; + case "run": + run(instances.get(command[1]), DeploymentContext::runJob, List.of(command).subList(2, command.length)); + break; + case "fail": + run(instances.get(command[1]), DeploymentContext::failDeployment, List.of(command).subList(2, command.length)); + break; + case "upgrade": + deploymentTester.controllerTester().upgradeSystem(new Version(command[1])); + deploymentTester.controllerTester().computeVersionStatus(); + break; + case "submit": + instances.values().iterator().next().submit(ApplicationPackageBuilder.fromDeploymentXml(readDeploymentXml())); + break; + case "resubmit": + instances.values().iterator().next().resubmit(ApplicationPackageBuilder.fromDeploymentXml(readDeploymentXml())); + break; + case "advance": + deploymentTester.clock().advance(Duration.ofMinutes(Long.parseLong(command[1]))); + break; + case "auto": + switch (command[1]) { + case "on": on.set(true); break; + case "off": on.set(false); break; + default: System.err.println("Argument to 'auto' must be 'on' or 'off'"); break; + } + break; + default: + System.err.println("Cannot run '" + String.join(" ", command) + "'"); + } + } + } + catch (Throwable t) { + t.printStackTrace(); + } + } + } + + void auto(Map instances, AtomicBoolean on) { + while ( ! Thread.currentThread().isInterrupted()) { + try { + synchronized (monitor) { + monitor.wait(6000); + if ( ! on.get()) + continue; + + System.err.println("auto running"); + deploymentTester.clock().advance(Duration.ofSeconds(60)); + deploymentTester.controllerTester().computeVersionStatus(); + deploymentTester.outstandingChangeDeployer().run(); + deploymentTester.upgrader().run(); + deploymentTester.triggerJobs(); + deploymentTester.runner().run(); + for (Run run : deploymentTester.jobs().active()) + instances.get(run.id().application().instance().value()).runJob(run.id().type()); + } + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + catch (Throwable t) { + t.printStackTrace(); + } + } + } + + void run(DeploymentContext instance, BiConsumer action, List jobs) { + for (boolean triggered = true; triggered; ) { + triggered = false; + for (Run run : deploymentTester.jobs().active(instance.instanceId())) + if (jobs.isEmpty() || jobs.contains(run.id().type().jobName().replace("production-", ""))) { + action.accept(instance, run.id().type()); + triggered = true; + } + } + } + + @Override + protected String variablePartXml() { + return " \n" + + " \n" + + + " \n" + + " http://*/application/v4/*\n" + + " \n" + + " \n" + + " http://*/athenz/v1/*\n" + + " \n" + + " \n" + + " http://*/zone/v1\n" + + " http://*/zone/v1/*\n" + + " \n" + + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " " + + " \n" + + " http://localhost:3000\n" + + " http://localhost:8080\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " http://*/*\n" + + " \n" + + " \n" + + " \n" + + " " + + " \n" + + " http://localhost:3000\n" + + " http://localhost:8080\n" + + " \n" + + " \n" + + " \n" + + " http://*/*\n" + + " \n" + + " \n" + + " \n"; + } + +} + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml new file mode 100644 index 00000000000..cd8894b155e --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml @@ -0,0 +1,71 @@ + + + + + + + + + + us-east-3 + us-east-3 + + + + + + + + us-east-3 + us-east-3 + + + us-central-1 + us-central-1 + + + us-west-1 + us-west-1 + + + + + + + + + + us-east-3 + us-east-3 + + + us-central-1 + us-central-1 + + + us-west-1 + us-west-1 + + + + + + + + + + us-east-3 + us-east-3 + + + us-central-1 + us-central-1 + + + us-west-1 + us-west-1 + + + + + -- cgit v1.2.3 From 5f7841a5f2d28ea3cbd97d3ecdde425be20ac932 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 8 Mar 2022 10:30:51 +0100 Subject: More interesting --- .../restapi/playground/DeploymentPlayground.java | 227 ++++++++++++++++++++ .../playground/DeploymentPlaygroundTest.java | 229 --------------------- .../controller/restapi/playground/deployment.xml | 3 +- 3 files changed, 229 insertions(+), 230 deletions(-) create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlayground.java delete mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlaygroundTest.java diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlayground.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlayground.java new file mode 100644 index 00000000000..bf41a34c62b --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlayground.java @@ -0,0 +1,227 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.playground; + +import com.yahoo.application.Networking; +import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.deployment.Run; +import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; +import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiConsumer; + +public class DeploymentPlayground extends ControllerContainerTest { + + private final Object monitor = new Object(); + private DeploymentTester deploymentTester; + + @Override + protected Networking networking() { return Networking.enable; } + + public static void main(String[] args) throws IOException, InterruptedException { + DeploymentPlayground test = null; + try { + test = new DeploymentPlayground(); + test.startContainer(); + test.run(); + } + catch (Throwable t) { + t.printStackTrace(); + } + if (test != null && test.container != null) { + test.stopContainer(); + test.deploymentTester.runner().shutdown(); + test.deploymentTester.upgrader().shutdown(); + test.deploymentTester.readyJobsTrigger().shutdown(); + test.deploymentTester.outstandingChangeDeployer().shutdown(); + } + } + + public void run() throws IOException { + ContainerTester tester = new ContainerTester(container, ""); + deploymentTester = new DeploymentTester(new ControllerTester(tester)); + deploymentTester.controllerTester().computeVersionStatus(); + + AthenzDbMock.Domain domainMock = tester.athenzClientFactory().getSetup().getOrCreateDomain(AllowingFilter.domain); + domainMock.markAsVespaTenant(); + domainMock.admin(AllowingFilter.user.getIdentity()); + + + Map instances = new LinkedHashMap<>(); + for (String name : List.of("alpha", "beta", "prod5", "prod25", "prod100")) + instances.put(name, deploymentTester.newDeploymentContext("gemini", "core", name)); + + instances.values().iterator().next().submit(ApplicationPackageBuilder.fromDeploymentXml(readDeploymentXml())).deploy(); + + repl(instances); + } + + static String readDeploymentXml() throws IOException { + return Files.readString(Paths.get(System.getProperty("user.home") + "/git/" + + "vespa/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml")); + } + + void repl(Map instances) throws IOException { + String[] command; + BufferedReader in = new BufferedReader(new InputStreamReader(System.in)); + AtomicBoolean on = new AtomicBoolean(); + Thread auto = new Thread(() -> auto(instances, on)); + auto.setDaemon(true); + auto.start(); + while (true) { + try { + command = in.readLine().trim().split("\\s+"); + if (command.length == 0 || command[0].isEmpty()) continue; + synchronized (monitor) { + switch (command[0]) { + case "exit": + auto.interrupt(); + return; + case "tick": + deploymentTester.controllerTester().computeVersionStatus(); + deploymentTester.outstandingChangeDeployer().run(); + deploymentTester.upgrader().run(); + deploymentTester.triggerJobs(); + deploymentTester.runner().run(); + break; + case "run": + run(instances.get(command[1]), DeploymentContext::runJob, List.of(command).subList(2, command.length)); + break; + case "fail": + run(instances.get(command[1]), DeploymentContext::failDeployment, List.of(command).subList(2, command.length)); + break; + case "upgrade": + deploymentTester.controllerTester().upgradeSystem(new Version(command[1])); + deploymentTester.controllerTester().computeVersionStatus(); + break; + case "submit": + instances.values().iterator().next().submit(ApplicationPackageBuilder.fromDeploymentXml(readDeploymentXml())); + break; + case "resubmit": + instances.values().iterator().next().resubmit(ApplicationPackageBuilder.fromDeploymentXml(readDeploymentXml())); + break; + case "advance": + deploymentTester.clock().advance(Duration.ofMinutes(Long.parseLong(command[1]))); + break; + case "auto": + switch (command[1]) { + case "on": on.set(true); break; + case "off": on.set(false); break; + default: System.err.println("Argument to 'auto' must be 'on' or 'off'"); break; + } + break; + default: + System.err.println("Cannot run '" + String.join(" ", command) + "'"); + } + } + } + catch (Throwable t) { + t.printStackTrace(); + } + } + } + + void auto(Map instances, AtomicBoolean on) { + while ( ! Thread.currentThread().isInterrupted()) { + try { + synchronized (monitor) { + monitor.wait(6000); + if ( ! on.get()) + continue; + + System.err.println("auto running"); + deploymentTester.clock().advance(Duration.ofSeconds(60)); + deploymentTester.controllerTester().computeVersionStatus(); + deploymentTester.outstandingChangeDeployer().run(); + deploymentTester.upgrader().run(); + deploymentTester.triggerJobs(); + deploymentTester.runner().run(); + for (Run run : deploymentTester.jobs().active()) + if (run.versions().sourcePlatform().map(run.versions().targetPlatform()::equals).orElse(true) || Math.random() < 0.4) + instances.get(run.id().application().instance().value()).runJob(run.id().type()); + } + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + catch (Throwable t) { + t.printStackTrace(); + } + } + } + + void run(DeploymentContext instance, BiConsumer action, List jobs) { + for (boolean triggered = true; triggered; ) { + triggered = false; + for (Run run : deploymentTester.jobs().active(instance.instanceId())) + if (jobs.isEmpty() || jobs.contains(run.id().type().jobName().replace("production-", ""))) { + action.accept(instance, run.id().type()); + triggered = true; + } + } + } + + @Override + protected String variablePartXml() { + return " \n" + + " \n" + + + " \n" + + " http://*/application/v4/*\n" + + " \n" + + " \n" + + " http://*/athenz/v1/*\n" + + " \n" + + " \n" + + " http://*/zone/v1\n" + + " http://*/zone/v1/*\n" + + " \n" + + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " " + + " \n" + + " http://localhost:3000\n" + + " http://localhost:8080\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " http://*/*\n" + + " \n" + + " \n" + + " \n" + + " " + + " \n" + + " http://localhost:3000\n" + + " http://localhost:8080\n" + + " \n" + + " \n" + + " \n" + + " http://*/*\n" + + " \n" + + " \n" + + " \n"; + } + +} + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlaygroundTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlaygroundTest.java deleted file mode 100644 index f6fce6f7403..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlaygroundTest.java +++ /dev/null @@ -1,229 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.playground; - -import com.yahoo.application.Networking; -import com.yahoo.component.Version; -import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; -import com.yahoo.vespa.hosted.controller.deployment.Run; -import com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance; -import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; -import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; -import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.time.Duration; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.BiConsumer; - -public class DeploymentPlaygroundTest extends ControllerContainerTest { - - private final Object monitor = new Object(); - private ContainerTester tester; - private DeploymentTester deploymentTester; - - @Override - protected Networking networking() { return Networking.enable; } - - public static void main(String[] args) throws IOException, InterruptedException { - DeploymentPlaygroundTest test = null; - try { - test = new DeploymentPlaygroundTest(); - test.startContainer(); - test.run(); - } - catch (Throwable t) { - t.printStackTrace(); - } - if (test != null && test.container != null) { - test.stopContainer(); - test.deploymentTester.runner().shutdown(); - test.deploymentTester.upgrader().shutdown(); - test.deploymentTester.readyJobsTrigger().shutdown(); - test.deploymentTester.outstandingChangeDeployer().shutdown(); - } - } - - public void run() throws IOException { - tester = new ContainerTester(container, ""); - deploymentTester = new DeploymentTester(new ControllerTester(tester)); - deploymentTester.controllerTester().computeVersionStatus(); - - AthenzDbMock.Domain domainMock = tester.athenzClientFactory().getSetup().getOrCreateDomain(AllowingFilter.domain); - domainMock.markAsVespaTenant(); - domainMock.admin(AllowingFilter.user.getIdentity()); - - - Map instances = new LinkedHashMap<>(); - for (String name : List.of("alpha", "beta", "prod5", "prod25", "prod100")) - instances.put(name, deploymentTester.newDeploymentContext("gemini", "core", name)); - - instances.values().iterator().next().submit(ApplicationPackageBuilder.fromDeploymentXml(readDeploymentXml())).deploy(); - - repl(instances); - } - - static String readDeploymentXml() throws IOException { - return Files.readString(Paths.get(System.getProperty("user.home") + "/git/vespa/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml")); - } - - void repl(Map instances) throws IOException { - String[] command; - BufferedReader in = new BufferedReader(new InputStreamReader(System.in)); - AtomicBoolean on = new AtomicBoolean(); - Thread auto = new Thread(() -> auto(instances, on)); - auto.setDaemon(true); - auto.start(); - while (true) { - try { - command = in.readLine().trim().split("\\s+"); - if (command.length == 0 || command[0].isEmpty()) continue; - synchronized (monitor) { - switch (command[0]) { - case "exit": - auto.interrupt(); - return; - case "tick": - deploymentTester.controllerTester().computeVersionStatus(); - deploymentTester.outstandingChangeDeployer().run(); - deploymentTester.upgrader().run(); - deploymentTester.triggerJobs(); - deploymentTester.runner().run(); - break; - case "run": - run(instances.get(command[1]), DeploymentContext::runJob, List.of(command).subList(2, command.length)); - break; - case "fail": - run(instances.get(command[1]), DeploymentContext::failDeployment, List.of(command).subList(2, command.length)); - break; - case "upgrade": - deploymentTester.controllerTester().upgradeSystem(new Version(command[1])); - deploymentTester.controllerTester().computeVersionStatus(); - break; - case "submit": - instances.values().iterator().next().submit(ApplicationPackageBuilder.fromDeploymentXml(readDeploymentXml())); - break; - case "resubmit": - instances.values().iterator().next().resubmit(ApplicationPackageBuilder.fromDeploymentXml(readDeploymentXml())); - break; - case "advance": - deploymentTester.clock().advance(Duration.ofMinutes(Long.parseLong(command[1]))); - break; - case "auto": - switch (command[1]) { - case "on": on.set(true); break; - case "off": on.set(false); break; - default: System.err.println("Argument to 'auto' must be 'on' or 'off'"); break; - } - break; - default: - System.err.println("Cannot run '" + String.join(" ", command) + "'"); - } - } - } - catch (Throwable t) { - t.printStackTrace(); - } - } - } - - void auto(Map instances, AtomicBoolean on) { - while ( ! Thread.currentThread().isInterrupted()) { - try { - synchronized (monitor) { - monitor.wait(6000); - if ( ! on.get()) - continue; - - System.err.println("auto running"); - deploymentTester.clock().advance(Duration.ofSeconds(60)); - deploymentTester.controllerTester().computeVersionStatus(); - deploymentTester.outstandingChangeDeployer().run(); - deploymentTester.upgrader().run(); - deploymentTester.triggerJobs(); - deploymentTester.runner().run(); - for (Run run : deploymentTester.jobs().active()) - instances.get(run.id().application().instance().value()).runJob(run.id().type()); - } - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - catch (Throwable t) { - t.printStackTrace(); - } - } - } - - void run(DeploymentContext instance, BiConsumer action, List jobs) { - for (boolean triggered = true; triggered; ) { - triggered = false; - for (Run run : deploymentTester.jobs().active(instance.instanceId())) - if (jobs.isEmpty() || jobs.contains(run.id().type().jobName().replace("production-", ""))) { - action.accept(instance, run.id().type()); - triggered = true; - } - } - } - - @Override - protected String variablePartXml() { - return " \n" + - " \n" + - - " \n" + - " http://*/application/v4/*\n" + - " \n" + - " \n" + - " http://*/athenz/v1/*\n" + - " \n" + - " \n" + - " http://*/zone/v1\n" + - " http://*/zone/v1/*\n" + - " \n" + - - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " " + - " \n" + - " http://localhost:3000\n" + - " http://localhost:8080\n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " http://*/*\n" + - " \n" + - " \n" + - " \n" + - " " + - " \n" + - " http://localhost:3000\n" + - " http://localhost:8080\n" + - " \n" + - " \n" + - " \n" + - " http://*/*\n" + - " \n" + - " \n" + - " \n"; - } - -} - diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml index cd8894b155e..10e96d4145b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml @@ -8,6 +8,7 @@ us-east-3 + us-east-3 @@ -24,7 +25,7 @@ us-central-1 - us-west-1 + us-west-1 us-west-1 -- cgit v1.2.3 From c550e7afe2725d1daa0e474fd4357107c62f40d7 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 8 Mar 2022 12:39:01 +0100 Subject: Delay does not work with "deploy()" --- .../com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml index 10e96d4145b..a3642acb21a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml @@ -8,7 +8,6 @@ us-east-3 - us-east-3 -- cgit v1.2.3 From 3a495a015c1c4acdcf00b9bd3c601b170fbef5d7 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 8 Mar 2022 17:57:48 +0100 Subject: Fix API response --- .../application/JobControllerApiHandlerHelper.java | 27 ++++++++++++---------- .../application/responses/deployment-overview.json | 8 ------- .../restapi/playground/DeploymentPlayground.java | 1 - 3 files changed, 15 insertions(+), 21 deletions(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 2b52143f574..189192d8d9a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -266,10 +266,12 @@ class JobControllerApiHandlerHelper { stepObject.setBool("declared", stepStatus.isDeclared()); stepObject.setString("instance", stepStatus.instance().value()); - stepStatus.readyAt(change).ifPresent(ready -> stepObject.setLong("readyAt", ready.toEpochMilli())); - stepStatus.readyAt(change) - .filter(controller.clock().instant()::isBefore) - .ifPresent(until -> stepObject.setLong("delayedUntil", until.toEpochMilli())); + // TODO: recursively search dependents for what is the relevant partial change when this is a delay step ... + Optional readyAt = stepStatus.job().map(jobsToRun::get).map(jobs -> jobs.get(0).readyAt()) + .orElse(stepStatus.readyAt(change)); + readyAt.ifPresent(ready -> stepObject.setLong("readyAt", ready.toEpochMilli())); + readyAt.filter(controller.clock().instant()::isBefore) + .ifPresent(until -> stepObject.setLong("delayedUntil", until.toEpochMilli())); stepStatus.pausedUntil().ifPresent(until -> stepObject.setLong("pausedUntil", until.toEpochMilli())); stepStatus.coolingDownUntil(change).ifPresent(until -> stepObject.setLong("coolingDownUntil", until.toEpochMilli())); stepStatus.blockedUntil(Change.of(controller.systemVersion(versionStatus))) // Dummy version — just anything with a platform. @@ -304,17 +306,17 @@ class JobControllerApiHandlerHelper { for (VespaVersion available : availablePlatforms) { if ( deployments.stream().anyMatch(deployment -> deployment.version().isAfter(available.versionNumber())) || deployments.stream().noneMatch(deployment -> deployment.version().isBefore(available.versionNumber())) && ! deployments.isEmpty() - || change.platform().map(available.versionNumber()::compareTo).orElse(1) < 0) + || status.hasCompleted(stepStatus.instance(), Change.of(available.versionNumber())) + || change.platform().map(available.versionNumber()::compareTo).orElse(1) <= 0) break; - Cursor availableObject = availableArray.addObject(); - availableObject.setString("platform", available.versionNumber().toFullString()); + availableArray.addObject().setString("platform", available.versionNumber().toFullString()); } + change.platform().ifPresent(version -> availableArray.addObject().setString("platform", version.toFullString())); toSlime(latestPlatformObject.setArray("blockers"), blockers.stream().filter(ChangeBlocker::blocksVersions)); } - List availableApplications = new ArrayList<>(application.versions()); + List availableApplications = new ArrayList<>(application.deployableVersions(false)); if ( ! availableApplications.isEmpty()) { - Collections.reverse(availableApplications); var latestApplication = availableApplications.get(0); Cursor latestApplicationObject = latestVersionsObject.setObject("application"); toSlime(latestApplicationObject.setObject("application"), latestApplication); @@ -326,12 +328,13 @@ class JobControllerApiHandlerHelper { for (ApplicationVersion available : availableApplications) { if ( deployments.stream().anyMatch(deployment -> deployment.applicationVersion().compareTo(available) > 0) || deployments.stream().noneMatch(deployment -> deployment.applicationVersion().compareTo(available) < 0) && ! deployments.isEmpty() - || change.application().map(available::compareTo).orElse(1) < 0) + || status.hasCompleted(stepStatus.instance(), Change.of(available)) + || change.application().map(available::compareTo).orElse(1) <= 0) break; - Cursor availableObject = availableArray.addObject(); - toSlime(availableObject.setObject("application"), available); + toSlime(availableArray.addObject().setObject("application"), available); } + change.application().ifPresent(version -> toSlime(availableArray.addObject().setObject("application"), version)); toSlime(latestApplicationObject.setArray("blockers"), blockers.stream().filter(ChangeBlocker::blocksRevisions)); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json index 8677b621f0d..c0a6829b026 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json @@ -577,14 +577,6 @@ "commit": "commit1" } }, - { - "application": { - "build": 3, - "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - } - }, { "application": { "build": 2, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlayground.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlayground.java index bf41a34c62b..2c91aceb8b0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlayground.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlayground.java @@ -61,7 +61,6 @@ public class DeploymentPlayground extends ControllerContainerTest { domainMock.markAsVespaTenant(); domainMock.admin(AllowingFilter.user.getIdentity()); - Map instances = new LinkedHashMap<>(); for (String name : List.of("alpha", "beta", "prod5", "prod25", "prod100")) instances.put(name, deploymentTester.newDeploymentContext("gemini", "core", name)); -- cgit v1.2.3