diff options
author | Valerij Fredriksen <freva@users.noreply.github.com> | 2018-05-04 14:48:40 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-05-04 14:48:40 +0200 |
commit | 874a5c8209351e7780a0bb7033b16925d5128d24 (patch) | |
tree | 76bc262f4ca3b8356075d612c8ca78e042fe4220 | |
parent | 1d35cdbe25ce6e69ecdf30891ca60def0cd1ba7c (diff) | |
parent | 05e338db4a96e7b195f39bb7a35782ac8c4ece38 (diff) |
Merge pull request #5784 from vespa-engine/freva/infra-app-provision
Infrastructure app provision
16 files changed, 614 insertions, 21 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java new file mode 100644 index 00000000000..b6955195dcf --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java @@ -0,0 +1,104 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.Provisioner; +import com.yahoo.log.LogLevel; +import com.yahoo.transaction.Mutex; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.service.monitor.application.ConfigServerApplication; +import com.yahoo.vespa.service.monitor.application.ConfigServerHostApplication; +import com.yahoo.vespa.service.monitor.application.HostedVespaApplication; +import com.yahoo.vespa.service.monitor.application.ProxyHostApplication; + +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * This maintainer makes sure that infrastructure nodes are allocated with correct wanted + * version. Source for the wanted version comes from the target version set using + * /nodes/v2/upgrade/ endpoint. + * + * @author freva + */ +public class InfrastructureProvisioner extends Maintainer { + + private static final Logger logger = Logger.getLogger(InfrastructureProvisioner.class.getName()); + private static final List<HostedVespaApplication> HOSTED_VESPA_APPLICATIONS = Arrays.asList( + ConfigServerApplication.CONFIG_SERVER_APPLICATION, + ConfigServerHostApplication.CONFIG_SERVER_HOST_APPLICATION, + ProxyHostApplication.PROXY_HOST_APPLICATION); + + private final Provisioner provisioner; + private final InfrastructureVersions infrastructureVersions; + + public InfrastructureProvisioner(Provisioner provisioner, NodeRepository nodeRepository, + InfrastructureVersions infrastructureVersions, Duration interval, JobControl jobControl) { + super(nodeRepository, interval, jobControl); + this.provisioner = provisioner; + this.infrastructureVersions = infrastructureVersions; + } + + @Override + protected void maintain() { + for (HostedVespaApplication application: HOSTED_VESPA_APPLICATIONS) { + try (Mutex lock = nodeRepository().lock(application.getApplicationId())) { + Optional<Version> version = getTargetVersion(application.getCapacity().type()); + if (! version.isPresent()) continue; + + List<HostSpec> hostSpecs = provisioner.prepare( + application.getApplicationId(), + application.getClusterSpecWithVersion(version.get()), + application.getCapacity(), + 1, // groups + logger::log); + + NestedTransaction nestedTransaction = new NestedTransaction(); + provisioner.activate(nestedTransaction, application.getApplicationId(), hostSpecs); + nestedTransaction.commit(); + } + } + } + + /** + * Returns the version that the given node type should be provisioned to. This is + * the version returned by {@link InfrastructureVersions#getTargetVersionFor} unless a provisioning is: + * <ul> + * <li>not possible: no nodes of given type in legal state in node-repo</li> + * <li>redundant: all nodes that can be provisioned already have the right wanted Vespa version</li> + * </ul> + */ + Optional<Version> getTargetVersion(NodeType nodeType) { + Optional<Version> targetVersion = infrastructureVersions.getTargetVersionFor(nodeType); + if (!targetVersion.isPresent()) { + logger.log(LogLevel.DEBUG, "Skipping provision of " + nodeType + ": No target version set"); + return Optional.empty(); + } + + List<Version> wantedVersions = nodeRepository().getNodes(nodeType, + Node.State.ready, Node.State.reserved, Node.State.active, Node.State.inactive).stream() + .map(node -> node.allocation() + .map(allocation -> allocation.membership().cluster().vespaVersion()) + .orElse(null)) + .collect(Collectors.toList()); + if (wantedVersions.isEmpty()) { + logger.log(LogLevel.DEBUG, "Skipping provision of " + nodeType + ": No nodes to provision"); + return Optional.empty(); + } + + if (wantedVersions.stream().allMatch(targetVersion.get()::equals)) { + logger.log(LogLevel.DEBUG, "Skipping provision of " + nodeType + + ": Already provisioned to target version " + targetVersion); + return Optional.empty(); + } + return targetVersion; + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersions.java new file mode 100644 index 00000000000..52d7a63dc5e --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersions.java @@ -0,0 +1,63 @@ +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; + +import java.util.Collections; +import java.util.Map; +import java.util.Optional; +import java.util.logging.Logger; + +/** + * Multithread safe class to see and set target versions for infrastructure node types. + * {@link InfrastructureProvisioner} maintainer will then allocate the nodes of given node type + * with a wanted version equal to the given target version. + * + * @author freva + */ +public class InfrastructureVersions { + + private static Logger logger = Logger.getLogger(InfrastructureVersions.class.getName()); + + private final CuratorDatabaseClient db; + + public InfrastructureVersions(CuratorDatabaseClient db) { + this.db = db; + } + + public void setTargetVersion(NodeType nodeType, Version newTargetVersion, boolean force) { + if (nodeType != NodeType.config && nodeType != NodeType.confighost && nodeType != NodeType.proxyhost) { + throw new IllegalArgumentException("Cannot set version for type " + nodeType); + } + + try (Lock lock = db.lockInfrastructureVersions()) { + Map<NodeType, Version> infrastructureVersions = db.readInfrastructureVersions(); + Optional<Version> currentTargetVersion = Optional.ofNullable(infrastructureVersions.get(nodeType)); + + // Trying to set the version to the current version, skip + if (currentTargetVersion.equals(Optional.of(newTargetVersion))) return; + + // If we don't force the set, we must set the new version to higher than the already set version + if (!force && currentTargetVersion.isPresent()) { + if (currentTargetVersion.get().isAfter(newTargetVersion)) + throw new IllegalArgumentException(String.format("Cannot downgrade version without setting 'force'. " + + "Current target version: %s, attempted to set target version: %s", + currentTargetVersion.get().toFullString(), newTargetVersion.toFullString())); + } + + infrastructureVersions.put(nodeType, newTargetVersion); + db.writeInfrastructureVersions(infrastructureVersions); + logger.info("Set target version for " + nodeType + " to " + newTargetVersion.toFullString()); + } + } + + public Optional<Version> getTargetVersionFor(NodeType nodeType) { + return Optional.ofNullable(db.readInfrastructureVersions().get(nodeType)); + } + + public Map<NodeType, Version> getTargetVersions() { + return Collections.unmodifiableMap(db.readInfrastructureVersions()); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 18af498a38d..32c7a4035d9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -7,6 +7,7 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostLivenessTracker; +import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -46,24 +47,27 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final NodeRebooter nodeRebooter; private final NodeRetirer nodeRetirer; private final MetricsReporter metricsReporter; + private final InfrastructureProvisioner infrastructureProvisioner; private final JobControl jobControl; + private final InfrastructureVersions infrastructureVersions; @Inject - public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, - HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor, + public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, Provisioner provisioner, + HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor, Zone zone, Orchestrator orchestrator, Metric metric, ConfigserverConfig configserverConfig) { - this(nodeRepository, deployer, hostLivenessTracker, serviceMonitor, zone, Clock.systemUTC(), + this(nodeRepository, deployer, provisioner, hostLivenessTracker, serviceMonitor, zone, Clock.systemUTC(), orchestrator, metric, configserverConfig); } - public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, + public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, Provisioner provisioner, HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor, Zone zone, Clock clock, Orchestrator orchestrator, Metric metric, ConfigserverConfig configserverConfig) { DefaultTimes defaults = new DefaultTimes(zone.environment()); jobControl = new JobControl(nodeRepository.database()); + infrastructureVersions = new InfrastructureVersions(nodeRepository.database()); nodeFailer = new NodeFailer(deployer, hostLivenessTracker, serviceMonitor, nodeRepository, durationFromEnv("fail_grace").orElse(defaults.failGrace), clock, orchestrator, throttlePolicyFromEnv("throttle_policy").orElse(defaults.throttlePolicy), metric, jobControl, configserverConfig); periodicApplicationMaintainer = new PeriodicApplicationMaintainer(deployer, nodeRepository, durationFromEnv("periodic_redeploy_interval").orElse(defaults.periodicRedeployInterval), jobControl); @@ -76,6 +80,8 @@ public class NodeRepositoryMaintenance extends AbstractComponent { provisionedExpirer = new ProvisionedExpirer(nodeRepository, clock, durationFromEnv("provisioned_expiry").orElse(defaults.provisionedExpiry), jobControl); nodeRebooter = new NodeRebooter(nodeRepository, clock, durationFromEnv("reboot_interval").orElse(defaults.rebootInterval), jobControl); metricsReporter = new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, durationFromEnv("metrics_interval").orElse(defaults.metricsInterval), jobControl); + infrastructureProvisioner = new InfrastructureProvisioner(provisioner, nodeRepository, infrastructureVersions, durationFromEnv("infrastructure_provision_interval").orElse(defaults.infrastructureProvisionInterval), jobControl); + RetirementPolicy policy = new RetirementPolicyList(new RetireIPv4OnlyNodes(zone)); FlavorSpareChecker flavorSpareChecker = new FlavorSpareChecker( @@ -97,10 +103,15 @@ public class NodeRepositoryMaintenance extends AbstractComponent { nodeRetirer.deconstruct(); provisionedExpirer.deconstruct(); metricsReporter.deconstruct(); + infrastructureProvisioner.deconstruct(); } public JobControl jobControl() { return jobControl; } + public InfrastructureVersions infrastructureVersions() { + return infrastructureVersions; + } + private static Optional<Duration> durationFromEnv(String envVariable) { return Optional.ofNullable(System.getenv(envPrefix + envVariable)).map(Long::parseLong).map(Duration::ofSeconds); } @@ -136,6 +147,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final Duration nodeRetirerInterval; private final Duration metricsInterval; private final Duration retiredInterval; + private final Duration infrastructureProvisionInterval; private final NodeFailer.ThrottlePolicy throttlePolicy; @@ -148,6 +160,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { rebootInterval = Duration.ofDays(30); nodeRetirerInterval = Duration.ofMinutes(30); metricsInterval = Duration.ofMinutes(1); + infrastructureProvisionInterval = Duration.ofMinutes(3); throttlePolicy = NodeFailer.ThrottlePolicy.hosted; if (environment.isTest()) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 7fc305d397f..44eda5e83ec 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -2,9 +2,11 @@ package com.yahoo.vespa.hosted.provision.persistence; import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; @@ -22,6 +24,7 @@ import java.time.Clock; import java.time.Duration; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -70,6 +73,7 @@ public class CuratorDatabaseClient { for (Node.State state : Node.State.values()) curatorDatabase.create(toPath(state)); curatorDatabase.create(inactiveJobsPath()); + curatorDatabase.create(infrastructureVersionsPath()); } /** @@ -363,5 +367,27 @@ public class CuratorDatabaseClient { private Path inactiveJobsPath() { return root.append("inactiveJobs"); } - + + + public Map<NodeType, Version> readInfrastructureVersions() { + byte[] data = curatorDatabase.getData(infrastructureVersionsPath()).get(); + if (data.length == 0) return new HashMap<>(); // infrastructure versions have never been written + return InfrastructureVersionsSerializer.fromJson(data); + } + + public void writeInfrastructureVersions(Map<NodeType, Version> infrastructureVersions) { + NestedTransaction transaction = new NestedTransaction(); + CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + curatorTransaction.add(CuratorOperations.setData(infrastructureVersionsPath().getAbsolute(), + InfrastructureVersionsSerializer.toJson(infrastructureVersions))); + transaction.commit(); + } + + public Lock lockInfrastructureVersions() { + return lock(root.append("locks").append("infrastructureVersionsLock"), defaultLockTimeout); + } + + private Path infrastructureVersionsPath() { + return root.append("infrastructureVersions"); + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/InfrastructureVersionsSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/InfrastructureVersionsSerializer.java new file mode 100644 index 00000000000..a48888fb4f0 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/InfrastructureVersionsSerializer.java @@ -0,0 +1,42 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.NodeType; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.ObjectTraverser; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * @author freva + */ +class InfrastructureVersionsSerializer { + + private InfrastructureVersionsSerializer() {} + + static byte[] toJson(Map<NodeType, Version> versionsByNodeType) { + try { + Slime slime = new Slime(); + Cursor object = slime.setObject(); + versionsByNodeType.forEach((nodeType, version) -> + object.setString(NodeSerializer.toString(nodeType), version.toFullString())); + return SlimeUtils.toJsonBytes(slime); + } catch (IOException e) { + throw new RuntimeException("Serialization of a infrastructure version failed", e); + } + } + + static Map<NodeType, Version> fromJson(byte[] data) { + Map<NodeType, Version> infrastructureVersions = new HashMap<>(); + Inspector inspector = SlimeUtils.jsonToSlime(data).get(); + inspector.traverse((ObjectTraverser) (key, value) -> + infrastructureVersions.put(NodeSerializer.nodeTypeFromString(key), Version.fromString(value.asString()))); + return infrastructureVersions; + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index d164252b018..4d753599c4e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -317,7 +317,7 @@ public class NodeSerializer { throw new IllegalArgumentException("Serialized form of '" + agent + "' not defined"); } - private NodeType nodeTypeFromString(String typeString) { + static NodeType nodeTypeFromString(String typeString) { switch (typeString) { case "tenant" : return NodeType.tenant; case "host" : return NodeType.host; @@ -329,7 +329,7 @@ public class NodeSerializer { } } - private String toString(NodeType type) { + static String toString(NodeType type) { switch (type) { case tenant: return "tenant"; case host: return "host"; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index 03777078251..54202a15971 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi.v2; +import com.yahoo.component.Version; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.NodeType; import com.yahoo.container.jdisc.HttpRequest; @@ -91,7 +92,7 @@ public class NodesApiHandler extends LoggingRequestHandler { private HttpResponse handleGET(HttpRequest request) { String path = request.getUri().getPath(); - if (path.equals( "/nodes/v2/")) return ResourcesResponse.fromStrings(request.getUri(), "state", "node", "command", "maintenance"); + if (path.equals( "/nodes/v2/")) return ResourcesResponse.fromStrings(request.getUri(), "state", "node", "command", "maintenance", "upgrade"); if (path.equals( "/nodes/v2/node/")) return new NodesResponse(ResponseType.nodeList, request, orchestrator, nodeRepository); if (path.startsWith("/nodes/v2/node/")) return new NodesResponse(ResponseType.singleNode, request, orchestrator, nodeRepository); if (path.equals( "/nodes/v2/state/")) return new NodesResponse(ResponseType.stateList, request, orchestrator, nodeRepository); @@ -99,6 +100,7 @@ public class NodesApiHandler extends LoggingRequestHandler { if (path.startsWith("/nodes/v2/acl/")) return new NodeAclResponse(request, nodeRepository); if (path.equals( "/nodes/v2/command/")) return ResourcesResponse.fromStrings(request.getUri(), "restart", "reboot"); if (path.equals( "/nodes/v2/maintenance/")) return new JobsResponse(maintenance.jobControl()); + if (path.equals( "/nodes/v2/upgrade/")) return new UpgradeResponse(maintenance.infrastructureVersions()); throw new NotFoundException("Nothing at path '" + path + "'"); } @@ -135,10 +137,16 @@ public class NodesApiHandler extends LoggingRequestHandler { private HttpResponse handlePATCH(HttpRequest request) { String path = request.getUri().getPath(); - if ( ! path.startsWith("/nodes/v2/node/")) throw new NotFoundException("Nothing at '" + path + "'"); - Node node = nodeFromRequest(request); - nodeRepository.write(new NodePatcher(nodeFlavors, request.getData(), node, nodeRepository).apply()); - return new MessageResponse("Updated " + node.hostname()); + if (path.startsWith("/nodes/v2/node/")) { + Node node = nodeFromRequest(request); + nodeRepository.write(new NodePatcher(nodeFlavors, request.getData(), node, nodeRepository).apply()); + return new MessageResponse("Updated " + node.hostname()); + } + else if (path.startsWith("/nodes/v2/upgrade/")) { + return setInfrastructureVersion(request); + } + + throw new NotFoundException("Nothing at '" + path + "'"); } private HttpResponse handlePOST(HttpRequest request) { @@ -276,4 +284,18 @@ public class NodesApiHandler extends LoggingRequestHandler { return new MessageResponse((active ? "Re-activated" : "Deactivated" ) + " job '" + jobName + "'"); } + private MessageResponse setInfrastructureVersion(HttpRequest request) { + NodeType nodeType = NodeType.valueOf(lastElement(request.getUri().getPath()).toLowerCase()); + Inspector inspector = toSlime(request.getData()).get(); + + Inspector versionField = inspector.field("version"); + if (!versionField.valid()) + throw new IllegalArgumentException("'version' is missing"); + Version version = Version.fromString(versionField.asString()); + boolean force = inspector.field("force").asBool(); + + maintenance.infrastructureVersions().setTargetVersion(nodeType, version, force); + + return new MessageResponse("Set version for " + nodeType + " to " + version.toFullString()); + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java new file mode 100644 index 00000000000..20a0139a178 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java @@ -0,0 +1,46 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.v2; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Comparator; +import java.util.Map; + +/** + * A response containing infrastructure versions + * + * @author freva + */ +public class UpgradeResponse extends HttpResponse { + + private final InfrastructureVersions infrastructureVersions; + + public UpgradeResponse(InfrastructureVersions infrastructureVersions) { + super(200); + this.infrastructureVersions = infrastructureVersions; + } + + @Override + public void render(OutputStream stream) throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + + Cursor versionsObject = root.setObject("versions"); + infrastructureVersions.getTargetVersions().entrySet().stream() + .sorted(Comparator.comparing(Map.Entry::getKey)) // Sort for stable tests + .forEach(entry -> + versionsObject.setString(entry.getKey().name(), entry.getValue().toFullString())); + + new JsonFormat(true).encode(stream, slime); + } + + @Override + public String getContentType() { return "application/json"; } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java index c5067c0f959..6c43ed18645 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java @@ -16,6 +16,7 @@ public class ContainerConfig { " <component id='com.yahoo.vespa.curator.mock.MockCurator'/>" + " <component id='com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock'/>" + " <component id='com.yahoo.vespa.hosted.provision.testutils.MockDeployer'/>" + + " <component id='com.yahoo.vespa.hosted.provision.testutils.MockProvisioner'/>" + " <component id='com.yahoo.vespa.hosted.provision.testutils.TestHostLivenessTracker'/>" + " <component id='com.yahoo.vespa.hosted.provision.testutils.ServiceMonitorStub'/>" + " <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors'/>" + diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java new file mode 100644 index 00000000000..af0ca7b6b75 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java @@ -0,0 +1,45 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.testutils; + +import com.google.inject.Inject; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostFilter; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.ProvisionLogger; +import com.yahoo.config.provision.Provisioner; +import com.yahoo.transaction.NestedTransaction; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +/** + * @author freva + */ +public class MockProvisioner implements Provisioner { + + @Inject + public MockProvisioner() {} + + @Override + public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { + return Collections.emptyList(); + } + + @Override + public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) { + + } + + @Override + public void remove(NestedTransaction transaction, ApplicationId application) { + + } + + @Override + public void restart(ApplicationId application, HostFilter filter) { + + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java new file mode 100644 index 00000000000..586498619c6 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java @@ -0,0 +1,110 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.Provisioner; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.NodeRepositoryTester; +import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.node.Allocation; +import com.yahoo.vespa.hosted.provision.node.Generation; +import com.yahoo.vespa.service.monitor.application.ConfigServerApplication; + +import java.time.Duration; +import java.util.Optional; + +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author freva + */ +public class InfrastructureProvisionerTest { + + private final NodeRepositoryTester tester = new NodeRepositoryTester(); + + private final Provisioner provisioner = mock(Provisioner.class); + private final NodeRepository nodeRepository = tester.nodeRepository(); + private final InfrastructureVersions infrastructureVersions = mock(InfrastructureVersions.class); + private final InfrastructureProvisioner infrastructureProvisioner = new InfrastructureProvisioner( + provisioner, nodeRepository, infrastructureVersions, Duration.ofDays(99), new JobControl(nodeRepository.database())); + + @Test + public void returns_version_if_usable_nodes_on_old_version() { + Version target = Version.fromString("6.123.456"); + Version oldVersion = Version.fromString("6.122.333"); + when(infrastructureVersions.getTargetVersionFor(eq(NodeType.config))).thenReturn(Optional.of(target)); + + addNode(1, Node.State.failed, Optional.of(oldVersion)); + addNode(2, Node.State.dirty, Optional.empty()); + addNode(3, Node.State.active, Optional.of(oldVersion)); + + assertEquals(Optional.of(target), infrastructureProvisioner.getTargetVersion(NodeType.config)); + } + + @Test + public void returns_version_if_has_usable_nodes_without_version() { + Version target = Version.fromString("6.123.456"); + Version oldVersion = Version.fromString("6.122.333"); + when(infrastructureVersions.getTargetVersionFor(eq(NodeType.config))).thenReturn(Optional.of(target)); + + addNode(1, Node.State.failed, Optional.of(oldVersion)); + addNode(2, Node.State.ready, Optional.empty()); + addNode(3, Node.State.active, Optional.of(target)); + + assertEquals(Optional.of(target), infrastructureProvisioner.getTargetVersion(NodeType.config)); + } + + @Test + public void returns_empty_if_usable_nodes_on_target_version() { + Version target = Version.fromString("6.123.456"); + Version oldVersion = Version.fromString("6.122.333"); + when(infrastructureVersions.getTargetVersionFor(eq(NodeType.config))).thenReturn(Optional.of(target)); + + addNode(1, Node.State.failed, Optional.of(oldVersion)); + addNode(2, Node.State.parked, Optional.of(target)); + addNode(3, Node.State.active, Optional.of(target)); + addNode(4, Node.State.inactive, Optional.of(target)); + addNode(5, Node.State.dirty, Optional.empty()); + + assertEquals(Optional.empty(), infrastructureProvisioner.getTargetVersion(NodeType.config)); + } + + @Test + public void returns_empty_if_no_usable_nodes() { + when(infrastructureVersions.getTargetVersionFor(eq(NodeType.config))).thenReturn(Optional.of(Version.fromString("6.123.456"))); + + // No nodes in node repo + assertEquals(Optional.empty(), infrastructureProvisioner.getTargetVersion(NodeType.config)); + + // Add nodes in non-provisionable states + addNode(1, Node.State.dirty, Optional.empty()); + addNode(2, Node.State.failed, Optional.empty()); + assertEquals(Optional.empty(), infrastructureProvisioner.getTargetVersion(NodeType.config)); + } + + @Test + public void returns_empty_if_target_version_not_set() { + when(infrastructureVersions.getTargetVersionFor(eq(NodeType.config))).thenReturn(Optional.empty()); + assertEquals(Optional.empty(), infrastructureProvisioner.getTargetVersion(NodeType.config)); + } + + private Node addNode(int id, Node.State state, Optional<Version> wantedVespaVersion) { + Node node = tester.addNode("id-" + id, "node-" + id, "default", NodeType.config); + Optional<Node> nodeWithAllocation = wantedVespaVersion.map(version -> { + ConfigServerApplication application = ConfigServerApplication.CONFIG_SERVER_APPLICATION; + ClusterSpec clusterSpec = ClusterSpec.from(application.getClusterType(), application.getClusterId(), ClusterSpec.Group.from(0), version); + ClusterMembership membership = ClusterMembership.from(clusterSpec, 1); + Allocation allocation = new Allocation(application.getApplicationId(), membership, new Generation(0, 0), false); + return node.with(allocation); + }); + return nodeRepository.database().writeTo(state, nodeWithAllocation.orElse(node), Agent.system, Optional.empty()); + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersionsTest.java new file mode 100644 index 00000000000..37b0097b17c --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersionsTest.java @@ -0,0 +1,83 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.NodeRepositoryTester; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import static junit.framework.TestCase.fail; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author freva + */ +public class InfrastructureVersionsTest { + + private final NodeRepositoryTester tester = new NodeRepositoryTester(); + private final InfrastructureVersions infrastructureVersions = + new InfrastructureVersions(tester.nodeRepository().database()); + + private final Version version = Version.fromString("6.123.456"); + + @Test + public void can_only_downgrade_with_force() { + assertTrue(infrastructureVersions.getTargetVersions().isEmpty()); + + assertEquals(Optional.empty(), infrastructureVersions.getTargetVersionFor(NodeType.config)); + infrastructureVersions.setTargetVersion(NodeType.config, version, false); + assertEquals(Optional.of(version), infrastructureVersions.getTargetVersionFor(NodeType.config)); + + // Upgrading to new version without force is fine + Version new_version = Version.fromString("6.123.457"); // version + 1 + infrastructureVersions.setTargetVersion(NodeType.config, new_version, false); + assertEquals(Optional.of(new_version), infrastructureVersions.getTargetVersionFor(NodeType.config)); + + // Downgrading to old version without force fails + try { + infrastructureVersions.setTargetVersion(NodeType.config, version, false); + fail("Should not be able to downgrade without force"); + } catch (IllegalArgumentException ignored) { } + + infrastructureVersions.setTargetVersion(NodeType.config, version, true); + assertEquals(Optional.of(version), infrastructureVersions.getTargetVersionFor(NodeType.config)); + } + + @Test + public void can_only_set_version_on_certain_node_types() { + // We can set version for config + infrastructureVersions.setTargetVersion(NodeType.config, version, false); + + try { + infrastructureVersions.setTargetVersion(NodeType.tenant, version, false); + fail("Should not be able to set version for tenant nodes"); + } catch (IllegalArgumentException ignored) { } + + try { + // Using 'force' does not help, force only applies to version downgrade + infrastructureVersions.setTargetVersion(NodeType.tenant, version, true); + fail("Should not be able to set version for tenant nodes"); + } catch (IllegalArgumentException ignored) { } + } + + @Test + public void can_store_multiple_versions() { + Version version2 = Version.fromString("6.456.123"); + + infrastructureVersions.setTargetVersion(NodeType.config, version, false); + infrastructureVersions.setTargetVersion(NodeType.confighost, version2, false); + infrastructureVersions.setTargetVersion(NodeType.proxyhost, version, false); + + Map<NodeType, Version> expected = new HashMap<>(); + expected.put(NodeType.config, version); + expected.put(NodeType.confighost, version2); + expected.put(NodeType.proxyhost, version); + + assertEquals(expected, infrastructureVersions.getTargetVersions()); + } +}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index 2e69867e9b1..581b82e5fd5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -495,6 +495,44 @@ public class RestApiTest { assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6.json"); } + @Test + public void test_upgrade() throws IOException { + // Initially, no versions are set + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), "{\"versions\":{}}"); + + // Set version for config and confighost + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/config", + Utf8.toBytes("{\"version\": \"6.123.456\"}"), + Request.Method.PATCH), + "{\"message\":\"Set version for config to 6.123.456\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"version\": \"6.123.456\"}"), + Request.Method.PATCH), + "{\"message\":\"Set version for confighost to 6.123.456\"}"); + + // Verify versions are set + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), + "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.456\"}}"); + + // Downgrade without force fails + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"version\": \"6.123.1\"}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot downgrade version without setting 'force'. " + + "Current target version: 6.123.456, attempted to set target version: 6.123.1\"}"); + + // Downgrade with force is OK + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"version\": \"6.123.1\",\"force\":true}"), + Request.Method.PATCH), + "{\"message\":\"Set version for confighost to 6.123.1\"}"); + + // Verify version has been updated + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), + "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.1\"}}"); + } + /** Tests the rendering of each node separately to make it easier to find errors */ @Test public void test_single_node_rendering() throws Exception { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json index 28e28f9678e..99cb9fd91f5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json @@ -34,6 +34,9 @@ "name":"MetricsReporter" }, { + "name":"InfrastructureProvisioner" + }, + { "name":"NodeFailer" } ], diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json index 4224718ab06..86becefb146 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json @@ -11,6 +11,9 @@ }, { "url": "http://localhost:8080/nodes/v2/maintenance/" + }, + { + "url":"http://localhost:8080/nodes/v2/upgrade/" } ] }
\ No newline at end of file diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/HostedVespaApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/HostedVespaApplication.java index 55965705567..d1111bdd5d7 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/HostedVespaApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/HostedVespaApplication.java @@ -19,21 +19,19 @@ public abstract class HostedVespaApplication { private final Capacity capacity; private final ClusterSpec.Type clusterType; private final ClusterSpec.Id clusterId; - private final ClusterSpec.Group clusterGroup; protected HostedVespaApplication(String applicationName, NodeType nodeType, ClusterSpec.Type clusterType, ClusterSpec.Id clusterId) { this(createHostedVespaApplicationId(applicationName), Capacity.fromRequiredNodeType(nodeType), - clusterType, clusterId, ClusterSpec.Group.from(0)); + clusterType, clusterId); } protected HostedVespaApplication(ApplicationId applicationId, Capacity capacity, - ClusterSpec.Type clusterType, ClusterSpec.Id clusterId, ClusterSpec.Group clusterGroup) { + ClusterSpec.Type clusterType, ClusterSpec.Id clusterId) { this.applicationId = applicationId; this.capacity = capacity; this.clusterType = clusterType; this.clusterId = clusterId; - this.clusterGroup = clusterGroup; } public ApplicationId getApplicationId() { @@ -45,7 +43,7 @@ public abstract class HostedVespaApplication { } public ClusterSpec getClusterSpecWithVersion(Version version) { - return ClusterSpec.from(clusterType, clusterId, clusterGroup, version, true); + return ClusterSpec.request(clusterType, clusterId, version, true); } public ClusterSpec.Type getClusterType() { @@ -56,10 +54,6 @@ public abstract class HostedVespaApplication { return clusterId; } - public ClusterSpec.Group getClusterGroup() { - return clusterGroup; - } - public static ApplicationId createHostedVespaApplicationId(String applicationName) { return new ApplicationId.Builder() .tenant(TENANT_NAME) |