diff options
Diffstat (limited to 'node-repository/src/main/java/com/yahoo')
6 files changed, 59 insertions, 53 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java index 87b5719cc19..61589078b69 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java @@ -284,13 +284,13 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { try { Version osVersion = nodeRepository().osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion); List<Integer> provisionIndices = nodeRepository().database().readProvisionIndices(count); - List<Node> hosts = hostProvisioner.provisionHosts(provisionIndices, NodeType.host, nodeResources, - ApplicationId.defaultId(), osVersion, HostSharing.shared, - Optional.empty(), Optional.empty()) - .stream() - .map(ProvisionedHost::generateHost) - .collect(Collectors.toList()); - nodeRepository().nodes().addNodes(hosts, Agent.DynamicProvisioningMaintainer); + List<Node> hosts = new ArrayList<>(); + hostProvisioner.provisionHosts(provisionIndices, NodeType.host, nodeResources, ApplicationId.defaultId(), + osVersion, HostSharing.shared, Optional.empty(), Optional.empty(), + provisionedHosts -> { + hosts.addAll(provisionedHosts.stream().map(ProvisionedHost::generateHost).toList()); + nodeRepository().nodes().addNodes(hosts, Agent.DynamicProvisioningMaintainer); + }); return hosts; } catch (NodeAllocationException | IllegalArgumentException | IllegalStateException e) { throw new NodeAllocationException("Failed to provision " + count + " " + nodeResources + ": " + e.getMessage(), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java index 4aca1cbd056..7da10ce085a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java @@ -46,10 +46,8 @@ public class History { private static ImmutableMap<Event.Type, Event> toImmutableMap(Collection<Event> events) { ImmutableMap.Builder<Event.Type, Event> builder = new ImmutableMap.Builder<>(); - for (Event event : events) { - if (event.type() == Event.Type.requested) continue; // TODO (freva): Remove requested event after 8.70 + for (Event event : events) builder.put(event.type(), event); - } return builder.build(); } @@ -182,8 +180,6 @@ public class History { down, // The active node came up according to the service monitor up, - // The node made a config request, indicating it is live - requested, // The node resources/flavor were changed resized(false), // The node was rebooted diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 24c52f7b2e0..c0275de5798 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -423,7 +423,6 @@ public class NodeSerializer { case "deallocated" : return History.Event.Type.deallocated; case "down" : return History.Event.Type.down; case "up" : return History.Event.Type.up; - case "requested" : return History.Event.Type.requested; case "resized" : return History.Event.Type.resized; case "rebooted" : return History.Event.Type.rebooted; case "osUpgraded" : return History.Event.Type.osUpgraded; @@ -450,7 +449,6 @@ public class NodeSerializer { case deallocated : return "deallocated"; case down : return "down"; case up : return "up"; - case requested: return "requested"; case resized: return "resized"; case rebooted: return "rebooted"; case osUpgraded: return "osUpgraded"; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 35f04683157..e1657ec3358 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -15,8 +15,10 @@ import com.yahoo.vespa.hosted.provision.NodesAndHosts; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner.HostSharing; +import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.function.Consumer; import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; @@ -102,33 +104,35 @@ public class GroupPreparer { NodeAllocation allocation = prepareAllocation(application, cluster, requestedNodes, surplusActiveNodes, indices::next, wantedGroups, allNodesAndHosts); NodeType hostType = allocation.nodeType().hostType(); - if (canProvisionDynamically(hostType)) { + if (canProvisionDynamically(hostType) && allocation.hostDeficit().isPresent()) { HostSharing sharing = hostSharing(requestedNodes, hostType); Version osVersion = nodeRepository.osVersions().targetFor(hostType).orElse(Version.emptyVersion); - List<ProvisionedHost> provisionedHosts = allocation.hostDeficit() - .map(deficit -> hostProvisioner.get().provisionHosts(allocation.provisionIndices(deficit.count()), - hostType, - deficit.resources(), - application, - osVersion, - sharing, - Optional.of(cluster.type()), - requestedNodes.cloudAccount())) - .orElseGet(List::of); - - // At this point we have started provisioning of the hosts, the first priority is to make sure that - // the returned hosts are added to the node-repo so that they are tracked by the provision maintainers - List<Node> hosts = provisionedHosts.stream() - .map(ProvisionedHost::generateHost) - .collect(Collectors.toList()); - nodeRepository.nodes().addNodes(hosts, Agent.application); - - // Offer the nodes on the newly provisioned hosts, this should be enough to cover the deficit - List<NodeCandidate> candidates = provisionedHosts.stream() - .map(host -> NodeCandidate.createNewExclusiveChild(host.generateNode(), - host.generateHost())) - .collect(Collectors.toList()); - allocation.offer(candidates); + NodeAllocation.HostDeficit deficit = allocation.hostDeficit().get(); + + List<Node> hosts = new ArrayList<>(); + Consumer<List<ProvisionedHost>> provisionedHostsConsumer = provisionedHosts -> { + hosts.addAll(provisionedHosts.stream().map(ProvisionedHost::generateHost).toList()); + nodeRepository.nodes().addNodes(hosts, Agent.application); + + // Offer the nodes on the newly provisioned hosts, this should be enough to cover the deficit + List<NodeCandidate> candidates = provisionedHosts.stream() + .map(host -> NodeCandidate.createNewExclusiveChild(host.generateNode(), + host.generateHost())) + .collect(Collectors.toList()); + allocation.offer(candidates); + }; + + try { + hostProvisioner.get().provisionHosts( + allocation.provisionIndices(deficit.count()), hostType, deficit.resources(), application, + osVersion, sharing, Optional.of(cluster.type()), requestedNodes.cloudAccount(), provisionedHostsConsumer); + } catch (NodeAllocationException e) { + // Mark the nodes that were written to ZK in the consumer for deprovisioning. While these hosts do + // not exist, we cannot remove them from ZK here because other nodes may already have been + // allocated on them, so let DynamicProvisioningMaintainer deal with it + hosts.forEach(host -> nodeRepository.nodes().deprovision(host.hostname(), Agent.system, nodeRepository.clock().instant())); + throw e; + } } if (! allocation.fulfilled() && requestedNodes.canFail()) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java index 9b765adca89..1d95cdc09df 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostEvent; +import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; @@ -13,6 +14,7 @@ import com.yahoo.vespa.hosted.provision.Node; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.function.Consumer; /** * A service which supports provisioning container hosts dynamically. @@ -45,16 +47,21 @@ public interface HostProvisioner { * @param sharing puts requirements on sharing or exclusivity of the host to be provisioned. * @param clusterType provision host exclusively for this cluster type * @param cloudAccount the cloud account to use - * @return list of {@link ProvisionedHost} describing the provisioned nodes + * @param provisionedHostConsumer consumer of {@link ProvisionedHost}s describing the provisioned nodes, + * the {@link Node} returned from {@link ProvisionedHost#generateHost()} must be + * written to ZK immediately in case the config server goes down while waiting + * for the provisioning to finish. + * @throws NodeAllocationException if the cloud provider cannot satisfy the request */ - List<ProvisionedHost> provisionHosts(List<Integer> provisionIndices, - NodeType hostType, - NodeResources resources, - ApplicationId applicationId, - Version osVersion, - HostSharing sharing, - Optional<ClusterSpec.Type> clusterType, - Optional<CloudAccount> cloudAccount); + void provisionHosts(List<Integer> provisionIndices, + NodeType hostType, + NodeResources resources, + ApplicationId applicationId, + Version osVersion, + HostSharing sharing, + Optional<ClusterSpec.Type> clusterType, + Optional<CloudAccount> cloudAccount, + Consumer<List<ProvisionedHost>> provisionedHostConsumer) throws NodeAllocationException; /** * Continue provisioning of given list of Nodes. diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java index 3ebaf764115..f9af176fc00 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java @@ -26,6 +26,7 @@ import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -60,10 +61,10 @@ public class MockHostProvisioner implements HostProvisioner { } @Override - public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndices, NodeType hostType, NodeResources resources, - ApplicationId applicationId, Version osVersion, HostSharing sharing, - Optional<ClusterSpec.Type> clusterType, - Optional<CloudAccount> cloudAccount) { + public void provisionHosts(List<Integer> provisionIndices, NodeType hostType, NodeResources resources, + ApplicationId applicationId, Version osVersion, HostSharing sharing, + Optional<ClusterSpec.Type> clusterType, Optional<CloudAccount> cloudAccount, + Consumer<List<ProvisionedHost>> provisionedHostsConsumer) { Flavor hostFlavor = this.hostFlavor.orElseGet(() -> flavors.stream().filter(f -> compatible(f, resources)) .findFirst() .orElseThrow(() -> new NodeAllocationException("No host flavor matches " + resources, true))); @@ -82,7 +83,7 @@ public class MockHostProvisioner implements HostProvisioner { cloudAccount)); } provisionedHosts.addAll(hosts); - return hosts; + provisionedHostsConsumer.accept(hosts); } @Override |