diff options
Diffstat (limited to 'node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning')
28 files changed, 256 insertions, 116 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java index 9adff9f9d7a..9bf7d928138 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index e5599ac3d18..1e9adea4e95 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; @@ -9,6 +9,7 @@ import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeResources.DiskSpeed; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.StringFlag; @@ -90,7 +91,10 @@ public class CapacityPolicies { } public NodeResources specifyFully(NodeResources resources, ClusterSpec clusterSpec, ApplicationId applicationId) { - return resources.withUnspecifiedNumbersFrom(defaultResources(clusterSpec, applicationId)); + NodeResources amended = resources.withUnspecifiedFieldsFrom(defaultResources(clusterSpec, applicationId).with(DiskSpeed.any)); + // TODO jonmv: remove this after all apps are 8.248.8 or above; architecture for admin nodes was not picked up before this. + if (clusterSpec.vespaVersion().isBefore(Version.fromString("8.248.8"))) amended = amended.with(resources.architecture()); + return amended; } private NodeResources defaultResources(ClusterSpec clusterSpec, ApplicationId applicationId) { @@ -149,7 +153,9 @@ public class CapacityPolicies { return Architecture.valueOf(adminClusterNodeArchitecture.with(INSTANCE_ID, instance.serializedForm()).value()); } - /** Returns the resources for the newest version not newer than that requested in the cluster spec. */ + /** + * Returns the resources for the newest version not newer than that requested in the cluster spec. + */ static NodeResources versioned(ClusterSpec spec, Map<Version, NodeResources> resources) { return requireNonNull(new TreeMap<>(resources).floorEntry(spec.vespaVersion()), "no default resources applicable for " + spec + " among: " + resources) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImages.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImages.java index 583045083a9..a8f532a1a9f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImages.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImages.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.DockerImage; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java index b7e7ac7ee4b..0d6a98f50a3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.Flavor; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FatalProvisioningException.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FatalProvisioningException.java index afd105c1388..149279873be 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FatalProvisioningException.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FatalProvisioningException.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FirmwareChecks.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FirmwareChecks.java index ff375f2abe1..80806857897 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FirmwareChecks.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FirmwareChecks.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.lang.CachedSupplier; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java index 2e9cca21052..3153b3f6d10 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.Flavor; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupAssigner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupAssigner.java index 542f2c00f95..e314542a40c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupAssigner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupAssigner.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ClusterSpec; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java index 991bc22402d..dab6e9d1581 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.NodeResources; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostIpConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostIpConfig.java index 225f4827a18..a85e6c326f5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostIpConfig.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostIpConfig.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.vespa.hosted.provision.node.IP; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisionRequest.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisionRequest.java index f7b9c9016b1..ddf59ed3146 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisionRequest.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisionRequest.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java index 3362ec5d3b3..38cbfa7fe5f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.CloudAccount; @@ -22,14 +22,21 @@ public interface HostProvisioner { enum HostSharing { - /** The host must be provisioned exclusively for the applicationId */ + /** The host must be provisioned exclusively for the application ID. */ + provision, + + /** The host must be exclusive to a single application ID */ exclusive, /** The host must be provisioned to be shared with other applications. */ shared, /** The client has no requirements on whether the host must be provisioned exclusively or shared. */ - any + any; + + public boolean isExclusiveAllocation() { + return this == provision || this == exclusive; + } } @@ -43,8 +50,10 @@ public interface HostProvisioner { * written to ZK immediately in case the config server goes down while waiting * for the provisioning to finish. * @throws NodeAllocationException if the cloud provider cannot satisfy the request + * @return a runnable that waits for the provisioning request to finish. It can be run without holding any locks, + * but may fail with an exception that should be propagated to the user initiating prepare() */ - void provisionHosts(HostProvisionRequest request, Predicate<NodeResources> realHostResourcesWithinLimits, Consumer<List<ProvisionedHost>> whenProvisioned) throws NodeAllocationException; + Runnable provisionHosts(HostProvisionRequest request, Predicate<NodeResources> realHostResourcesWithinLimits, Consumer<List<ProvisionedHost>> whenProvisioned) throws NodeAllocationException; /** * Continue provisioning of given list of Nodes. diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java index 90cdf932f17..e474ae6eea3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.Flavor; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java index 1f424a1e1d5..39c14be4d2b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import ai.vespa.http.DomainName; @@ -13,7 +13,6 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InfraDeployer; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Provisioner; -import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions; @@ -60,7 +59,7 @@ public class InfraDeployerImpl implements InfraDeployer { .forEach(api -> { var application = api.getApplicationId(); var deployment = new InfraDeployment(api); - try { + try (var lock = nodeRepository.applications().lockMaintenance(application)) { deployment.activate(); } catch (RuntimeException e) { logger.log(Level.INFO, "Failed to activate " + application, e); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 22909122079..4ee1573269a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index ca170d2af6b..e1be5b48e2d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; @@ -84,9 +84,10 @@ class NodeAllocation { private final NodeRepository nodeRepository; private final Optional<String> requiredHostFlavor; + private final boolean makeExclusive; NodeAllocation(NodeList allNodes, ApplicationId application, ClusterSpec cluster, NodeSpec requested, - Supplier<Integer> nextIndex, NodeRepository nodeRepository) { + Supplier<Integer> nextIndex, NodeRepository nodeRepository, boolean makeExclusive) { this.allNodes = allNodes; this.application = application; this.cluster = cluster; @@ -99,6 +100,7 @@ class NodeAllocation { .with(FetchVector.Dimension.CLUSTER_ID, cluster.id().value()) .value()) .filter(s -> !s.isBlank()); + this.makeExclusive = makeExclusive; } /** @@ -117,21 +119,21 @@ class NodeAllocation { ClusterMembership membership = allocation.membership(); if ( ! allocation.owner().equals(application)) continue; // wrong application if ( ! membership.cluster().satisfies(cluster)) continue; // wrong cluster id/type - if ( candidate.state() == Node.State.active && allocation.removable()) continue; // don't accept; causes removal - if ( candidate.state() == Node.State.active && candidate.wantToFail()) continue; // don't accept; causes failing - if ( indexes.contains(membership.index())) continue; // duplicate index (just to be sure) + if (candidate.state() == Node.State.active && allocation.removable()) continue; // don't accept; causes removal + if (candidate.state() == Node.State.active && candidate.wantToFail()) continue; // don't accept; causes failing + if (indexes.contains(membership.index())) continue; // duplicate index (just to be sure) if (nodeRepository.zone().cloud().allowEnclave() && candidate.parent.isPresent() && ! candidate.parent.get().cloudAccount().equals(requested.cloudAccount())) continue; // wrong account boolean resizeable = requested.considerRetiring() && candidate.isResizable; - if ((! saturated() && hasCompatibleResources(candidate) && requested.acceptable(candidate)) || acceptIncompatible(candidate)) { + if (( ! saturated() && hasCompatibleResources(candidate) && requested.acceptable(candidate)) || acceptIncompatible(candidate)) { candidate = candidate.withNode(); if (candidate.isValid()) acceptNode(candidate, shouldRetire(candidate, candidates), resizeable); } } - else if (! saturated() && hasCompatibleResources(candidate)) { - if (! nodeRepository.nodeResourceLimits().isWithinRealLimits(candidate, application, cluster)) { + else if ( ! saturated() && hasCompatibleResources(candidate)) { + if ( ! nodeRepository.nodeResourceLimits().isWithinRealLimits(candidate, application, cluster)) { ++rejectedDueToInsufficientRealResources; continue; } @@ -139,9 +141,13 @@ class NodeAllocation { ++rejectedDueToClashingParentHost; continue; } - if ( violatesExclusivity(candidate)) { - ++rejectedDueToExclusivity; - continue; + switch (violatesExclusivity(candidate)) { + case PARENT_HOST_NOT_EXCLUSIVE -> candidate = candidate.withExclusiveParent(true); + case NONE -> {} + case YES -> { + ++rejectedDueToExclusivity; + continue; + } } if (candidate.wantToRetire()) { continue; @@ -169,7 +175,7 @@ class NodeAllocation { if (candidate.parent.map(node -> node.status().wantToUpgradeFlavor()).orElse(false)) return Retirement.violatesHostFlavorGeneration; if (candidate.wantToRetire()) return Retirement.hardRequest; if (candidate.preferToRetire() && candidate.replaceableBy(candidates)) return Retirement.softRequest; - if (violatesExclusivity(candidate)) return Retirement.violatesExclusivity; + if (violatesExclusivity(candidate) != NodeCandidate.ExclusivityViolation.NONE) return Retirement.violatesExclusivity; if (requiredHostFlavor.isPresent() && ! candidate.parent.map(node -> node.flavor().name()).equals(requiredHostFlavor)) return Retirement.violatesHostFlavor; if (candidate.violatesSpares) return Retirement.violatesSpares; return Retirement.none; @@ -186,39 +192,15 @@ class NodeAllocation { } private boolean offeredNodeHasParentHostnameAlreadyAccepted(NodeCandidate candidate) { - for (NodeCandidate acceptedNode : nodes.values()) { - if (acceptedNode.parentHostname().isPresent() && candidate.parentHostname().isPresent() && - acceptedNode.parentHostname().get().equals(candidate.parentHostname().get())) { - return true; - } - } - return false; - } - - private boolean violatesExclusivity(NodeCandidate candidate) { if (candidate.parentHostname().isEmpty()) return false; - if (requested.type() != NodeType.tenant) return false; - - // In zones which does not allow host sharing, exclusivity is violated if... - if ( ! nodeRepository.zone().cloud().allowHostSharing()) { - // TODO: Write this in a way that is simple to read - // If either the parent is dedicated to a cluster type different from this cluster - return ! candidate.parent.flatMap(Node::exclusiveToClusterType).map(cluster.type()::equals).orElse(true) || - // or the parent is dedicated to a different application - ! candidate.parent.flatMap(Node::exclusiveToApplicationId).map(application::equals).orElse(true) || - // or this cluster requires exclusivity, but the host is not exclusive - (nodeRepository.exclusiveAllocation(cluster) && candidate.parent.flatMap(Node::exclusiveToApplicationId).isEmpty()); - } + return nodes.values().stream().anyMatch(acceptedNode -> acceptedNode.parentHostname().equals(candidate.parentHostname())); + } - // In zones with shared hosts we require that if either of the nodes on the host requires exclusivity, - // then all the nodes on the host must have the same owner - for (Node nodeOnHost : allNodes.childrenOf(candidate.parentHostname().get())) { - if (nodeOnHost.allocation().isEmpty()) continue; - if (nodeRepository.exclusiveAllocation(cluster) || nodeOnHost.allocation().get().membership().cluster().isExclusive()) { - if ( ! nodeOnHost.allocation().get().owner().equals(application)) return true; - } - } - return false; + private NodeCandidate.ExclusivityViolation violatesExclusivity(NodeCandidate candidate) { + return candidate.violatesExclusivity(cluster, application, + nodeRepository.exclusiveAllocation(cluster), + nodeRepository.exclusiveProvisioning(cluster), + nodeRepository.zone().cloud().allowHostSharing(), allNodes, makeExclusive); } /** @@ -399,6 +381,14 @@ class NodeAllocation { return requested.type(); } + List<Node> parentsRequiredToBeExclusive() { + return nodes.values() + .stream() + .filter(candidate -> candidate.exclusiveParent) + .map(candidate -> candidate.parent.orElseThrow()) + .toList(); + } + List<Node> finalNodes() { GroupAssigner groupAssigner = new GroupAssigner(requested, allNodes, nodeRepository.clock()); Collection<NodeCandidate> finalNodes = groupAssigner.assignTo(nodes.values()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java index 27136811fa1..1547a266e15 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java @@ -1,14 +1,16 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.Nodelike; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.IP; @@ -20,6 +22,8 @@ import java.util.Objects; import java.util.Optional; import java.util.logging.Logger; +import static com.yahoo.collections.Optionals.emptyOrEqual; + /** * A node candidate containing the details required to prioritize it for allocation. This is immutable. * @@ -59,7 +63,11 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat /** This node can be resized to the new NodeResources */ final boolean isResizable; - private NodeCandidate(NodeResources freeParentCapacity, Optional<Node> parent, boolean violatesSpares, boolean exclusiveSwitch, boolean isSurplus, boolean isNew, boolean isResizeable) { + /** The parent host must become exclusive to the implied application */ + final boolean exclusiveParent; + + private NodeCandidate(NodeResources freeParentCapacity, Optional<Node> parent, boolean violatesSpares, boolean exclusiveSwitch, + boolean exclusiveParent, boolean isSurplus, boolean isNew, boolean isResizeable) { if (isResizeable && isNew) throw new IllegalArgumentException("A new node cannot be resizable"); @@ -67,6 +75,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat this.parent = parent; this.violatesSpares = violatesSpares; this.exclusiveSwitch = exclusiveSwitch; + this.exclusiveParent = exclusiveParent; this.isSurplus = isSurplus; this.isNew = isNew; this.isResizable = isResizeable; @@ -95,6 +104,8 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat /** Returns a copy of this with exclusive switch set to given value */ public abstract NodeCandidate withExclusiveSwitch(boolean exclusiveSwitch); + public abstract NodeCandidate withExclusiveParent(boolean exclusiveParent); + /** * Returns the node instance of this candidate, allocating it if necessary. * @@ -224,7 +235,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat /** Returns a copy of this with node set to given value */ NodeCandidate withNode(Node node, boolean retiredNow) { - return new ConcreteNodeCandidate(node, retiredNow, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizable); + return new ConcreteNodeCandidate(node, retiredNow, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, exclusiveParent, isSurplus, isNew, isResizable); } /** Returns the switch priority, based on switch exclusivity, of this compared to other */ @@ -267,7 +278,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat boolean isSurplus, boolean isNew, boolean isResizeable) { - return new ConcreteNodeCandidate(node, false, freeParentCapacity, Optional.of(parent), violatesSpares, true, isSurplus, isNew, isResizeable); + return new ConcreteNodeCandidate(node, false, freeParentCapacity, Optional.of(parent), violatesSpares, true, false, isSurplus, isNew, isResizeable); } public static NodeCandidate createNewChild(NodeResources resources, @@ -276,15 +287,15 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat boolean violatesSpares, LockedNodeList allNodes, IP.Allocation.Context ipAllocationContext) { - return new VirtualNodeCandidate(resources, freeParentCapacity, parent, violatesSpares, true, allNodes, ipAllocationContext); + return new VirtualNodeCandidate(resources, freeParentCapacity, parent, violatesSpares, true, false, allNodes, ipAllocationContext); } public static NodeCandidate createNewExclusiveChild(Node node, Node parent) { - return new ConcreteNodeCandidate(node, false, node.resources(), Optional.of(parent), false, true, false, true, false); + return new ConcreteNodeCandidate(node, false, node.resources(), Optional.of(parent), false, true, false, false, true, false); } public static NodeCandidate createStandalone(Node node, boolean isSurplus, boolean isNew) { - return new ConcreteNodeCandidate(node, false, node.resources(), Optional.empty(), false, true, isSurplus, isNew, false); + return new ConcreteNodeCandidate(node, false, node.resources(), Optional.empty(), false, true, false, isSurplus, isNew, false); } /** A candidate backed by a node */ @@ -296,9 +307,9 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat ConcreteNodeCandidate(Node node, boolean retiredNow, NodeResources freeParentCapacity, Optional<Node> parent, - boolean violatesSpares, boolean exclusiveSwitch, + boolean violatesSpares, boolean exclusiveSwitch, boolean exclusiveParent, boolean isSurplus, boolean isNew, boolean isResizeable) { - super(freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizeable); + super(freeParentCapacity, parent, violatesSpares, exclusiveSwitch, exclusiveParent, isSurplus, isNew, isResizeable); this.retiredNow = retiredNow; this.node = Objects.requireNonNull(node, "Node cannot be null"); } @@ -336,7 +347,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat @Override public NodeCandidate allocate(ApplicationId owner, ClusterMembership membership, NodeResources requestedResources, Instant at) { return new ConcreteNodeCandidate(node.allocate(owner, membership, requestedResources, at), retiredNow, - freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizable); + freeParentCapacity, parent, violatesSpares, exclusiveSwitch, exclusiveParent, isSurplus, isNew, isResizable); } /** Called when the node described by this candidate must be created */ @@ -346,7 +357,13 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat @Override public NodeCandidate withExclusiveSwitch(boolean exclusiveSwitch) { return new ConcreteNodeCandidate(node, retiredNow, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, - isSurplus, isNew, isResizable); + exclusiveParent, isSurplus, isNew, isResizable); + } + + @Override + public NodeCandidate withExclusiveParent(boolean exclusiveParent) { + return new ConcreteNodeCandidate(node, retiredNow, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, + exclusiveParent, isSurplus, isNew, isResizable); } @Override @@ -387,9 +404,10 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat Node parent, boolean violatesSpares, boolean exclusiveSwitch, + boolean exclusiveParent, LockedNodeList allNodes, IP.Allocation.Context ipAllocationContext) { - super(freeParentCapacity, Optional.of(parent), violatesSpares, exclusiveSwitch, false, true, false); + super(freeParentCapacity, Optional.of(parent), violatesSpares, exclusiveSwitch, exclusiveParent, false, true, false); this.resources = resources; this.allNodes = allNodes; this.ipAllocationContext = ipAllocationContext; @@ -449,13 +467,18 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat NodeType.tenant) .cloudAccount(parent.get().cloudAccount()) .build(); - return new ConcreteNodeCandidate(node, false, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizable); + return new ConcreteNodeCandidate(node, false, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, exclusiveParent, isSurplus, isNew, isResizable); } @Override public NodeCandidate withExclusiveSwitch(boolean exclusiveSwitch) { - return new VirtualNodeCandidate(resources, freeParentCapacity, parent.get(), violatesSpares, exclusiveSwitch, allNodes, ipAllocationContext); + return new VirtualNodeCandidate(resources, freeParentCapacity, parent.get(), violatesSpares, exclusiveSwitch, exclusiveParent, allNodes, ipAllocationContext); + } + + @Override + public NodeCandidate withExclusiveParent(boolean exclusiveParent) { + return new VirtualNodeCandidate(resources, freeParentCapacity, parent.get(), violatesSpares, exclusiveSwitch, exclusiveParent, allNodes, ipAllocationContext); } @Override @@ -492,7 +515,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat private InvalidNodeCandidate(NodeResources resources, NodeResources freeParentCapacity, Node parent, String invalidReason) { - super(freeParentCapacity, Optional.of(parent), false, false, false, true, false); + super(freeParentCapacity, Optional.of(parent), false, false, false, false, true, false); this.resources = resources; this.invalidReason = invalidReason; } @@ -540,6 +563,11 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat } @Override + public NodeCandidate withExclusiveParent(boolean exclusiveParent) { + return this; + } + + @Override public Node toNode() { throw new IllegalStateException("Candidate node on " + parent.get() + " is invalid: " + invalidReason); } @@ -559,4 +587,61 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat } + public enum ExclusivityViolation { + NONE, YES, + + /** No violation IF AND ONLY IF the parent host's exclusiveToApplicationId is set to this application. */ + PARENT_HOST_NOT_EXCLUSIVE + } + + public ExclusivityViolation violatesExclusivity(ClusterSpec cluster, ApplicationId application, + boolean exclusiveAllocation, boolean exclusiveProvisioning, + boolean hostSharing, NodeList allNodes, boolean makeExclusive) { + if (parentHostname().isEmpty()) return ExclusivityViolation.NONE; + if (type() != NodeType.tenant) return ExclusivityViolation.NONE; + + if (hostSharing) { + // In zones with shared hosts we require that if any node on the host requires exclusivity, + // then all the nodes on the host must have the same owner. + for (Node nodeOnHost : allNodes.childrenOf(parentHostname().get())) { + if (nodeOnHost.allocation().isEmpty()) continue; + if (exclusiveAllocation || nodeOnHost.allocation().get().membership().cluster().isExclusive()) { + if ( ! nodeOnHost.allocation().get().owner().equals(application)) return ExclusivityViolation.YES; + } + } + } else { + // the parent is exclusive to another cluster type + if ( ! emptyOrEqual(parent.flatMap(Node::exclusiveToClusterType), cluster.type())) + return ExclusivityViolation.YES; + + // the parent is provisioned for another application + if ( ! emptyOrEqual(parent.flatMap(Node::provisionedForApplicationId), application)) + return ExclusivityViolation.YES; + + // this cluster requires a parent that was provisioned for this application + if (exclusiveProvisioning && parent.flatMap(Node::provisionedForApplicationId).isEmpty()) + return ExclusivityViolation.YES; + + // the parent is exclusive to another application + if ( ! emptyOrEqual(parent.flatMap(Node::exclusiveToApplicationId), application)) + return ExclusivityViolation.YES; + + // this cluster requires exclusivity, but the parent is not exclusive + if (exclusiveAllocation && parent.flatMap(Node::exclusiveToApplicationId).isEmpty()) + return canMakeHostExclusive(makeExclusive, type(), hostSharing) ? + ExclusivityViolation.PARENT_HOST_NOT_EXCLUSIVE : + ExclusivityViolation.YES; + } + + return ExclusivityViolation.NONE; + } + + /** + * Whether it is allowed to take a host not exclusive to anyone, and make it exclusive to an application. + * Returns false if {@code makeExclusive} is false, which can be used to guard this feature. + */ + public static boolean canMakeHostExclusive(boolean makeExclusive, NodeType type, boolean allowHostSharing) { + return makeExclusive && type == NodeType.tenant && !allowHostSharing; + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeIndices.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeIndices.java index 1ffd54872d3..49eaedaa4ec 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeIndices.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeIndices.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ClusterSpec; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index fe9f3443299..b92d6fb6d18 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; @@ -9,7 +9,6 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.Nodes; -import com.yahoo.vespa.hosted.provision.persistence.NameResolver; import java.util.ArrayList; import java.util.Collections; @@ -40,14 +39,17 @@ public class NodePrioritizer { private final IP.Allocation.Context ipAllocationContext; private final Nodes nodes; private final boolean dynamicProvisioning; + private final boolean allowHostSharing; + private final boolean exclusiveAllocation; + private final boolean makeExclusive; private final boolean canAllocateToSpareHosts; private final boolean topologyChange; private final int currentClusterSize; private final Set<Node> spareHosts; public NodePrioritizer(LockedNodeList allNodes, ApplicationId application, ClusterSpec clusterSpec, NodeSpec nodeSpec, - boolean dynamicProvisioning, IP.Allocation.Context ipAllocationContext, Nodes nodes, - HostResourcesCalculator hostResourcesCalculator, int spareCount) { + boolean dynamicProvisioning, boolean allowHostSharing, IP.Allocation.Context ipAllocationContext, Nodes nodes, + HostResourcesCalculator hostResourcesCalculator, int spareCount, boolean exclusiveAllocation, boolean makeExclusive) { this.allNodes = allNodes; this.calculator = hostResourcesCalculator; this.capacity = new HostCapacity(this.allNodes, hostResourcesCalculator); @@ -55,6 +57,9 @@ public class NodePrioritizer { this.clusterSpec = clusterSpec; this.application = application; this.dynamicProvisioning = dynamicProvisioning; + this.allowHostSharing = allowHostSharing; + this.exclusiveAllocation = exclusiveAllocation; + this.makeExclusive = makeExclusive; this.spareHosts = dynamicProvisioning ? capacity.findSpareHostsInDynamicallyProvisionedZones(this.allNodes.asList()) : capacity.findSpareHosts(this.allNodes.asList(), spareCount); @@ -122,7 +127,13 @@ public class NodePrioritizer { if (nodes.suspended(host)) continue; // Hosts that are suspended may be down for some time, e.g. for OS upgrade if (host.reservedTo().isPresent() && !host.reservedTo().get().equals(application.tenant())) continue; if (host.reservedTo().isPresent() && application.instance().isTester()) continue; - if (host.exclusiveToApplicationId().isPresent() && ! fitsPerfectly(host)) continue; + if (makeExclusive) { + if ( ! allowHostSharing && exclusiveAllocation && ! fitsPerfectly(host)) continue; + } else { + if (host.exclusiveToApplicationId().isPresent() && ! fitsPerfectly(host)) continue; + } + if ( ! host.provisionedForApplicationId().map(application::equals).orElse(true)) continue; + if ( ! host.exclusiveToApplicationId().map(application::equals).orElse(true)) continue; if ( ! host.exclusiveToClusterType().map(clusterSpec.type()::equals).orElse(true)) continue; if (spareHosts.contains(host) && !canAllocateToSpareHosts) continue; if ( ! capacity.hasCapacity(host, requested.resources().get())) continue; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 132cd0e6d67..e2f1b7358cf 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.annotation.Inject; @@ -12,10 +12,9 @@ import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.ProvisionLock; +import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; -import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; import com.yahoo.transaction.Mutex; @@ -145,7 +144,10 @@ public class NodeRepositoryProvisioner implements Provisioner { @Override public void restart(ApplicationId application, HostFilter filter) { - nodeRepository.nodes().restartActive(ApplicationFilter.from(application).and(NodeHostFilter.from(filter))); + List<Node> updated = nodeRepository.nodes().restartActive(ApplicationFilter.from(application).and(NodeHostFilter.from(filter))); + if (updated.isEmpty()) { + throw new IllegalArgumentException("No matching nodes found"); + } } @Override @@ -155,8 +157,8 @@ public class NodeRepositoryProvisioner implements Provisioner { } @Override - public ProvisionLock lock(ApplicationId application) { - return new ProvisionLock(application, nodeRepository.applications().lock(application)); + public ApplicationMutex lock(ApplicationId application) { + return new ApplicationMutex(application, nodeRepository.applications().lock(application)); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceComparator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceComparator.java index 2038390abcf..11a5ee39a3d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceComparator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceComparator.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.NodeResources; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java index ab222c45252..cd331cbd5fa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java index 77f37cadc0b..cb50f6dff2b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.CloudAccount; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index 67dae48cff7..270fda8314e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; @@ -8,9 +8,13 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.jdisc.Metric; import com.yahoo.text.internal.SnippetGenerator; import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.applicationmodel.InfrastructureApplication; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; @@ -18,6 +22,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner.HostSharing; +import com.yahoo.yolean.Exceptions; import java.util.LinkedHashSet; import java.util.List; @@ -44,12 +49,14 @@ public class Preparer { private final Optional<HostProvisioner> hostProvisioner; private final Optional<LoadBalancerProvisioner> loadBalancerProvisioner; private final ProvisioningThrottler throttler; + private final BooleanFlag makeExclusiveFlag; public Preparer(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner, Optional<LoadBalancerProvisioner> loadBalancerProvisioner, Metric metric) { this.nodeRepository = nodeRepository; this.hostProvisioner = hostProvisioner; this.loadBalancerProvisioner = loadBalancerProvisioner; this.throttler = new ProvisioningThrottler(nodeRepository, metric); + this.makeExclusiveFlag = Flags.MAKE_EXCLUSIVE.bindTo(nodeRepository.flagSource()); } /** @@ -69,11 +76,12 @@ public class Preparer { loadBalancerProvisioner.ifPresent(provisioner -> provisioner.prepare(application, cluster, requested)); + boolean makeExclusive = makeExclusiveFlag.value(); // Try preparing in memory without global unallocated lock. Most of the time there should be no changes, // and we can return nodes previously allocated. LockedNodeList allNodes = nodeRepository.nodes().list(PROBE_LOCK); NodeIndices indices = new NodeIndices(cluster.id(), allNodes); - NodeAllocation probeAllocation = prepareAllocation(application, cluster, requested, indices::probeNext, allNodes); + NodeAllocation probeAllocation = prepareAllocation(application, cluster, requested, indices::probeNext, allNodes, makeExclusive); if (probeAllocation.fulfilledAndNoChanges()) { List<Node> acceptedNodes = probeAllocation.finalNodes(); indices.commitProbe(); @@ -81,16 +89,25 @@ public class Preparer { } else { // There were some changes, so re-do the allocation with locks indices.resetProbe(); - return prepareWithLocks(application, cluster, requested, indices); + return prepareWithLocks(application, cluster, requested, indices, makeExclusive); } } + private ApplicationMutex parentLockOrNull(boolean makeExclusive, NodeType type) { + return NodeCandidate.canMakeHostExclusive(makeExclusive, type, nodeRepository.zone().cloud().allowHostSharing()) ? + nodeRepository.applications().lock(InfrastructureApplication.withNodeType(type.parentNodeType()).id()) : + null; + } + /// Note that this will write to the node repo. - private List<Node> prepareWithLocks(ApplicationId application, ClusterSpec cluster, NodeSpec requested, NodeIndices indices) { + private List<Node> prepareWithLocks(ApplicationId application, ClusterSpec cluster, NodeSpec requested, NodeIndices indices, boolean makeExclusive) { + Runnable waiter = null; + List<Node> acceptedNodes; try (Mutex lock = nodeRepository.applications().lock(application); + ApplicationMutex parentLockOrNull = parentLockOrNull(makeExclusive, requested.type()); Mutex allocationLock = nodeRepository.nodes().lockUnallocated()) { LockedNodeList allNodes = nodeRepository.nodes().list(allocationLock); - NodeAllocation allocation = prepareAllocation(application, cluster, requested, indices::next, allNodes); + NodeAllocation allocation = prepareAllocation(application, cluster, requested, indices::next, allNodes, makeExclusive); NodeType hostType = allocation.nodeType().hostType(); if (canProvisionDynamically(hostType) && allocation.hostDeficit().isPresent()) { HostSharing sharing = hostSharing(cluster, hostType); @@ -127,12 +144,13 @@ public class Preparer { requested.cloudAccount(), deficit.dueToFlavorUpgrade()); Predicate<NodeResources> realHostResourcesWithinLimits = resources -> nodeRepository.nodeResourceLimits().isWithinRealLimits(resources, application, cluster); - hostProvisioner.get().provisionHosts(request, realHostResourcesWithinLimits, whenProvisioned); + waiter = hostProvisioner.get().provisionHosts(request, realHostResourcesWithinLimits, whenProvisioned); } catch (NodeAllocationException e) { // Mark the nodes that were written to ZK in the consumer for deprovisioning. While these hosts do // not exist, we cannot remove them from ZK here because other nodes may already have been // allocated on them, so let HostDeprovisioner deal with it - hosts.forEach(host -> nodeRepository.nodes().deprovision(host.hostname(), Agent.system, nodeRepository.clock().instant())); + hosts.forEach(host -> nodeRepository.nodes().parkRecursively(host.hostname(), Agent.system, true, + "Failed to provision: " + Exceptions.toMessageString(e))); throw e; } } else if (allocation.hostDeficit().isPresent() && requested.canFail() && @@ -140,7 +158,7 @@ public class Preparer { // Non-dynamically provisioned zone with a deficit because we just now retired some nodes. // Try again, but without retiring indices.resetProbe(); - List<Node> accepted = prepareWithLocks(application, cluster, cns.withoutRetiring(), indices); + List<Node> accepted = prepareWithLocks(application, cluster, cns.withoutRetiring(), indices, makeExclusive); log.warning("Prepared " + application + " " + cluster.id() + " without retirement due to lack of capacity"); return accepted; } @@ -150,7 +168,12 @@ public class Preparer { allocation.allocationFailureDetails(), true); // Carry out and return allocation - List<Node> acceptedNodes = allocation.finalNodes(); + if (parentLockOrNull != null) { + List<Node> exclusiveParents = allocation.parentsRequiredToBeExclusive(); + nodeRepository.nodes().setExclusiveToApplicationId(exclusiveParents, parentLockOrNull); + // TODO: also update tags + } + acceptedNodes = allocation.finalNodes(); nodeRepository.nodes().reserve(allocation.reservableNodes()); nodeRepository.nodes().addReservedNodes(new LockedNodeList(allocation.newNodes(), allocationLock)); @@ -160,14 +183,16 @@ public class Preparer { .filter(node -> node.parentHostname().isEmpty() || activeHosts.parentOf(node).isPresent()) .toList(); } - return acceptedNodes; } + + if (waiter != null) waiter.run(); + return acceptedNodes; } private NodeAllocation prepareAllocation(ApplicationId application, ClusterSpec cluster, NodeSpec requested, - Supplier<Integer> nextIndex, LockedNodeList allNodes) { + Supplier<Integer> nextIndex, LockedNodeList allNodes, boolean makeExclusive) { validateAccount(requested.cloudAccount(), application, allNodes); - NodeAllocation allocation = new NodeAllocation(allNodes, application, cluster, requested, nextIndex, nodeRepository); + NodeAllocation allocation = new NodeAllocation(allNodes, application, cluster, requested, nextIndex, nodeRepository, makeExclusive); var allocationContext = IP.Allocation.Context.from(nodeRepository.zone().cloud().name(), requested.cloudAccount().isExclave(nodeRepository.zone()), nodeRepository.nameResolver()); @@ -176,10 +201,13 @@ public class Preparer { cluster, requested, nodeRepository.zone().cloud().dynamicProvisioning(), + nodeRepository.zone().cloud().allowHostSharing(), allocationContext, nodeRepository.nodes(), nodeRepository.resourcesCalculator(), - nodeRepository.spareCount()); + nodeRepository.spareCount(), + nodeRepository.exclusiveAllocation(cluster), + makeExclusive); allocation.offer(prioritizer.collect()); return allocation; } @@ -208,7 +236,9 @@ public class Preparer { private HostSharing hostSharing(ClusterSpec cluster, NodeType hostType) { if ( hostType.isSharable()) - return nodeRepository.exclusiveAllocation(cluster) ? HostSharing.exclusive : HostSharing.any; + return nodeRepository.exclusiveProvisioning(cluster) ? HostSharing.provision : + nodeRepository.exclusiveAllocation(cluster) ? HostSharing.exclusive : + HostSharing.any; else return HostSharing.any; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java index 65039aaca77..601a4008110 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.EndpointsChecker.HealthCheckerProvider; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java index 460b1bd23a0..8a84cfef09a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; @@ -30,6 +30,7 @@ public class ProvisionedHost { private final String hostHostname; private final Flavor hostFlavor; private final NodeType hostType; + private final Optional<ApplicationId> provisionedForApplicationId; private final Optional<ApplicationId> exclusiveToApplicationId; private final Optional<ClusterSpec.Type> exclusiveToClusterType; private final List<HostName> nodeHostnames; @@ -38,7 +39,9 @@ public class ProvisionedHost { private final CloudAccount cloudAccount; public ProvisionedHost(String id, String hostHostname, Flavor hostFlavor, NodeType hostType, - Optional<ApplicationId> exclusiveToApplicationId, Optional<ClusterSpec.Type> exclusiveToClusterType, + Optional<ApplicationId> provisionedForApplicationId, + Optional<ApplicationId> exclusiveToApplicationId, + Optional<ClusterSpec.Type> exclusiveToClusterType, List<HostName> nodeHostnames, NodeResources nodeResources, Version osVersion, CloudAccount cloudAccount) { if (!hostType.isHost()) throw new IllegalArgumentException(hostType + " is not a host"); @@ -46,6 +49,7 @@ public class ProvisionedHost { this.hostHostname = Objects.requireNonNull(hostHostname, "Host hostname must be set"); this.hostFlavor = Objects.requireNonNull(hostFlavor, "Host flavor must be set"); this.hostType = Objects.requireNonNull(hostType, "Host type must be set"); + this.provisionedForApplicationId = Objects.requireNonNull(provisionedForApplicationId, "provisionedForApplicationId must be set"); this.exclusiveToApplicationId = Objects.requireNonNull(exclusiveToApplicationId, "exclusiveToApplicationId must be set"); this.exclusiveToClusterType = Objects.requireNonNull(exclusiveToClusterType, "exclusiveToClusterType must be set"); this.nodeHostnames = validateNodeAddresses(nodeHostnames); @@ -67,6 +71,7 @@ public class ProvisionedHost { Node.Builder builder = Node.create(id, IP.Config.of(List.of(), List.of(), nodeHostnames), hostHostname, hostFlavor, hostType) .status(Status.initial().withOsVersion(OsVersion.EMPTY.withCurrent(Optional.of(osVersion)))) .cloudAccount(cloudAccount); + provisionedForApplicationId.ifPresent(builder::provisionedForApplicationId); exclusiveToApplicationId.ifPresent(builder::exclusiveToApplicationId); exclusiveToClusterType.ifPresent(builder::exclusiveToClusterType); if ( ! hostTTL.isZero()) builder.hostTTL(hostTTL); @@ -84,6 +89,7 @@ public class ProvisionedHost { public String hostHostname() { return hostHostname; } public Flavor hostFlavor() { return hostFlavor; } public NodeType hostType() { return hostType; } + public Optional<ApplicationId> provisionedForApplicationId() { return provisionedForApplicationId; } public Optional<ApplicationId> exclusiveToApplicationId() { return exclusiveToApplicationId; } public Optional<ClusterSpec.Type> exclusiveToClusterType() { return exclusiveToClusterType; } public List<HostName> nodeHostnames() { return nodeHostnames; } @@ -102,6 +108,7 @@ public class ProvisionedHost { hostHostname.equals(that.hostHostname) && hostFlavor.equals(that.hostFlavor) && hostType == that.hostType && + provisionedForApplicationId.equals(that.provisionedForApplicationId) && exclusiveToApplicationId.equals(that.exclusiveToApplicationId) && exclusiveToClusterType.equals(that.exclusiveToClusterType) && nodeHostnames.equals(that.nodeHostnames) && @@ -112,7 +119,7 @@ public class ProvisionedHost { @Override public int hashCode() { - return Objects.hash(id, hostHostname, hostFlavor, hostType, exclusiveToApplicationId, exclusiveToClusterType, nodeHostnames, nodeResources, osVersion, cloudAccount); + return Objects.hash(id, hostHostname, hostFlavor, hostType, provisionedForApplicationId, exclusiveToApplicationId, exclusiveToClusterType, nodeHostnames, nodeResources, osVersion, cloudAccount); } @Override @@ -122,9 +129,10 @@ public class ProvisionedHost { ", hostHostname='" + hostHostname + '\'' + ", hostFlavor=" + hostFlavor + ", hostType=" + hostType + + ", provisionedForApplicationId=" + provisionedForApplicationId + ", exclusiveToApplicationId=" + exclusiveToApplicationId + ", exclusiveToClusterType=" + exclusiveToClusterType + - ", nodeAddresses=" + nodeHostnames + + ", nodeHostnames=" + nodeHostnames + ", nodeResources=" + nodeResources + ", osVersion=" + osVersion + ", cloudAccount=" + cloudAccount + diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottler.java index 1cb7daae33f..fa5f1ff86f0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottler.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import ai.vespa.metrics.ConfigServerMetrics; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/package-info.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/package-info.java index 561f0846d14..5657543effa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/package-info.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/package-info.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage package com.yahoo.vespa.hosted.provision.provisioning; |