diff options
Diffstat (limited to 'node-repository/src/main/java/com/yahoo/vespa/hosted')
191 files changed, 775 insertions, 438 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/LockedNodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/LockedNodeList.java index e760e36f90b..7282867bc56 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/LockedNodeList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/LockedNodeList.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision; import com.yahoo.transaction.Mutex; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NoSuchNodeException.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NoSuchNodeException.java index 3b10f6c166b..043ce60cc89 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NoSuchNodeException.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NoSuchNodeException.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision; /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index d5e891a33c7..567a5c03f43 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision; import com.yahoo.component.Version; @@ -56,6 +56,7 @@ public final class Node implements Nodelike { private final Optional<String> modelName; private final Optional<TenantName> reservedTo; private final Optional<ApplicationId> exclusiveToApplicationId; + private final Optional<ApplicationId> provisionedForApplicationId; private final Optional<Duration> hostTTL; private final Optional<Instant> hostEmptyAt; private final Optional<ClusterSpec.Type> exclusiveToClusterType; @@ -93,9 +94,9 @@ public final class Node implements Nodelike { public Node(String id, Optional<String> extraId, IP.Config ipConfig, String hostname, Optional<String> parentHostname, Flavor flavor, Status status, State state, Optional<Allocation> allocation, History history, NodeType type, Reports reports, Optional<String> modelName, Optional<TenantName> reservedTo, - Optional<ApplicationId> exclusiveToApplicationId, Optional<Duration> hostTTL, Optional<Instant> hostEmptyAt, - Optional<ClusterSpec.Type> exclusiveToClusterType, Optional<String> switchHostname, - List<TrustStoreItem> trustStoreItems, CloudAccount cloudAccount, + Optional<ApplicationId> exclusiveToApplicationId, Optional<ApplicationId> provisionedForApplicationId, + Optional<Duration> hostTTL, Optional<Instant> hostEmptyAt, Optional<ClusterSpec.Type> exclusiveToClusterType, + Optional<String> switchHostname, List<TrustStoreItem> trustStoreItems, CloudAccount cloudAccount, Optional<WireguardKeyWithTimestamp> wireguardPubKey) { this.id = Objects.requireNonNull(id, "A node must have an ID"); this.extraId = Objects.requireNonNull(extraId, "Extra ID cannot be null"); @@ -112,6 +113,7 @@ public final class Node implements Nodelike { this.modelName = Objects.requireNonNull(modelName, "A null modelName is not permitted"); this.reservedTo = Objects.requireNonNull(reservedTo, "reservedTo cannot be null"); this.exclusiveToApplicationId = Objects.requireNonNull(exclusiveToApplicationId, "exclusiveToApplicationId cannot be null"); + this.provisionedForApplicationId = Objects.requireNonNull(provisionedForApplicationId, "provisionedForApplicationId cannot be null"); this.hostTTL = Objects.requireNonNull(hostTTL, "hostTTL cannot be null"); this.hostEmptyAt = Objects.requireNonNull(hostEmptyAt, "hostEmptyAt cannot be null"); this.exclusiveToClusterType = Objects.requireNonNull(exclusiveToClusterType, "exclusiveToClusterType cannot be null"); @@ -141,6 +143,9 @@ public final class Node implements Nodelike { if (type != NodeType.host && exclusiveToApplicationId.isPresent()) throw new IllegalArgumentException("Only tenant hosts can be exclusive to an application"); + if (provisionedForApplicationId.isPresent() && ! exclusiveToApplicationId.equals(provisionedForApplicationId)) + throw new IllegalArgumentException("exclusiveToApplicationId must be the same as provisionedForApplicationId when this is set"); + if (type != NodeType.host && hostTTL.isPresent()) throw new IllegalArgumentException("Only tenant hosts can have a TTL"); @@ -221,12 +226,21 @@ public final class Node implements Nodelike { /** * Returns the application this host is exclusive to, if any. Only tenant hosts can be exclusive to an application. - * If this is set, resources on this host cannot be allocated to any other application. This is set during - * provisioning and applies for the entire lifetime of the host + * If this is set, resources on this host cannot be allocated to any other application. Additionally, the host will + * not be reused once its allocated containers are deleted, i.e., this property can only be set <em>once</em> per host. */ public Optional<ApplicationId> exclusiveToApplicationId() { return exclusiveToApplicationId; } /** + * Returns the application this host was provisioned specifically for, if any. Only tenant hosts can be exclusive + * to an application. This property, when set, also implies {@link #exclusiveToApplicationId()}. + * This is set during provisioning and applies for the entire lifetime of the host. Provisioning a host specifically + * for an application allows access to application-specific resources, through integration with cloud providers' + * provisioning-with-secrets mechanisms. + */ + public Optional<ApplicationId> provisionedForApplicationId() { return provisionedForApplicationId; } + + /** * Returns the additional time to live of tenant host, in a dynamically provisioned zone, after all its child * nodes are removed, before being deprovisioned, if any. * This is set during provisioning and applies for the entire lifetime of the host. @@ -359,14 +373,14 @@ public final class Node implements Nodelike { /** Returns a node with the status assigned to the given value */ public Node with(Status status) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, - reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a node with the type assigned to the given value */ public Node with(NodeType type) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, - reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } @@ -375,35 +389,35 @@ public final class Node implements Nodelike { if (flavor.equals(this.flavor)) return this; History updateHistory = history.with(new History.Event(History.Event.Type.resized, agent, instant)); return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, updateHistory, type, - reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this with the reboot generation set to generation */ public Node withReboot(Generation generation) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status.withReboot(generation), state, allocation, - history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + history, type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this with given id set */ public Node withId(String id) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, - history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + history, type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this with model name set to given value */ public Node withModelName(String modelName) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, Optional.of(modelName), reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, Optional.of(modelName), reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this with model name cleared */ public Node withoutModelName() { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, Optional.empty(), reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, Optional.empty(), reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } @@ -445,21 +459,21 @@ public final class Node implements Nodelike { */ public Node with(Allocation allocation) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, Optional.of(allocation), history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this node with IP config set to the given value. */ public Node with(IP.Config ipConfig) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this node with the parent hostname assigned to the given value. */ public Node withParentHostname(String parentHostname) { return new Node(id, extraId, ipConfig, hostname, Optional.of(parentHostname), flavor, status, state, allocation, - history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + history, type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } @@ -467,50 +481,56 @@ public final class Node implements Nodelike { if (type != NodeType.host) throw new IllegalArgumentException("Only host nodes can be reserved, " + hostname + " has type " + type); return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, Optional.of(tenant), exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, modelName, Optional.of(tenant), exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this node which is not reserved to a tenant */ public Node withoutReservedTo() { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, Optional.empty(), exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, modelName, Optional.empty(), exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withExclusiveToApplicationId(ApplicationId exclusiveTo) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, Optional.ofNullable(exclusiveTo), hostTTL, hostEmptyAt, + type, reports, modelName, reservedTo, Optional.ofNullable(exclusiveTo), provisionedForApplicationId, hostTTL, hostEmptyAt, + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); + } + + public Node withProvisionedForApplicationId(ApplicationId provisionedFor) { + return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, + type, reports, modelName, reservedTo, Optional.ofNullable(provisionedFor), Optional.ofNullable(provisionedFor), hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withExtraId(Optional<String> extraId) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withHostTTL(Duration hostTTL) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, Optional.ofNullable(hostTTL), hostEmptyAt, + type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, Optional.ofNullable(hostTTL), hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withHostEmptyAt(Instant hostEmptyAt) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, Optional.ofNullable(hostEmptyAt), + type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, Optional.ofNullable(hostEmptyAt), exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withExclusiveToClusterType(ClusterSpec.Type exclusiveTo) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, Optional.ofNullable(exclusiveTo), switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withWireguardPubkey(WireguardKeyWithTimestamp wireguardPubkey) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, Optional.ofNullable(wireguardPubkey)); } @@ -518,7 +538,7 @@ public final class Node implements Nodelike { /** Returns a copy of this node with switch hostname set to given value */ public Node withSwitchHostname(String switchHostname) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, Optional.ofNullable(switchHostname), trustStoreItems, cloudAccount, wireguardPubKey); } @@ -572,19 +592,19 @@ public final class Node implements Nodelike { /** Returns a copy of this node with the given history. */ public Node with(History history) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node with(Reports reports) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node with(List<TrustStoreItem> trustStoreItems) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } @@ -722,6 +742,7 @@ public final class Node implements Nodelike { private String modelName; private TenantName reservedTo; private ApplicationId exclusiveToApplicationId; + private ApplicationId provisionedForApplicationId; private Duration hostTTL; private Instant hostEmptyAt; private ClusterSpec.Type exclusiveToClusterType; @@ -763,6 +784,11 @@ public final class Node implements Nodelike { return this; } + public Builder provisionedForApplicationId(ApplicationId provisionedFor) { + this.provisionedForApplicationId = provisionedFor; + return exclusiveToApplicationId(provisionedFor); + } + public Builder hostTTL(Duration hostTTL) { this.hostTTL = hostTTL; return this; @@ -833,9 +859,9 @@ public final class Node implements Nodelike { flavor, Optional.ofNullable(status).orElseGet(Status::initial), state, Optional.ofNullable(allocation), Optional.ofNullable(history).orElseGet(History::empty), type, Optional.ofNullable(reports).orElseGet(Reports::new), Optional.ofNullable(modelName), Optional.ofNullable(reservedTo), Optional.ofNullable(exclusiveToApplicationId), - Optional.ofNullable(hostTTL), Optional.ofNullable(hostEmptyAt), Optional.ofNullable(exclusiveToClusterType), - Optional.ofNullable(switchHostname), Optional.ofNullable(trustStoreItems).orElseGet(List::of), cloudAccount, - Optional.ofNullable(wireguardPubKey)); + Optional.ofNullable(provisionedForApplicationId), Optional.ofNullable(hostTTL), Optional.ofNullable(hostEmptyAt), + Optional.ofNullable(exclusiveToClusterType), Optional.ofNullable(switchHostname), + Optional.ofNullable(trustStoreItems).orElseGet(List::of), cloudAccount, Optional.ofNullable(wireguardPubKey)); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java index 2d2d8d934d8..957996f05e4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision; import com.yahoo.collections.AbstractFilteringList; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeMutex.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeMutex.java index 20c246b3ebd..cb69dc2ac19 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeMutex.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeMutex.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision; import com.yahoo.transaction.Mutex; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java index 1460ce70686..b032ce2cc39 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision; import com.yahoo.collections.Pair; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index eafaed2a217..449e1c07bf8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision; import com.yahoo.component.AbstractComponent; @@ -212,6 +212,11 @@ public class NodeRepository extends AbstractComponent { ( !zone().cloud().allowHostSharing() && !sharedHosts.value().isEnabled(clusterSpec.type().name())); } + /** Whether the nodes of this cluster must be running on hosts that are specifically provisioned for the application. */ + public boolean exclusiveProvisioning(ClusterSpec clusterSpec) { + return !zone.cloud().allowHostSharing() && clusterSpec.isExclusive(); + } + /** * Returns ACLs for the children of the given host. * diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Nodelike.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Nodelike.java index 9a15fc9f609..245b66d9361 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Nodelike.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Nodelike.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision; import com.yahoo.config.provision.NodeResources; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java index df5044de05c..f52c8d8a879 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.applications; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java index 8daac029c7b..e68b2f6103b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java @@ -1,9 +1,10 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.applications; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.ApplicationTransaction; +import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.persistence.CuratorDb; @@ -64,18 +65,18 @@ public class Applications { } /** Create a lock which provides exclusive rights to making changes to the given application */ - public Mutex lock(ApplicationId application) { - return db.lock(application); + public ApplicationMutex lock(ApplicationId application) { + return new ApplicationMutex(application, db.lock(application)); } /** Create a lock with a timeout which provides exclusive rights to making changes to the given application */ - public Mutex lock(ApplicationId application, Duration timeout) { - return db.lock(application, timeout); + public ApplicationMutex lock(ApplicationId application, Duration timeout) { + return new ApplicationMutex(application, db.lock(application, timeout)); } /** Create a lock which provides exclusive rights to perform a maintenance deployment */ - public Mutex lockMaintenance(ApplicationId application) { - return db.lockMaintenance(application); + public ApplicationMutex lockMaintenance(ApplicationId application) { + return new ApplicationMutex(application, db.lockMaintenance(application)); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/BcpGroupInfo.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/BcpGroupInfo.java index e9f3303c9cb..11d54b02dc7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/BcpGroupInfo.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/BcpGroupInfo.java @@ -1,3 +1,4 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.applications; import java.util.Objects; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index 796bc2eeb92..606605ed1e4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.applications; import com.yahoo.config.provision.ClusterInfo; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/ScalingEvent.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/ScalingEvent.java index 91270f14fbb..a68ac44e2aa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/ScalingEvent.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/ScalingEvent.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.applications; import com.yahoo.config.provision.ClusterResources; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Status.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Status.java index 0e0d8b2310b..9b10be343a0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Status.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Status.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.applications; import java.util.Objects; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java index e3bb768a9c2..c4cbbd8bf80 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.archive; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUris.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUris.java index 7ec9bd36744..0d41f552342 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUris.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUris.java @@ -1,3 +1,4 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.archive; import com.yahoo.config.provision.CloudAccount; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java index 286ec2451f8..544436dc902 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java index f650d8ec269..ff30f9d6163 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.ClusterResources; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index b5f86be68f6..8e8474c6a6d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.ClusterResources; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java index fad280d6c29..e05b1c53c76 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java @@ -1,3 +1,4 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.ClusterResources; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterMetricSnapshot.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterMetricSnapshot.java index e60bab4c80c..fdadb5ae043 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterMetricSnapshot.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterMetricSnapshot.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; import java.time.Instant; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java index 8976dd9ff08..4c5ace3d51a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.ClusterSpec; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java index d694085729f..5ae184c50ca 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.vespa.hosted.provision.NodeList; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java index 131873b0137..80a9dcd49ce 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.ClusterSpec; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java index 46eb9e9014a..ab93e585c88 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.IntRange; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Load.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Load.java index a2fa6e63922..799ed621807 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Load.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Load.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.NodeResources; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java index d132d574658..6c75170ba63 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.collections.Pair; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsDb.java index a7e13486a48..acd2e01f301 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsDb.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.collections.Pair; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsFetcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsFetcher.java index e7e826fec91..981a6a57052 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsFetcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsFetcher.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java index 92ec88ed9f4..01bcabeb11b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; import ai.vespa.metrics.ContainerMetrics; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java index 11f245ee332..748c9335c3c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; import ai.vespa.util.http.hc5.VespaAsyncHttpClientBuilder; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricSnapshot.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricSnapshot.java index 706692d58d5..91903a476e2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricSnapshot.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricSnapshot.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; import java.time.Instant; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java index f694070a8db..ed4332881f5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.vespa.hosted.provision.Node; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java index ad150cfc389..38127fa3093 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.component.annotation.Inject; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceChange.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceChange.java index 7a26a217e61..9ac51407873 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceChange.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceChange.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.ClusterSpec; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/DnsZone.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/DnsZone.java index 446c00808ce..d950175c2eb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/DnsZone.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/DnsZone.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.lb; import java.util.Objects; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java index a5aa7f42bc4..36f2eb4bb2c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.lb; import com.yahoo.vespa.applicationmodel.InfrastructureApplication; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerId.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerId.java index 7a699feea4d..008d569937a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerId.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerId.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.lb; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java index f42d1ce9bd3..c0931ecbc70 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.lb; import ai.vespa.http.DomainName; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java index 3e7da831bc4..45d20c68198 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.lb; import com.yahoo.collections.AbstractFilteringList; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java index 4f33e079d8f..6d41f664fad 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.lb; import com.yahoo.config.provision.ClusterSpec; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java index c79ccc2aece..df4b83d1543 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.lb; import ai.vespa.http.DomainName; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java index 6be8bd71553..e49e5aed739 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.lb; import com.google.common.collect.ImmutableSortedSet; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancers.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancers.java index 1d218e6c973..966005817bb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancers.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancers.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.lb; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PrivateServiceId.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PrivateServiceId.java index fac82139953..7531eb6992d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PrivateServiceId.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PrivateServiceId.java @@ -1,3 +1,4 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.lb; import ai.vespa.validation.PatternedStringWrapper; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/Real.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/Real.java index fc8f665c8b4..391dc8aa270 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/Real.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/Real.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.lb; import ai.vespa.http.DomainName; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java index 073662b39fe..e463f5aabe6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.lb; import ai.vespa.http.DomainName; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/package-info.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/package-info.java index 85288e67b57..0f4d3d2490b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/package-info.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/package-info.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** * @author mpolden */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java index 6b084704474..a210ddac400 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.concurrent.DaemonThreadFactory; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index eb290d9ec2a..620026c4ac6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java index d37ef50ca46..83d5657c1f6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.NodeResources; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DeprovisionedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DeprovisionedExpirer.java index 92062f13f1a..d7a290e52e1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DeprovisionedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DeprovisionedExpirer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.jdisc.Metric; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java index e300591fbb2..ab103b0bfcf 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.jdisc.Metric; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacer.java index ab310761140..7907a14e8fd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.jdisc.Metric; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ExpeditedChangeApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ExpeditedChangeApplicationMaintainer.java index 1b81a977019..72d20f7b4c6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ExpeditedChangeApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ExpeditedChangeApplicationMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java index 1684ebbb38f..a79603dc833 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.jdisc.Metric; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java index 4f2201adba0..c3fea72fab9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.Environment; @@ -99,7 +99,8 @@ public class FailedExpirer extends NodeRepositoryMaintainer { .map(Node::hostname) .toList(); if (unparkedChildren.isEmpty()) { - return Optional.of(nodeRepository.nodes().park(node.hostname(), true, Agent.FailedExpirer, + // Only forcing de-provisioning of off premises nodes + return Optional.of(nodeRepository.nodes().park(node.hostname(), nodeRepository.zone().cloud().dynamicProvisioning(), Agent.FailedExpirer, "Parked by FailedExpirer due to " + reason.get())); } else { log.info(String.format("Expired failed node %s was not parked because of unparked children: %s", diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java index ea5ef276b95..be6c420c63b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.component.Version; @@ -13,7 +13,9 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.jdisc.Metric; import com.yahoo.lang.MutableInteger; import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.ListFlag; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.custom.ClusterCapacity; @@ -59,6 +61,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { private final HostProvisioner hostProvisioner; private final ListFlag<ClusterCapacity> preprovisionCapacityFlag; + private final BooleanFlag makeExclusiveFlag; private final ProvisioningThrottler throttler; HostCapacityMaintainer(NodeRepository nodeRepository, @@ -69,6 +72,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { super(nodeRepository, interval, metric); this.hostProvisioner = hostProvisioner; this.preprovisionCapacityFlag = PermanentFlags.PREPROVISION_CAPACITY.bindTo(flagSource); + this.makeExclusiveFlag = Flags.MAKE_EXCLUSIVE.bindTo(flagSource); this.throttler = new ProvisioningThrottler(nodeRepository, metric); } @@ -187,6 +191,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { */ private List<Node> provisionUntilNoDeficit(NodeList nodeList) { List<ClusterCapacity> preprovisionCapacity = preprovisionCapacityFlag.value(); + boolean makeExclusive = makeExclusiveFlag.value(); // Worst-case each ClusterCapacity in preprovisionCapacity will require an allocation. int maxProvisions = preprovisionCapacity.size(); @@ -194,7 +199,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { var nodesPlusProvisioned = new ArrayList<>(nodeList.asList()); for (int numProvisions = 0;; ++numProvisions) { var nodesPlusProvisionedPlusAllocated = new ArrayList<>(nodesPlusProvisioned); - Optional<ClusterCapacity> deficit = allocatePreprovisionCapacity(preprovisionCapacity, nodesPlusProvisionedPlusAllocated); + Optional<ClusterCapacity> deficit = allocatePreprovisionCapacity(preprovisionCapacity, nodesPlusProvisionedPlusAllocated, makeExclusive); if (deficit.isEmpty()) { return nodesPlusProvisionedPlusAllocated; } @@ -204,32 +209,39 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { } ClusterCapacity clusterCapacityDeficit = deficit.get(); - var clusterType = Optional.ofNullable(clusterCapacityDeficit.clusterType()); nodesPlusProvisioned.addAll(provisionHosts(clusterCapacityDeficit.count(), toNodeResources(clusterCapacityDeficit), - clusterType.map(ClusterSpec.Type::from), + Optional.ofNullable(clusterCapacityDeficit.clusterType()), nodeList)); } } - private List<Node> provisionHosts(int count, NodeResources nodeResources, Optional<ClusterSpec.Type> clusterType, NodeList allNodes) { + private List<Node> provisionHosts(int count, NodeResources nodeResources, Optional<String> clusterType, NodeList allNodes) { try { if (throttler.throttle(allNodes, Agent.HostCapacityMaintainer)) { throw new NodeAllocationException("Host provisioning is being throttled", true); } Version osVersion = nodeRepository().osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion); List<Integer> provisionIndices = nodeRepository().database().readProvisionIndices(count); + HostSharing sharingMode = nodeRepository().exclusiveAllocation(asSpec(clusterType, 0)) ? HostSharing.exclusive : HostSharing.shared; HostProvisionRequest request = new HostProvisionRequest(provisionIndices, NodeType.host, nodeResources, ApplicationId.defaultId(), osVersion, - HostSharing.shared, clusterType, Optional.empty(), + sharingMode, clusterType.map(ClusterSpec.Type::valueOf), Optional.empty(), nodeRepository().zone().cloud().account(), false); List<Node> hosts = new ArrayList<>(); - hostProvisioner.provisionHosts(request, - resources -> true, - provisionedHosts -> { - hosts.addAll(provisionedHosts.stream().map(host -> host.generateHost(Duration.ZERO)).toList()); - nodeRepository().nodes().addNodes(hosts, Agent.HostCapacityMaintainer); - }); + Runnable waiter; + try (var lock = nodeRepository().nodes().lockUnallocated()) { + waiter = hostProvisioner.provisionHosts(request, + resources -> true, + provisionedHosts -> { + hosts.addAll(provisionedHosts.stream() + .map(host -> host.generateHost(Duration.ZERO)) + .map(host -> host.withExclusiveToApplicationId(null)) + .toList()); + nodeRepository().nodes().addNodes(hosts, Agent.HostCapacityMaintainer); + }); + } + waiter.run(); return hosts; } catch (NodeAllocationException | IllegalArgumentException | IllegalStateException e) { throw new NodeAllocationException("Failed to provision " + count + " " + nodeResources + ": " + e.getMessage(), @@ -247,11 +259,12 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { * @return the part of a cluster capacity it was unable to allocate, if any */ private Optional<ClusterCapacity> allocatePreprovisionCapacity(List<ClusterCapacity> preprovisionCapacity, - ArrayList<Node> mutableNodes) { + ArrayList<Node> mutableNodes, + boolean makeExclusive) { for (int clusterIndex = 0; clusterIndex < preprovisionCapacity.size(); ++clusterIndex) { ClusterCapacity clusterCapacity = preprovisionCapacity.get(clusterIndex); LockedNodeList allNodes = new LockedNodeList(mutableNodes, () -> {}); - List<Node> candidates = findCandidates(clusterCapacity, clusterIndex, allNodes); + List<Node> candidates = findCandidates(clusterCapacity, clusterIndex, allNodes, makeExclusive); int deficit = Math.max(0, clusterCapacity.count() - candidates.size()); if (deficit > 0) { return Optional.of(clusterCapacity.withCount(deficit)); @@ -264,40 +277,51 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { return Optional.empty(); } - private List<Node> findCandidates(ClusterCapacity clusterCapacity, int clusterIndex, LockedNodeList allNodes) { + private List<Node> findCandidates(ClusterCapacity clusterCapacity, int clusterIndex, LockedNodeList allNodes, boolean makeExclusive) { NodeResources nodeResources = toNodeResources(clusterCapacity); // We'll allocate each ClusterCapacity as a unique cluster in a dummy application ApplicationId applicationId = ApplicationId.defaultId(); - ClusterSpec.Id clusterId = ClusterSpec.Id.from(String.valueOf(clusterIndex)); - ClusterSpec.Type type = clusterCapacity.clusterType() != null - ? ClusterSpec.Type.from(clusterCapacity.clusterType()) - : ClusterSpec.Type.content; - ClusterSpec clusterSpec = ClusterSpec.request(type, clusterId) - // build() requires a version, even though it is not (should not be) used - .vespaVersion(Vtag.currentVersion) - .build(); + ClusterSpec cluster = asSpec(Optional.ofNullable(clusterCapacity.clusterType()), clusterIndex); NodeSpec nodeSpec = NodeSpec.from(clusterCapacity.count(), 1, nodeResources, false, true, nodeRepository().zone().cloud().account(), Duration.ZERO); var allocationContext = IP.Allocation.Context.from(nodeRepository().zone().cloud().name(), nodeSpec.cloudAccount().isExclave(nodeRepository().zone()), nodeRepository().nameResolver()); - NodePrioritizer prioritizer = new NodePrioritizer(allNodes, applicationId, clusterSpec, nodeSpec, - true, allocationContext, nodeRepository().nodes(), nodeRepository().resourcesCalculator(), - nodeRepository().spareCount()); - List<NodeCandidate> nodeCandidates = prioritizer.collect(); + NodePrioritizer prioritizer = new NodePrioritizer(allNodes, applicationId, cluster, nodeSpec, + true, false, allocationContext, nodeRepository().nodes(), + nodeRepository().resourcesCalculator(), nodeRepository().spareCount(), + nodeRepository().exclusiveAllocation(cluster), makeExclusive); + List<NodeCandidate> nodeCandidates = prioritizer.collect() + .stream() + .filter(node -> node.violatesExclusivity(cluster, + applicationId, + nodeRepository().exclusiveAllocation(cluster), + false, + nodeRepository().zone().cloud().allowHostSharing(), + allNodes, + makeExclusive) + != NodeCandidate.ExclusivityViolation.YES) + .toList(); MutableInteger index = new MutableInteger(0); return nodeCandidates .stream() .limit(clusterCapacity.count()) .map(candidate -> candidate.toNode() .allocate(applicationId, - ClusterMembership.from(clusterSpec, index.next()), + ClusterMembership.from(cluster, index.next()), nodeResources, nodeRepository().clock().instant())) .toList(); } + private static ClusterSpec asSpec(Optional<String> clusterType, int index) { + return ClusterSpec.request(clusterType.map(ClusterSpec.Type::from).orElse(ClusterSpec.Type.content), + ClusterSpec.Id.from(String.valueOf(index))) + .vespaVersion(Vtag.currentVersion) // Needed, but should not be used here. + .build(); + } + private static NodeResources toNodeResources(ClusterCapacity clusterCapacity) { return new NodeResources(clusterCapacity.vcpu(), clusterCapacity.memoryGb(), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisioner.java index 0fa16f22061..0a33252303a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisioner.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.jdisc.Metric; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostFlavorUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostFlavorUpgrader.java index 83a710cb5a9..01da75f90b6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostFlavorUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostFlavorUpgrader.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.Deployer; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java index c1407929cc9..2e3c6d1755a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.CloudAccount; @@ -61,10 +61,13 @@ public class HostResumeProvisioner extends NodeRepositoryMaintainer { log.log(Level.INFO, "Could not provision " + host.hostname() + ", will retry in " + interval() + ": " + Exceptions.toMessageString(e)); } catch (FatalProvisioningException e) { + // FatalProvisioningException is thrown if node is not found in the cloud, allow for + // some time for the state to propagate + if (host.history().age(clock().instant()).getSeconds() < 30) continue; failures++; log.log(Level.SEVERE, "Failed to provision " + host.hostname() + ", failing out the host recursively", e); - nodeRepository().nodes().failOrMarkRecursively( - host.hostname(), Agent.HostResumeProvisioner, "Failed by HostResumeProvisioner due to provisioning failure"); + nodeRepository().nodes().parkRecursively( + host.hostname(), Agent.HostResumeProvisioner, true, "Failed by HostResumeProvisioner due to provisioning failure"); } catch (RuntimeException e) { if (e.getCause() instanceof NamingException) log.log(Level.INFO, "Could not provision " + host.hostname() + ", will retry in " + interval() + ": " + Exceptions.toMessageString(e)); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostRetirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostRetirer.java index 7e238470a0c..0212d0e229c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostRetirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostRetirer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.CloudAccount; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java index 503ac4be86c..d97f4566e57 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.jdisc.Metric; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfraApplicationRedeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfraApplicationRedeployer.java new file mode 100644 index 00000000000..434e6161a4c --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfraApplicationRedeployer.java @@ -0,0 +1,117 @@ +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.component.annotation.Inject; +import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.concurrent.UncheckedTimeoutException; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Deployment; +import com.yahoo.config.provision.InfraDeployer; +import com.yahoo.config.provision.NodeType; +import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.applicationmodel.InfrastructureApplication; +import com.yahoo.vespa.hosted.provision.Node.State; +import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.NodeRepository; + +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.logging.Logger; + +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.INFO; +import static java.util.logging.Level.WARNING; + +/** + * Performs on-demand redeployment of the {@link InfrastructureApplication}s, to minimise time between + * host provisioning for a deployment completing, and deployment of its application containers succeeding. + * + * @author jonmv + */ +public class InfraApplicationRedeployer implements AutoCloseable { + + private static final Logger log = Logger.getLogger(InfraApplicationRedeployer.class.getName()); + + private final ExecutorService executor = Executors.newSingleThreadExecutor(new DaemonThreadFactory("infra-application-redeployer-")); + private final Set<InfrastructureApplication> readiedTypes = new ConcurrentSkipListSet<>(); + private final InfraDeployer deployer; + private final Function<ApplicationId, Mutex> locks; + private final Supplier<NodeList> nodes; + + @Inject + public InfraApplicationRedeployer(InfraDeployer deployer, NodeRepository nodes) { + this(deployer, nodes.applications()::lockMaintenance, nodes.nodes()::list); + } + + InfraApplicationRedeployer(InfraDeployer deployer, Function<ApplicationId, Mutex> locks, Supplier<NodeList> nodes) { + this.deployer = deployer; + this.locks = locks; + this.nodes = nodes; + } + + public void readied(NodeType type) { + applicationOf(type).ifPresent(this::readied); + } + + private void readied(InfrastructureApplication application) { + if (application == null) return; + if (readiedTypes.add(application)) executor.execute(() -> checkAndRedeploy(application)); + } + + private void checkAndRedeploy(InfrastructureApplication application) { + if ( ! readiedTypes.remove(application)) return; + try (Mutex lock = locks.apply(application.id())) { + if (application.nodeType().isHost() && nodes.get().state(State.ready).nodeType(application.nodeType()).isEmpty()) return; + log.log(FINE, () -> "Redeploying " + application.id() + " after completing provisioning for " + application.name()); + try { + deployer.getDeployment(application.id()).ifPresent(Deployment::activate); + childOf(application).ifPresent(this::readied); + } + catch (RuntimeException e) { + log.log(INFO, "Failed redeploying " + application.id() + ", will be retried by maintainer", e); + } + } + catch (UncheckedTimeoutException collision) { + readied(application); + } + } + + private static Optional<InfrastructureApplication> applicationOf(NodeType type) { + return switch (type) { + case host -> Optional.of(InfrastructureApplication.TENANT_HOST); + case confighost -> Optional.of(InfrastructureApplication.CONFIG_SERVER_HOST); + case config -> Optional.of(InfrastructureApplication.CONFIG_SERVER); + case controllerhost -> Optional.of(InfrastructureApplication.CONTROLLER_HOST); + case controller -> Optional.of(InfrastructureApplication.CONTROLLER); + case proxyhost -> Optional.of(InfrastructureApplication.PROXY_HOST); + default -> Optional.empty(); + }; + } + + private static Optional<InfrastructureApplication> childOf(InfrastructureApplication application) { + return switch (application) { + case CONFIG_SERVER_HOST -> Optional.of(InfrastructureApplication.CONFIG_SERVER); + case CONTROLLER_HOST -> Optional.of(InfrastructureApplication.CONTROLLER); + default -> Optional.empty(); + }; + } + + @Override + public void close() { + executor.shutdown(); + try { + if (executor.awaitTermination(10, TimeUnit.SECONDS)) return; + log.log(WARNING, "Redeployer did not shut down within 10 seconds"); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + executor.shutdownNow(); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java index 5392df3d5b3..653da9e9645 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.InfraDeployer; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersions.java index 19d6c79d680..dfd5cf5db9b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersions.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersions.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.component.Version; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index 895fdd522e7..04ea831c45c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.google.common.collect.Sets; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java index ae128f70529..d82fa342f8a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import ai.vespa.metrics.ConfigServerMetrics; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index a7e82250275..99d627e58b1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import ai.vespa.metrics.ConfigServerMetrics; @@ -169,8 +169,9 @@ public class MetricsReporter extends NodeRepositoryMaintainer { * NB: Keep this metric set in sync with internal configserver metric pre-aggregation */ private void updateNodeMetrics(Node node, ServiceModel serviceModel) { + if (node.state() != State.active) + return; Metric.Context context; - Optional<Allocation> allocation = node.allocation(); if (allocation.isPresent()) { ApplicationId applicationId = allocation.get().owner(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index 5673b2d74ea..6c4be09c489 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import ai.vespa.metrics.ConfigServerMetrics; @@ -111,7 +111,7 @@ public class NodeFailer extends NodeRepositoryMaintainer { for (Node node : activeNodes) { Instant graceTimeStart = clock().instant().minus(nodeRepository().nodes().suspended(node) ? suspendedDownTimeLimit : downTimeLimit); - if (node.isDown() && node.history().hasEventBefore(History.Event.Type.down, graceTimeStart) && !applicationSuspended(node) && !undergoingCmr(node)) { + if (node.isDown() && node.history().hasEventBefore(History.Event.Type.down, graceTimeStart) && !applicationSuspended(node) && !affectedByMaintenance(node)) { // Allow a grace period after node re-activation if (!node.history().hasEventAfter(History.Event.Type.activated, graceTimeStart)) failingNodes.add(new FailingNode(node, "Node has been down longer than " + downTimeLimit)); @@ -146,7 +146,7 @@ public class NodeFailer extends NodeRepositoryMaintainer { /** Returns whether node has any kind of hardware issue */ static boolean hasHardwareIssue(Node node, NodeList allNodes) { Node host = node.parentHostname().flatMap(allNodes::node).orElse(node); - return reasonsToFailHost(host).size() > 0; + return !reasonsToFailHost(host).isEmpty(); } private boolean applicationSuspended(Node node) { @@ -159,17 +159,18 @@ public class NodeFailer extends NodeRepositoryMaintainer { } } - private boolean undergoingCmr(Node node) { + /** Is a maintenance event affecting this node? */ + private boolean affectedByMaintenance(Node node) { return node.reports().getReport("vcmr") - .map(report -> - SlimeUtils.entriesStream(report.getInspector().field("upcoming")) - .anyMatch(cmr -> { - var startTime = cmr.field("plannedStartTime").asLong(); - var endTime = cmr.field("plannedEndTime").asLong(); - var now = clock().instant().getEpochSecond(); - return now > startTime && now < endTime; - }) - ).orElse(false); + .map(report -> + SlimeUtils.entriesStream(report.getInspector().field("upcoming")) + .anyMatch(cmr -> { + var startTime = cmr.field("plannedStartTime").asLong(); + var endTime = cmr.field("plannedEndTime").asLong(); + var now = clock().instant().getEpochSecond(); + return now > startTime && now < endTime; + }) + ).orElse(false); } /** Is the node and all active children suspended? */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java index 979a2771082..e0421b78ff2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java index 4c5ea45f3ec..0ff8acdcf1f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMover.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMover.java index 36db8bff8e9..0da3ca11fe7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMover.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMover.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java index 27945fad7f5..c65a8d889f3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.jdisc.Metric; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java index b8d37c7eb5c..c7be505acaa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.concurrent.maintenance.JobMetrics; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 573298c142c..433a37e9686 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.component.AbstractComponent; @@ -137,11 +137,11 @@ public class NodeRepositoryMaintenance extends AbstractComponent { hostResumeProvisionerInterval = Duration.ofMinutes(3); diskReplacerInterval = Duration.ofMinutes(3); failedExpirerInterval = Duration.ofMinutes(10); - failGrace = Duration.ofMinutes(20); + failGrace = Duration.ofMinutes(10); infrastructureProvisionInterval = Duration.ofMinutes(3); loadBalancerExpirerInterval = Duration.ofMinutes(5); metricsInterval = Duration.ofMinutes(1); - nodeFailerInterval = Duration.ofMinutes(7); + nodeFailerInterval = Duration.ofMinutes(4); nodeFailureStatusUpdateInterval = Duration.ofMinutes(2); nodeMetricsCollectionInterval = Duration.ofMinutes(1); expeditedChangeRedeployInterval = Duration.ofMinutes(3); @@ -158,7 +158,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { switchRebalancerInterval = Duration.ofHours(1); throttlePolicy = NodeFailer.ThrottlePolicy.hosted; hostRetirerInterval = Duration.ofMinutes(30); - hostFlavorUpgraderInterval = Duration.ofMinutes(30); + hostFlavorUpgraderInterval = Duration.ofHours(1); // CD, test and staging (de)provisions hosts frequently. Expire deprovisioned ones earlier deprovisionedExpiry = (isCdZone || isTest) ? Duration.ofDays(3) : Duration.ofDays(30); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java index 90d02dfad9b..c5252f344dc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.NodeType; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java index 9cbf1778b3b..3cec8483a45 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; @@ -11,10 +11,12 @@ import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.yolean.Exceptions; import java.time.Duration; import java.util.Map; +import static java.util.logging.Level.INFO; import static java.util.stream.Collectors.toMap; /** @@ -38,10 +40,16 @@ public class PeriodicApplicationMaintainer extends ApplicationMaintainer { @Override protected boolean canDeployNow(ApplicationId application) { - return deployer().deployTime(application) - .map(lastDeployTime -> lastDeployTime.isBefore(nodeRepository().clock().instant().minus(minTimeBetweenRedeployments)) - || deployer().readiedReindexingAfter(application, lastDeployTime)) - .orElse(false); + try { + return deployer().deployTime(application) + .map(lastDeployTime -> lastDeployTime.isBefore(nodeRepository().clock().instant().minus(minTimeBetweenRedeployments)) + || deployer().readiedReindexingAfter(application, lastDeployTime)) + .orElse(false); + } + catch (Exception e) { + log.log(INFO, () -> "Failed finding last deploy time or reindexing status for " + application + Exceptions.toMessageString(e)); + throw e; + } } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java index d2f8ae1e5a7..24901cb10a9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.NodeType; @@ -38,10 +38,9 @@ public class ProvisionedExpirer extends Expirer { for (Node expiredNode : expired) { if (expiredNode.type() != NodeType.host) continue; - nodeRepository().nodes().parkRecursively(expiredNode.hostname(), Agent.ProvisionedExpirer, "Node is stuck in provisioned"); - if (MAXIMUM_ALLOWED_EXPIRED_HOSTS < ++previouslyExpired) { - nodeRepository.nodes().deprovision(expiredNode.hostname(), Agent.ProvisionedExpirer, nodeRepository.clock().instant()); - } + boolean forceDeprovision = MAXIMUM_ALLOWED_EXPIRED_HOSTS < ++previouslyExpired; + nodeRepository().nodes().parkRecursively(expiredNode.hostname(), Agent.ProvisionedExpirer, + forceDeprovision, "Node is stuck in provisioned"); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java index 98407c9d13c..c73b4d378c8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import ai.vespa.metrics.ConfigServerMetrics; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java index 2484f496ece..c73ed57485d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.jdisc.Metric; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java index 1ae9b00d794..ec1fbc18915 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.concurrent.UncheckedTimeoutException; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java index 60688e3f460..fd93d202795 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java index da05656fcee..89f2740bbab 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import ai.vespa.metrics.ConfigServerMetrics; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SwitchRebalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SwitchRebalancer.java index 9f009b47983..5e600b990ae 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SwitchRebalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SwitchRebalancer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java index d8c1344416c..d1c590ad83e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node; /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Allocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Allocation.java index ec1c6cd4b1a..b9c15d26a62 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Allocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Allocation.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/ClusterId.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/ClusterId.java index 49eb25025a8..4e6ec7ff78b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/ClusterId.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/ClusterId.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Dns.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Dns.java index af1e7d5cc0f..75ff0d29bd3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Dns.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Dns.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node; import com.yahoo.config.provision.CloudAccount; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Generation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Generation.java index 5a3e5527a0a..c20ab068468 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Generation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Generation.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node; /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java index 93b28d07471..491b388416b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node; import com.google.common.collect.ImmutableMap; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java index d9eba6cc8a5..264e981558a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java @@ -1,8 +1,7 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node; import com.google.common.net.InetAddresses; -import com.google.common.primitives.UnsignedBytes; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.HostName; @@ -75,31 +74,6 @@ public record IP() { public int hashCode() { return Objects.hash(version); } } - /** Comparator for sorting IP addresses by their natural order */ - public static final Comparator<InetAddress> NATURAL_ORDER = (ip1, ip2) -> { - byte[] address1 = ip1.getAddress(); - byte[] address2 = ip2.getAddress(); - - // IPv4 always sorts before IPv6 - if (address1.length < address2.length) return -1; - if (address1.length > address2.length) return 1; - - // Compare each octet - for (int i = 0; i < address1.length; i++) { - int b1 = UnsignedBytes.toInt(address1[i]); - int b2 = UnsignedBytes.toInt(address2[i]); - if (b1 == b2) { - continue; - } - if (b1 < b2) { - return -1; - } else { - return 1; - } - } - return 0; - }; - /** * IP configuration of a node * @@ -248,11 +222,23 @@ public record IP() { this.hostnames = List.copyOf(Objects.requireNonNull(hostnames, "hostnames must be non-null")); } + /** The number of hosts in this pool: each host has a name and/or one or two IP addresses. */ + public long size() { + return hostnames().isEmpty() ? + Math.max(ipAddresses.addresses.stream().filter(IP::isV4).count(), + ipAddresses.addresses.stream().filter(IP::isV6).count()) : + hostnames().size(); + } + public List<String> ips() { return ipAddresses.addresses; } /** * Find a free allocation in this pool. Note that the allocation is not final until it is assigned to a node * + * <p>TODO(2023-09-20): Once all dynamically provisioned hosts have been reprovisioned, the order of IP addresses + * is significant and should be 1:1 with the order of hostnames, if both are filled. This needs to be verified. + * This can be used to strengthen validation, and simplify the selection of the Allocation to return.</p> + * * @param context allocation context * @param nodes a locked list of all nodes in the repository * @return an allocation from the pool, if any can be made diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeAcl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeAcl.java index d88c9189157..16aa7197587 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeAcl.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/NodeAcl.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node; import com.google.common.collect.ImmutableSet; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index 560abb1f0e8..5c0614dd8d6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node; import com.yahoo.component.Version; @@ -6,8 +6,8 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.config.provision.Zone; import com.yahoo.time.TimeBudget; import com.yahoo.transaction.Mutex; @@ -106,24 +106,10 @@ public class Nodes { public NodeList list(Node.State... inState) { NodeList allNodes = NodeList.copyOf(db.readNodes()); NodeList nodes = inState.length == 0 ? allNodes : allNodes.state(Set.of(inState)); - nodes = NodeList.copyOf(nodes.stream().map(node -> specifyFully(node, allNodes)).toList()); + nodes = NodeList.copyOf(nodes.stream().toList()); return nodes; } - // Repair underspecified node resources. TODO: Remove this after June 2023 - private Node specifyFully(Node node, NodeList allNodes) { - if (node.resources().isUnspecified()) return node; - - if (node.resources().bandwidthGbpsIsUnspecified()) - node = node.with(new Flavor(node.resources().withBandwidthGbps(0.3)), Agent.system, clock.instant()); - if ( node.resources().architecture() == NodeResources.Architecture.any) { - Optional<Node> parent = allNodes.parentOf(node); - if (parent.isPresent()) - node = node.with(new Flavor(node.resources().with(parent.get().resources().architecture())), Agent.system, clock.instant()); - } - return node; - } - /** Returns a locked list of all nodes in this repository */ public LockedNodeList list(Mutex lock) { return new LockedNodeList(list().asList(), lock); @@ -238,6 +224,23 @@ public class Nodes { performOn(nodes, (node, mutex) -> write(node.with(node.allocation().get().removable(true, reusable)), mutex)); } + /** Sets the exclusiveToApplicationId field. The nodes must be tenant hosts without the field already. */ + public void setExclusiveToApplicationId(List<Node> hosts, ApplicationMutex lock) { + List<Node> hostsToWrite = hosts.stream() + .filter(host -> !host.exclusiveToApplicationId().equals(Optional.of(lock.application()))) + .peek(host -> { + if (host.type() != NodeType.host) + throw new IllegalArgumentException("Unable to set " + host + " exclusive to " + lock.application() + + ": the node is not a tenant host"); + if (host.exclusiveToApplicationId().isPresent()) + throw new IllegalArgumentException("Unable to set " + host + " exclusive to " + lock.application() + + ": it is already set exclusive to " + host.exclusiveToApplicationId().get()); + }) + .map(host -> host.withExclusiveToApplicationId(lock.application())) + .toList(); + write(hostsToWrite, lock); + } + /** * Deactivates these nodes in a transaction and returns the nodes in the new state which will hold if the * transaction commits. @@ -400,8 +403,8 @@ public class Nodes { * * @return List of all the parked nodes in their new state */ - public List<Node> parkRecursively(String hostname, Agent agent, String reason) { - return moveRecursively(hostname, Node.State.parked, agent, Optional.of(reason)); + public List<Node> parkRecursively(String hostname, Agent agent, boolean forceDeprovision, String reason) { + return moveRecursively(hostname, Node.State.parked, agent, forceDeprovision, Optional.of(reason)); } /** @@ -430,12 +433,12 @@ public class Nodes { } } - private List<Node> moveRecursively(String hostname, Node.State toState, Agent agent, Optional<String> reason) { + private List<Node> moveRecursively(String hostname, Node.State toState, Agent agent, boolean forceDeprovision, Optional<String> reason) { try (RecursiveNodeMutexes locked = lockAndGetRecursively(hostname, Optional.empty())) { List<Node> moved = new ArrayList<>(); NestedTransaction transaction = new NestedTransaction(); for (NodeMutex node : locked.nodes().nodes()) - moved.add(move(node.node().hostname(), toState, agent, false, reason, transaction)); + moved.add(move(node.node().hostname(), toState, agent, forceDeprovision, reason, transaction)); transaction.commit(); return moved; } @@ -481,11 +484,12 @@ public class Nodes { } } - /* - * This method is used by the REST API to handle readying nodes for new allocations. For Linux - * containers this will remove the node from node repository, otherwise the node will be moved to state ready. + /** + * This method is used by the REST API to handle readying nodes for new allocations. + * Tenant containers will be removed, while other nodes will be moved to the ready state. + * Returns true if a node was updated, or false if the node was removed, or already was ready. */ - public Node markNodeAvailableForNewAllocation(String hostname, Agent agent, String reason) { + public boolean markNodeAvailableForNewAllocation(String hostname, Agent agent, String reason) { try (NodeMutex nodeMutex = lockAndGetRequired(hostname)) { Node node = nodeMutex.node(); if (node.type() == NodeType.tenant) { @@ -495,17 +499,18 @@ public class Nodes { NestedTransaction transaction = new NestedTransaction(); db.removeNodes(List.of(node), transaction); transaction.commit(); - return node; + return false; } - if (node.state() == Node.State.ready) return node; + if (node.state() == Node.State.ready) return false; Node parentHost = node.parentHostname().flatMap(this::node).orElse(node); List<String> failureReasons = NodeFailer.reasonsToFailHost(parentHost); if (!failureReasons.isEmpty()) illegal(node + " cannot be readied because it has hard failures: " + failureReasons); - return setReady(nodeMutex, agent, reason); + setReady(nodeMutex, agent, reason); + return true; } } @@ -618,17 +623,9 @@ public class Nodes { * @return the nodes in their new state */ public List<Node> restartActive(Predicate<Node> filter) { - return restart(NodeFilter.in(Set.of(Node.State.active)).and(filter)); - } - - /** - * Increases the restart generation of the any nodes matching given filter. - * - * @return the nodes in their new state - */ - public List<Node> restart(Predicate<Node> filter) { - return performOn(filter, (node, lock) -> write(node.withRestart(node.allocation().get().restartGeneration().withIncreasedWanted()), - lock)); + return performOn(NodeFilter.in(Set.of(State.active)).and(filter), + (node, lock) -> write(node.withRestart(node.allocation().get().restartGeneration().withIncreasedWanted()), + lock)); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/OsVersion.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/OsVersion.java index ffb9477c88e..c914fb4ace8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/OsVersion.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/OsVersion.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node; import com.yahoo.component.Version; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java index 32a10c95c67..92fb69d6437 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node; import com.yahoo.slime.Cursor; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java index 135f1220be9..8e82307deb1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node; import com.yahoo.slime.Cursor; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java index 6e5c5a07fc2..04f08240e43 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node; import com.yahoo.component.Version; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/TrustStoreItem.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/TrustStoreItem.java index 6fb94d0bc62..c24a2ec463a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/TrustStoreItem.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/TrustStoreItem.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node; @@ -65,4 +65,4 @@ public class TrustStoreItem { public int hashCode() { return Objects.hash(fingerprint, expiry); } -}
\ No newline at end of file +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java index 401554f940d..89324fc9b89 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node.filter; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/CloudAccountFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/CloudAccountFilter.java index c7a4f50ab7c..638d0b94c13 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/CloudAccountFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/CloudAccountFilter.java @@ -1,3 +1,4 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node.filter; import com.yahoo.config.provision.CloudAccount; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeFilter.java index a65ec30264f..55e13a21861 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeFilter.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node.filter; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeHostFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeHostFilter.java index 24cda94d12c..87c34e4aaba 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeHostFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeHostFilter.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node.filter; import com.yahoo.config.provision.HostFilter; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeListFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeListFilter.java index 2b790ff7392..b81a57b1496 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeListFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeListFilter.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node.filter; import com.yahoo.vespa.hosted.provision.Node; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeOsVersionFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeOsVersionFilter.java index cbb004324cb..b2f1a5106f6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeOsVersionFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeOsVersionFilter.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node.filter; import com.yahoo.component.Version; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeTypeFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeTypeFilter.java index 95436dcbf10..0c6a9828947 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeTypeFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeTypeFilter.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node.filter; import com.yahoo.config.provision.NodeType; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java index 7386ba066aa..d5aa82a8dc2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node.filter; import com.yahoo.text.StringUtilities; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/package-info.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/package-info.java index 2800d925aff..75981cce3c0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/package-info.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/package-info.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** * @author bjorncs */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/CompositeOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/CompositeOsUpgrader.java index 6bab393945b..02f1b951c8e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/CompositeOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/CompositeOsUpgrader.java @@ -1,3 +1,4 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.os; import com.yahoo.config.provision.NodeType; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java index 4ee0774db8f..c2d3f511711 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.os; import com.yahoo.config.provision.NodeType; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java index f8becd31792..436181f99ba 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.os; import com.yahoo.config.provision.NodeType; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersionChange.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersionChange.java index a21c8aa539b..d8cd909e64c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersionChange.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersionChange.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.os; import com.yahoo.component.Version; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersionTarget.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersionTarget.java index 784df833e14..08577f11d74 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersionTarget.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersionTarget.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.os; import com.yahoo.component.Version; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java index 75ab91c4efc..daed86dc2ab 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.os; import com.yahoo.component.Version; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java index 805793b41a4..108093d8379 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.os; import com.yahoo.component.Version; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java index ccb7f40b0de..c1e8f2b6fa4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.os; import com.yahoo.component.Version; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/package-info.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/package-info.java index 5777f7f23b6..cbc98bf3d75 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/package-info.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/package-info.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage package com.yahoo.vespa.hosted.provision; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java index f62e019e408..c4e7d3b9acc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; import com.yahoo.config.provision.ClusterInfo; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ArchiveUriSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ArchiveUriSerializer.java index 3bf37816dd4..8cbd42ca30b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ArchiveUriSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ArchiveUriSerializer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; import com.yahoo.config.provision.CloudAccount; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CacheStats.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CacheStats.java index d53b6398d4b..aa43d616599 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CacheStats.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CacheStats.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CachingCurator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CachingCurator.java index f26703dc598..b74cb689292 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CachingCurator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CachingCurator.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; import com.google.common.cache.AbstractCache; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CountingCuratorTransaction.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CountingCuratorTransaction.java index b5771f3a846..155158c51ee 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CountingCuratorTransaction.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CountingCuratorTransaction.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; import com.yahoo.vespa.curator.Curator; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java index e4e08e5a15c..4b81e580b64 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; import ai.vespa.http.DomainName; @@ -221,7 +221,7 @@ public class CuratorDb { toState.isAllocated() ? node.allocation() : Optional.empty(), node.history().recordStateTransition(node.state(), toState, agent, clock.instant()), node.type(), node.reports(), node.modelName(), node.reservedTo(), - node.exclusiveToApplicationId(), node.hostTTL(), node.hostEmptyAt(), + node.exclusiveToApplicationId(), node.provisionedForApplicationId(), node.hostTTL(), node.hostEmptyAt(), node.exclusiveToClusterType(), node.switchHostname(), node.trustedCertificates(), node.cloudAccount(), node.wireguardPubKey()); curatorTransaction.add(createOrSet(nodePath(newNode), nodeSerializer.toJson(newNode))); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java index 14b961c7882..f6c8006b703 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; import com.google.common.net.InetAddresses; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/JobControlFlags.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/JobControlFlags.java index 0632fc0818b..f85e70550e3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/JobControlFlags.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/JobControlFlags.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; import com.yahoo.concurrent.maintenance.JobControlState; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java index d329676f842..7262daf758b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; import ai.vespa.http.DomainName; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NameResolver.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NameResolver.java index c32c96bc0dc..56ebd1d5dea 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NameResolver.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NameResolver.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; import java.util.Optional; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeResourcesSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeResourcesSerializer.java index 8843ce69994..01eb24ec2ac 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeResourcesSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeResourcesSerializer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; import com.yahoo.config.provision.NodeResources; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 73531d650d5..5efe5d8b2a8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; import com.yahoo.component.Version; @@ -50,7 +50,7 @@ import java.util.Optional; */ public class NodeSerializer { - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // WARNING: Since there are multiple servers in a ZooKeeper cluster, and they upgrade one by one // (and rewrite all nodes on startup), changes to the serialized format must be made // such that what is serialized on version N+1 can be read by version N: // - ADDING FIELDS: Always ok @@ -92,6 +92,7 @@ public class NodeSerializer { private static final String modelNameKey = "modelName"; private static final String reservedToKey = "reservedTo"; private static final String exclusiveToApplicationIdKey = "exclusiveTo"; + private static final String provisionedForApplicationIdKey = "provisionedFor"; private static final String hostTTLKey = "hostTTL"; private static final String hostEmptyAtKey = "hostEmptyAt"; private static final String exclusiveToClusterTypeKey = "exclusiveToClusterType"; @@ -182,6 +183,7 @@ public class NodeSerializer { node.modelName().ifPresent(modelName -> object.setString(modelNameKey, modelName)); node.reservedTo().ifPresent(tenant -> object.setString(reservedToKey, tenant.value())); node.exclusiveToApplicationId().ifPresent(applicationId -> object.setString(exclusiveToApplicationIdKey, applicationId.serializedForm())); + node.provisionedForApplicationId().ifPresent(applicationId -> object.setString(provisionedForApplicationIdKey, applicationId.serializedForm())); node.hostTTL().ifPresent(hostTTL -> object.setLong(hostTTLKey, hostTTL.toMillis())); node.hostEmptyAt().ifPresent(emptyAt -> object.setLong(hostEmptyAtKey, emptyAt.toEpochMilli())); node.exclusiveToClusterType().ifPresent(clusterType -> object.setString(exclusiveToClusterTypeKey, clusterType.name())); @@ -237,7 +239,7 @@ public class NodeSerializer { private void toSlime(List<String> addresses, Cursor array, boolean dummyDueToErasure) { // Validating IP address string literals is expensive, so we do it at serialization time instead of Node // construction time - addresses.stream().map(IP::parse).sorted(IP.NATURAL_ORDER).map(IP::asString).forEach(array::addString); + addresses.stream().map(IP::parse).map(IP::asString).forEach(array::addString); } private void toSlime(List<HostName> hostnames, Cursor object) { @@ -281,6 +283,7 @@ public class NodeSerializer { SlimeUtils.optionalString(object.field(modelNameKey)), SlimeUtils.optionalString(object.field(reservedToKey)).map(TenantName::from), SlimeUtils.optionalString(object.field(exclusiveToApplicationIdKey)).map(ApplicationId::fromSerializedForm), + SlimeUtils.optionalString(object.field(provisionedForApplicationIdKey)).map(ApplicationId::fromSerializedForm), SlimeUtils.optionalDuration(object.field(hostTTLKey)), SlimeUtils.optionalInstant(object.field(hostEmptyAtKey)), SlimeUtils.optionalString(object.field(exclusiveToClusterTypeKey)).map(ClusterSpec.Type::from), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java index 45a5a768575..d97c6517ef6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; import com.yahoo.component.Version; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionChangeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionChangeSerializer.java index fd9e5eb34a9..891a39a6901 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionChangeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionChangeSerializer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; import com.yahoo.component.Version; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java index 9adff9f9d7a..9bf7d928138 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index e5599ac3d18..1e9adea4e95 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; @@ -9,6 +9,7 @@ import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeResources.DiskSpeed; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.StringFlag; @@ -90,7 +91,10 @@ public class CapacityPolicies { } public NodeResources specifyFully(NodeResources resources, ClusterSpec clusterSpec, ApplicationId applicationId) { - return resources.withUnspecifiedNumbersFrom(defaultResources(clusterSpec, applicationId)); + NodeResources amended = resources.withUnspecifiedFieldsFrom(defaultResources(clusterSpec, applicationId).with(DiskSpeed.any)); + // TODO jonmv: remove this after all apps are 8.248.8 or above; architecture for admin nodes was not picked up before this. + if (clusterSpec.vespaVersion().isBefore(Version.fromString("8.248.8"))) amended = amended.with(resources.architecture()); + return amended; } private NodeResources defaultResources(ClusterSpec clusterSpec, ApplicationId applicationId) { @@ -149,7 +153,9 @@ public class CapacityPolicies { return Architecture.valueOf(adminClusterNodeArchitecture.with(INSTANCE_ID, instance.serializedForm()).value()); } - /** Returns the resources for the newest version not newer than that requested in the cluster spec. */ + /** + * Returns the resources for the newest version not newer than that requested in the cluster spec. + */ static NodeResources versioned(ClusterSpec spec, Map<Version, NodeResources> resources) { return requireNonNull(new TreeMap<>(resources).floorEntry(spec.vespaVersion()), "no default resources applicable for " + spec + " among: " + resources) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImages.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImages.java index 583045083a9..a8f532a1a9f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImages.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImages.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.DockerImage; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java index b7e7ac7ee4b..0d6a98f50a3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.Flavor; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FatalProvisioningException.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FatalProvisioningException.java index afd105c1388..149279873be 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FatalProvisioningException.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FatalProvisioningException.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FirmwareChecks.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FirmwareChecks.java index ff375f2abe1..80806857897 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FirmwareChecks.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FirmwareChecks.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.lang.CachedSupplier; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java index 2e9cca21052..3153b3f6d10 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.Flavor; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupAssigner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupAssigner.java index 542f2c00f95..e314542a40c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupAssigner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupAssigner.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ClusterSpec; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java index 991bc22402d..dab6e9d1581 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.NodeResources; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostIpConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostIpConfig.java index 225f4827a18..a85e6c326f5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostIpConfig.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostIpConfig.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.vespa.hosted.provision.node.IP; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisionRequest.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisionRequest.java index f7b9c9016b1..ddf59ed3146 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisionRequest.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisionRequest.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java index 3362ec5d3b3..38cbfa7fe5f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.CloudAccount; @@ -22,14 +22,21 @@ public interface HostProvisioner { enum HostSharing { - /** The host must be provisioned exclusively for the applicationId */ + /** The host must be provisioned exclusively for the application ID. */ + provision, + + /** The host must be exclusive to a single application ID */ exclusive, /** The host must be provisioned to be shared with other applications. */ shared, /** The client has no requirements on whether the host must be provisioned exclusively or shared. */ - any + any; + + public boolean isExclusiveAllocation() { + return this == provision || this == exclusive; + } } @@ -43,8 +50,10 @@ public interface HostProvisioner { * written to ZK immediately in case the config server goes down while waiting * for the provisioning to finish. * @throws NodeAllocationException if the cloud provider cannot satisfy the request + * @return a runnable that waits for the provisioning request to finish. It can be run without holding any locks, + * but may fail with an exception that should be propagated to the user initiating prepare() */ - void provisionHosts(HostProvisionRequest request, Predicate<NodeResources> realHostResourcesWithinLimits, Consumer<List<ProvisionedHost>> whenProvisioned) throws NodeAllocationException; + Runnable provisionHosts(HostProvisionRequest request, Predicate<NodeResources> realHostResourcesWithinLimits, Consumer<List<ProvisionedHost>> whenProvisioned) throws NodeAllocationException; /** * Continue provisioning of given list of Nodes. diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java index 90cdf932f17..e474ae6eea3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.Flavor; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java index 1f424a1e1d5..39c14be4d2b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import ai.vespa.http.DomainName; @@ -13,7 +13,6 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InfraDeployer; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Provisioner; -import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions; @@ -60,7 +59,7 @@ public class InfraDeployerImpl implements InfraDeployer { .forEach(api -> { var application = api.getApplicationId(); var deployment = new InfraDeployment(api); - try { + try (var lock = nodeRepository.applications().lockMaintenance(application)) { deployment.activate(); } catch (RuntimeException e) { logger.log(Level.INFO, "Failed to activate " + application, e); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 22909122079..4ee1573269a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index ca170d2af6b..e1be5b48e2d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; @@ -84,9 +84,10 @@ class NodeAllocation { private final NodeRepository nodeRepository; private final Optional<String> requiredHostFlavor; + private final boolean makeExclusive; NodeAllocation(NodeList allNodes, ApplicationId application, ClusterSpec cluster, NodeSpec requested, - Supplier<Integer> nextIndex, NodeRepository nodeRepository) { + Supplier<Integer> nextIndex, NodeRepository nodeRepository, boolean makeExclusive) { this.allNodes = allNodes; this.application = application; this.cluster = cluster; @@ -99,6 +100,7 @@ class NodeAllocation { .with(FetchVector.Dimension.CLUSTER_ID, cluster.id().value()) .value()) .filter(s -> !s.isBlank()); + this.makeExclusive = makeExclusive; } /** @@ -117,21 +119,21 @@ class NodeAllocation { ClusterMembership membership = allocation.membership(); if ( ! allocation.owner().equals(application)) continue; // wrong application if ( ! membership.cluster().satisfies(cluster)) continue; // wrong cluster id/type - if ( candidate.state() == Node.State.active && allocation.removable()) continue; // don't accept; causes removal - if ( candidate.state() == Node.State.active && candidate.wantToFail()) continue; // don't accept; causes failing - if ( indexes.contains(membership.index())) continue; // duplicate index (just to be sure) + if (candidate.state() == Node.State.active && allocation.removable()) continue; // don't accept; causes removal + if (candidate.state() == Node.State.active && candidate.wantToFail()) continue; // don't accept; causes failing + if (indexes.contains(membership.index())) continue; // duplicate index (just to be sure) if (nodeRepository.zone().cloud().allowEnclave() && candidate.parent.isPresent() && ! candidate.parent.get().cloudAccount().equals(requested.cloudAccount())) continue; // wrong account boolean resizeable = requested.considerRetiring() && candidate.isResizable; - if ((! saturated() && hasCompatibleResources(candidate) && requested.acceptable(candidate)) || acceptIncompatible(candidate)) { + if (( ! saturated() && hasCompatibleResources(candidate) && requested.acceptable(candidate)) || acceptIncompatible(candidate)) { candidate = candidate.withNode(); if (candidate.isValid()) acceptNode(candidate, shouldRetire(candidate, candidates), resizeable); } } - else if (! saturated() && hasCompatibleResources(candidate)) { - if (! nodeRepository.nodeResourceLimits().isWithinRealLimits(candidate, application, cluster)) { + else if ( ! saturated() && hasCompatibleResources(candidate)) { + if ( ! nodeRepository.nodeResourceLimits().isWithinRealLimits(candidate, application, cluster)) { ++rejectedDueToInsufficientRealResources; continue; } @@ -139,9 +141,13 @@ class NodeAllocation { ++rejectedDueToClashingParentHost; continue; } - if ( violatesExclusivity(candidate)) { - ++rejectedDueToExclusivity; - continue; + switch (violatesExclusivity(candidate)) { + case PARENT_HOST_NOT_EXCLUSIVE -> candidate = candidate.withExclusiveParent(true); + case NONE -> {} + case YES -> { + ++rejectedDueToExclusivity; + continue; + } } if (candidate.wantToRetire()) { continue; @@ -169,7 +175,7 @@ class NodeAllocation { if (candidate.parent.map(node -> node.status().wantToUpgradeFlavor()).orElse(false)) return Retirement.violatesHostFlavorGeneration; if (candidate.wantToRetire()) return Retirement.hardRequest; if (candidate.preferToRetire() && candidate.replaceableBy(candidates)) return Retirement.softRequest; - if (violatesExclusivity(candidate)) return Retirement.violatesExclusivity; + if (violatesExclusivity(candidate) != NodeCandidate.ExclusivityViolation.NONE) return Retirement.violatesExclusivity; if (requiredHostFlavor.isPresent() && ! candidate.parent.map(node -> node.flavor().name()).equals(requiredHostFlavor)) return Retirement.violatesHostFlavor; if (candidate.violatesSpares) return Retirement.violatesSpares; return Retirement.none; @@ -186,39 +192,15 @@ class NodeAllocation { } private boolean offeredNodeHasParentHostnameAlreadyAccepted(NodeCandidate candidate) { - for (NodeCandidate acceptedNode : nodes.values()) { - if (acceptedNode.parentHostname().isPresent() && candidate.parentHostname().isPresent() && - acceptedNode.parentHostname().get().equals(candidate.parentHostname().get())) { - return true; - } - } - return false; - } - - private boolean violatesExclusivity(NodeCandidate candidate) { if (candidate.parentHostname().isEmpty()) return false; - if (requested.type() != NodeType.tenant) return false; - - // In zones which does not allow host sharing, exclusivity is violated if... - if ( ! nodeRepository.zone().cloud().allowHostSharing()) { - // TODO: Write this in a way that is simple to read - // If either the parent is dedicated to a cluster type different from this cluster - return ! candidate.parent.flatMap(Node::exclusiveToClusterType).map(cluster.type()::equals).orElse(true) || - // or the parent is dedicated to a different application - ! candidate.parent.flatMap(Node::exclusiveToApplicationId).map(application::equals).orElse(true) || - // or this cluster requires exclusivity, but the host is not exclusive - (nodeRepository.exclusiveAllocation(cluster) && candidate.parent.flatMap(Node::exclusiveToApplicationId).isEmpty()); - } + return nodes.values().stream().anyMatch(acceptedNode -> acceptedNode.parentHostname().equals(candidate.parentHostname())); + } - // In zones with shared hosts we require that if either of the nodes on the host requires exclusivity, - // then all the nodes on the host must have the same owner - for (Node nodeOnHost : allNodes.childrenOf(candidate.parentHostname().get())) { - if (nodeOnHost.allocation().isEmpty()) continue; - if (nodeRepository.exclusiveAllocation(cluster) || nodeOnHost.allocation().get().membership().cluster().isExclusive()) { - if ( ! nodeOnHost.allocation().get().owner().equals(application)) return true; - } - } - return false; + private NodeCandidate.ExclusivityViolation violatesExclusivity(NodeCandidate candidate) { + return candidate.violatesExclusivity(cluster, application, + nodeRepository.exclusiveAllocation(cluster), + nodeRepository.exclusiveProvisioning(cluster), + nodeRepository.zone().cloud().allowHostSharing(), allNodes, makeExclusive); } /** @@ -399,6 +381,14 @@ class NodeAllocation { return requested.type(); } + List<Node> parentsRequiredToBeExclusive() { + return nodes.values() + .stream() + .filter(candidate -> candidate.exclusiveParent) + .map(candidate -> candidate.parent.orElseThrow()) + .toList(); + } + List<Node> finalNodes() { GroupAssigner groupAssigner = new GroupAssigner(requested, allNodes, nodeRepository.clock()); Collection<NodeCandidate> finalNodes = groupAssigner.assignTo(nodes.values()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java index 27136811fa1..1547a266e15 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java @@ -1,14 +1,16 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.Nodelike; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.IP; @@ -20,6 +22,8 @@ import java.util.Objects; import java.util.Optional; import java.util.logging.Logger; +import static com.yahoo.collections.Optionals.emptyOrEqual; + /** * A node candidate containing the details required to prioritize it for allocation. This is immutable. * @@ -59,7 +63,11 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat /** This node can be resized to the new NodeResources */ final boolean isResizable; - private NodeCandidate(NodeResources freeParentCapacity, Optional<Node> parent, boolean violatesSpares, boolean exclusiveSwitch, boolean isSurplus, boolean isNew, boolean isResizeable) { + /** The parent host must become exclusive to the implied application */ + final boolean exclusiveParent; + + private NodeCandidate(NodeResources freeParentCapacity, Optional<Node> parent, boolean violatesSpares, boolean exclusiveSwitch, + boolean exclusiveParent, boolean isSurplus, boolean isNew, boolean isResizeable) { if (isResizeable && isNew) throw new IllegalArgumentException("A new node cannot be resizable"); @@ -67,6 +75,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat this.parent = parent; this.violatesSpares = violatesSpares; this.exclusiveSwitch = exclusiveSwitch; + this.exclusiveParent = exclusiveParent; this.isSurplus = isSurplus; this.isNew = isNew; this.isResizable = isResizeable; @@ -95,6 +104,8 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat /** Returns a copy of this with exclusive switch set to given value */ public abstract NodeCandidate withExclusiveSwitch(boolean exclusiveSwitch); + public abstract NodeCandidate withExclusiveParent(boolean exclusiveParent); + /** * Returns the node instance of this candidate, allocating it if necessary. * @@ -224,7 +235,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat /** Returns a copy of this with node set to given value */ NodeCandidate withNode(Node node, boolean retiredNow) { - return new ConcreteNodeCandidate(node, retiredNow, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizable); + return new ConcreteNodeCandidate(node, retiredNow, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, exclusiveParent, isSurplus, isNew, isResizable); } /** Returns the switch priority, based on switch exclusivity, of this compared to other */ @@ -267,7 +278,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat boolean isSurplus, boolean isNew, boolean isResizeable) { - return new ConcreteNodeCandidate(node, false, freeParentCapacity, Optional.of(parent), violatesSpares, true, isSurplus, isNew, isResizeable); + return new ConcreteNodeCandidate(node, false, freeParentCapacity, Optional.of(parent), violatesSpares, true, false, isSurplus, isNew, isResizeable); } public static NodeCandidate createNewChild(NodeResources resources, @@ -276,15 +287,15 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat boolean violatesSpares, LockedNodeList allNodes, IP.Allocation.Context ipAllocationContext) { - return new VirtualNodeCandidate(resources, freeParentCapacity, parent, violatesSpares, true, allNodes, ipAllocationContext); + return new VirtualNodeCandidate(resources, freeParentCapacity, parent, violatesSpares, true, false, allNodes, ipAllocationContext); } public static NodeCandidate createNewExclusiveChild(Node node, Node parent) { - return new ConcreteNodeCandidate(node, false, node.resources(), Optional.of(parent), false, true, false, true, false); + return new ConcreteNodeCandidate(node, false, node.resources(), Optional.of(parent), false, true, false, false, true, false); } public static NodeCandidate createStandalone(Node node, boolean isSurplus, boolean isNew) { - return new ConcreteNodeCandidate(node, false, node.resources(), Optional.empty(), false, true, isSurplus, isNew, false); + return new ConcreteNodeCandidate(node, false, node.resources(), Optional.empty(), false, true, false, isSurplus, isNew, false); } /** A candidate backed by a node */ @@ -296,9 +307,9 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat ConcreteNodeCandidate(Node node, boolean retiredNow, NodeResources freeParentCapacity, Optional<Node> parent, - boolean violatesSpares, boolean exclusiveSwitch, + boolean violatesSpares, boolean exclusiveSwitch, boolean exclusiveParent, boolean isSurplus, boolean isNew, boolean isResizeable) { - super(freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizeable); + super(freeParentCapacity, parent, violatesSpares, exclusiveSwitch, exclusiveParent, isSurplus, isNew, isResizeable); this.retiredNow = retiredNow; this.node = Objects.requireNonNull(node, "Node cannot be null"); } @@ -336,7 +347,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat @Override public NodeCandidate allocate(ApplicationId owner, ClusterMembership membership, NodeResources requestedResources, Instant at) { return new ConcreteNodeCandidate(node.allocate(owner, membership, requestedResources, at), retiredNow, - freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizable); + freeParentCapacity, parent, violatesSpares, exclusiveSwitch, exclusiveParent, isSurplus, isNew, isResizable); } /** Called when the node described by this candidate must be created */ @@ -346,7 +357,13 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat @Override public NodeCandidate withExclusiveSwitch(boolean exclusiveSwitch) { return new ConcreteNodeCandidate(node, retiredNow, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, - isSurplus, isNew, isResizable); + exclusiveParent, isSurplus, isNew, isResizable); + } + + @Override + public NodeCandidate withExclusiveParent(boolean exclusiveParent) { + return new ConcreteNodeCandidate(node, retiredNow, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, + exclusiveParent, isSurplus, isNew, isResizable); } @Override @@ -387,9 +404,10 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat Node parent, boolean violatesSpares, boolean exclusiveSwitch, + boolean exclusiveParent, LockedNodeList allNodes, IP.Allocation.Context ipAllocationContext) { - super(freeParentCapacity, Optional.of(parent), violatesSpares, exclusiveSwitch, false, true, false); + super(freeParentCapacity, Optional.of(parent), violatesSpares, exclusiveSwitch, exclusiveParent, false, true, false); this.resources = resources; this.allNodes = allNodes; this.ipAllocationContext = ipAllocationContext; @@ -449,13 +467,18 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat NodeType.tenant) .cloudAccount(parent.get().cloudAccount()) .build(); - return new ConcreteNodeCandidate(node, false, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizable); + return new ConcreteNodeCandidate(node, false, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, exclusiveParent, isSurplus, isNew, isResizable); } @Override public NodeCandidate withExclusiveSwitch(boolean exclusiveSwitch) { - return new VirtualNodeCandidate(resources, freeParentCapacity, parent.get(), violatesSpares, exclusiveSwitch, allNodes, ipAllocationContext); + return new VirtualNodeCandidate(resources, freeParentCapacity, parent.get(), violatesSpares, exclusiveSwitch, exclusiveParent, allNodes, ipAllocationContext); + } + + @Override + public NodeCandidate withExclusiveParent(boolean exclusiveParent) { + return new VirtualNodeCandidate(resources, freeParentCapacity, parent.get(), violatesSpares, exclusiveSwitch, exclusiveParent, allNodes, ipAllocationContext); } @Override @@ -492,7 +515,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat private InvalidNodeCandidate(NodeResources resources, NodeResources freeParentCapacity, Node parent, String invalidReason) { - super(freeParentCapacity, Optional.of(parent), false, false, false, true, false); + super(freeParentCapacity, Optional.of(parent), false, false, false, false, true, false); this.resources = resources; this.invalidReason = invalidReason; } @@ -540,6 +563,11 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat } @Override + public NodeCandidate withExclusiveParent(boolean exclusiveParent) { + return this; + } + + @Override public Node toNode() { throw new IllegalStateException("Candidate node on " + parent.get() + " is invalid: " + invalidReason); } @@ -559,4 +587,61 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat } + public enum ExclusivityViolation { + NONE, YES, + + /** No violation IF AND ONLY IF the parent host's exclusiveToApplicationId is set to this application. */ + PARENT_HOST_NOT_EXCLUSIVE + } + + public ExclusivityViolation violatesExclusivity(ClusterSpec cluster, ApplicationId application, + boolean exclusiveAllocation, boolean exclusiveProvisioning, + boolean hostSharing, NodeList allNodes, boolean makeExclusive) { + if (parentHostname().isEmpty()) return ExclusivityViolation.NONE; + if (type() != NodeType.tenant) return ExclusivityViolation.NONE; + + if (hostSharing) { + // In zones with shared hosts we require that if any node on the host requires exclusivity, + // then all the nodes on the host must have the same owner. + for (Node nodeOnHost : allNodes.childrenOf(parentHostname().get())) { + if (nodeOnHost.allocation().isEmpty()) continue; + if (exclusiveAllocation || nodeOnHost.allocation().get().membership().cluster().isExclusive()) { + if ( ! nodeOnHost.allocation().get().owner().equals(application)) return ExclusivityViolation.YES; + } + } + } else { + // the parent is exclusive to another cluster type + if ( ! emptyOrEqual(parent.flatMap(Node::exclusiveToClusterType), cluster.type())) + return ExclusivityViolation.YES; + + // the parent is provisioned for another application + if ( ! emptyOrEqual(parent.flatMap(Node::provisionedForApplicationId), application)) + return ExclusivityViolation.YES; + + // this cluster requires a parent that was provisioned for this application + if (exclusiveProvisioning && parent.flatMap(Node::provisionedForApplicationId).isEmpty()) + return ExclusivityViolation.YES; + + // the parent is exclusive to another application + if ( ! emptyOrEqual(parent.flatMap(Node::exclusiveToApplicationId), application)) + return ExclusivityViolation.YES; + + // this cluster requires exclusivity, but the parent is not exclusive + if (exclusiveAllocation && parent.flatMap(Node::exclusiveToApplicationId).isEmpty()) + return canMakeHostExclusive(makeExclusive, type(), hostSharing) ? + ExclusivityViolation.PARENT_HOST_NOT_EXCLUSIVE : + ExclusivityViolation.YES; + } + + return ExclusivityViolation.NONE; + } + + /** + * Whether it is allowed to take a host not exclusive to anyone, and make it exclusive to an application. + * Returns false if {@code makeExclusive} is false, which can be used to guard this feature. + */ + public static boolean canMakeHostExclusive(boolean makeExclusive, NodeType type, boolean allowHostSharing) { + return makeExclusive && type == NodeType.tenant && !allowHostSharing; + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeIndices.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeIndices.java index 1ffd54872d3..49eaedaa4ec 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeIndices.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeIndices.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ClusterSpec; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index fe9f3443299..b92d6fb6d18 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; @@ -9,7 +9,6 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.Nodes; -import com.yahoo.vespa.hosted.provision.persistence.NameResolver; import java.util.ArrayList; import java.util.Collections; @@ -40,14 +39,17 @@ public class NodePrioritizer { private final IP.Allocation.Context ipAllocationContext; private final Nodes nodes; private final boolean dynamicProvisioning; + private final boolean allowHostSharing; + private final boolean exclusiveAllocation; + private final boolean makeExclusive; private final boolean canAllocateToSpareHosts; private final boolean topologyChange; private final int currentClusterSize; private final Set<Node> spareHosts; public NodePrioritizer(LockedNodeList allNodes, ApplicationId application, ClusterSpec clusterSpec, NodeSpec nodeSpec, - boolean dynamicProvisioning, IP.Allocation.Context ipAllocationContext, Nodes nodes, - HostResourcesCalculator hostResourcesCalculator, int spareCount) { + boolean dynamicProvisioning, boolean allowHostSharing, IP.Allocation.Context ipAllocationContext, Nodes nodes, + HostResourcesCalculator hostResourcesCalculator, int spareCount, boolean exclusiveAllocation, boolean makeExclusive) { this.allNodes = allNodes; this.calculator = hostResourcesCalculator; this.capacity = new HostCapacity(this.allNodes, hostResourcesCalculator); @@ -55,6 +57,9 @@ public class NodePrioritizer { this.clusterSpec = clusterSpec; this.application = application; this.dynamicProvisioning = dynamicProvisioning; + this.allowHostSharing = allowHostSharing; + this.exclusiveAllocation = exclusiveAllocation; + this.makeExclusive = makeExclusive; this.spareHosts = dynamicProvisioning ? capacity.findSpareHostsInDynamicallyProvisionedZones(this.allNodes.asList()) : capacity.findSpareHosts(this.allNodes.asList(), spareCount); @@ -122,7 +127,13 @@ public class NodePrioritizer { if (nodes.suspended(host)) continue; // Hosts that are suspended may be down for some time, e.g. for OS upgrade if (host.reservedTo().isPresent() && !host.reservedTo().get().equals(application.tenant())) continue; if (host.reservedTo().isPresent() && application.instance().isTester()) continue; - if (host.exclusiveToApplicationId().isPresent() && ! fitsPerfectly(host)) continue; + if (makeExclusive) { + if ( ! allowHostSharing && exclusiveAllocation && ! fitsPerfectly(host)) continue; + } else { + if (host.exclusiveToApplicationId().isPresent() && ! fitsPerfectly(host)) continue; + } + if ( ! host.provisionedForApplicationId().map(application::equals).orElse(true)) continue; + if ( ! host.exclusiveToApplicationId().map(application::equals).orElse(true)) continue; if ( ! host.exclusiveToClusterType().map(clusterSpec.type()::equals).orElse(true)) continue; if (spareHosts.contains(host) && !canAllocateToSpareHosts) continue; if ( ! capacity.hasCapacity(host, requested.resources().get())) continue; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 132cd0e6d67..e2f1b7358cf 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.annotation.Inject; @@ -12,10 +12,9 @@ import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.ProvisionLock; +import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; -import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; import com.yahoo.transaction.Mutex; @@ -145,7 +144,10 @@ public class NodeRepositoryProvisioner implements Provisioner { @Override public void restart(ApplicationId application, HostFilter filter) { - nodeRepository.nodes().restartActive(ApplicationFilter.from(application).and(NodeHostFilter.from(filter))); + List<Node> updated = nodeRepository.nodes().restartActive(ApplicationFilter.from(application).and(NodeHostFilter.from(filter))); + if (updated.isEmpty()) { + throw new IllegalArgumentException("No matching nodes found"); + } } @Override @@ -155,8 +157,8 @@ public class NodeRepositoryProvisioner implements Provisioner { } @Override - public ProvisionLock lock(ApplicationId application) { - return new ProvisionLock(application, nodeRepository.applications().lock(application)); + public ApplicationMutex lock(ApplicationId application) { + return new ApplicationMutex(application, nodeRepository.applications().lock(application)); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceComparator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceComparator.java index 2038390abcf..11a5ee39a3d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceComparator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceComparator.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.NodeResources; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java index ab222c45252..cd331cbd5fa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java index 77f37cadc0b..cb50f6dff2b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.CloudAccount; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index 67dae48cff7..270fda8314e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; @@ -8,9 +8,13 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.jdisc.Metric; import com.yahoo.text.internal.SnippetGenerator; import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.applicationmodel.InfrastructureApplication; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; @@ -18,6 +22,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner.HostSharing; +import com.yahoo.yolean.Exceptions; import java.util.LinkedHashSet; import java.util.List; @@ -44,12 +49,14 @@ public class Preparer { private final Optional<HostProvisioner> hostProvisioner; private final Optional<LoadBalancerProvisioner> loadBalancerProvisioner; private final ProvisioningThrottler throttler; + private final BooleanFlag makeExclusiveFlag; public Preparer(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner, Optional<LoadBalancerProvisioner> loadBalancerProvisioner, Metric metric) { this.nodeRepository = nodeRepository; this.hostProvisioner = hostProvisioner; this.loadBalancerProvisioner = loadBalancerProvisioner; this.throttler = new ProvisioningThrottler(nodeRepository, metric); + this.makeExclusiveFlag = Flags.MAKE_EXCLUSIVE.bindTo(nodeRepository.flagSource()); } /** @@ -69,11 +76,12 @@ public class Preparer { loadBalancerProvisioner.ifPresent(provisioner -> provisioner.prepare(application, cluster, requested)); + boolean makeExclusive = makeExclusiveFlag.value(); // Try preparing in memory without global unallocated lock. Most of the time there should be no changes, // and we can return nodes previously allocated. LockedNodeList allNodes = nodeRepository.nodes().list(PROBE_LOCK); NodeIndices indices = new NodeIndices(cluster.id(), allNodes); - NodeAllocation probeAllocation = prepareAllocation(application, cluster, requested, indices::probeNext, allNodes); + NodeAllocation probeAllocation = prepareAllocation(application, cluster, requested, indices::probeNext, allNodes, makeExclusive); if (probeAllocation.fulfilledAndNoChanges()) { List<Node> acceptedNodes = probeAllocation.finalNodes(); indices.commitProbe(); @@ -81,16 +89,25 @@ public class Preparer { } else { // There were some changes, so re-do the allocation with locks indices.resetProbe(); - return prepareWithLocks(application, cluster, requested, indices); + return prepareWithLocks(application, cluster, requested, indices, makeExclusive); } } + private ApplicationMutex parentLockOrNull(boolean makeExclusive, NodeType type) { + return NodeCandidate.canMakeHostExclusive(makeExclusive, type, nodeRepository.zone().cloud().allowHostSharing()) ? + nodeRepository.applications().lock(InfrastructureApplication.withNodeType(type.parentNodeType()).id()) : + null; + } + /// Note that this will write to the node repo. - private List<Node> prepareWithLocks(ApplicationId application, ClusterSpec cluster, NodeSpec requested, NodeIndices indices) { + private List<Node> prepareWithLocks(ApplicationId application, ClusterSpec cluster, NodeSpec requested, NodeIndices indices, boolean makeExclusive) { + Runnable waiter = null; + List<Node> acceptedNodes; try (Mutex lock = nodeRepository.applications().lock(application); + ApplicationMutex parentLockOrNull = parentLockOrNull(makeExclusive, requested.type()); Mutex allocationLock = nodeRepository.nodes().lockUnallocated()) { LockedNodeList allNodes = nodeRepository.nodes().list(allocationLock); - NodeAllocation allocation = prepareAllocation(application, cluster, requested, indices::next, allNodes); + NodeAllocation allocation = prepareAllocation(application, cluster, requested, indices::next, allNodes, makeExclusive); NodeType hostType = allocation.nodeType().hostType(); if (canProvisionDynamically(hostType) && allocation.hostDeficit().isPresent()) { HostSharing sharing = hostSharing(cluster, hostType); @@ -127,12 +144,13 @@ public class Preparer { requested.cloudAccount(), deficit.dueToFlavorUpgrade()); Predicate<NodeResources> realHostResourcesWithinLimits = resources -> nodeRepository.nodeResourceLimits().isWithinRealLimits(resources, application, cluster); - hostProvisioner.get().provisionHosts(request, realHostResourcesWithinLimits, whenProvisioned); + waiter = hostProvisioner.get().provisionHosts(request, realHostResourcesWithinLimits, whenProvisioned); } catch (NodeAllocationException e) { // Mark the nodes that were written to ZK in the consumer for deprovisioning. While these hosts do // not exist, we cannot remove them from ZK here because other nodes may already have been // allocated on them, so let HostDeprovisioner deal with it - hosts.forEach(host -> nodeRepository.nodes().deprovision(host.hostname(), Agent.system, nodeRepository.clock().instant())); + hosts.forEach(host -> nodeRepository.nodes().parkRecursively(host.hostname(), Agent.system, true, + "Failed to provision: " + Exceptions.toMessageString(e))); throw e; } } else if (allocation.hostDeficit().isPresent() && requested.canFail() && @@ -140,7 +158,7 @@ public class Preparer { // Non-dynamically provisioned zone with a deficit because we just now retired some nodes. // Try again, but without retiring indices.resetProbe(); - List<Node> accepted = prepareWithLocks(application, cluster, cns.withoutRetiring(), indices); + List<Node> accepted = prepareWithLocks(application, cluster, cns.withoutRetiring(), indices, makeExclusive); log.warning("Prepared " + application + " " + cluster.id() + " without retirement due to lack of capacity"); return accepted; } @@ -150,7 +168,12 @@ public class Preparer { allocation.allocationFailureDetails(), true); // Carry out and return allocation - List<Node> acceptedNodes = allocation.finalNodes(); + if (parentLockOrNull != null) { + List<Node> exclusiveParents = allocation.parentsRequiredToBeExclusive(); + nodeRepository.nodes().setExclusiveToApplicationId(exclusiveParents, parentLockOrNull); + // TODO: also update tags + } + acceptedNodes = allocation.finalNodes(); nodeRepository.nodes().reserve(allocation.reservableNodes()); nodeRepository.nodes().addReservedNodes(new LockedNodeList(allocation.newNodes(), allocationLock)); @@ -160,14 +183,16 @@ public class Preparer { .filter(node -> node.parentHostname().isEmpty() || activeHosts.parentOf(node).isPresent()) .toList(); } - return acceptedNodes; } + + if (waiter != null) waiter.run(); + return acceptedNodes; } private NodeAllocation prepareAllocation(ApplicationId application, ClusterSpec cluster, NodeSpec requested, - Supplier<Integer> nextIndex, LockedNodeList allNodes) { + Supplier<Integer> nextIndex, LockedNodeList allNodes, boolean makeExclusive) { validateAccount(requested.cloudAccount(), application, allNodes); - NodeAllocation allocation = new NodeAllocation(allNodes, application, cluster, requested, nextIndex, nodeRepository); + NodeAllocation allocation = new NodeAllocation(allNodes, application, cluster, requested, nextIndex, nodeRepository, makeExclusive); var allocationContext = IP.Allocation.Context.from(nodeRepository.zone().cloud().name(), requested.cloudAccount().isExclave(nodeRepository.zone()), nodeRepository.nameResolver()); @@ -176,10 +201,13 @@ public class Preparer { cluster, requested, nodeRepository.zone().cloud().dynamicProvisioning(), + nodeRepository.zone().cloud().allowHostSharing(), allocationContext, nodeRepository.nodes(), nodeRepository.resourcesCalculator(), - nodeRepository.spareCount()); + nodeRepository.spareCount(), + nodeRepository.exclusiveAllocation(cluster), + makeExclusive); allocation.offer(prioritizer.collect()); return allocation; } @@ -208,7 +236,9 @@ public class Preparer { private HostSharing hostSharing(ClusterSpec cluster, NodeType hostType) { if ( hostType.isSharable()) - return nodeRepository.exclusiveAllocation(cluster) ? HostSharing.exclusive : HostSharing.any; + return nodeRepository.exclusiveProvisioning(cluster) ? HostSharing.provision : + nodeRepository.exclusiveAllocation(cluster) ? HostSharing.exclusive : + HostSharing.any; else return HostSharing.any; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java index 65039aaca77..601a4008110 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.EndpointsChecker.HealthCheckerProvider; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java index 460b1bd23a0..8a84cfef09a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; @@ -30,6 +30,7 @@ public class ProvisionedHost { private final String hostHostname; private final Flavor hostFlavor; private final NodeType hostType; + private final Optional<ApplicationId> provisionedForApplicationId; private final Optional<ApplicationId> exclusiveToApplicationId; private final Optional<ClusterSpec.Type> exclusiveToClusterType; private final List<HostName> nodeHostnames; @@ -38,7 +39,9 @@ public class ProvisionedHost { private final CloudAccount cloudAccount; public ProvisionedHost(String id, String hostHostname, Flavor hostFlavor, NodeType hostType, - Optional<ApplicationId> exclusiveToApplicationId, Optional<ClusterSpec.Type> exclusiveToClusterType, + Optional<ApplicationId> provisionedForApplicationId, + Optional<ApplicationId> exclusiveToApplicationId, + Optional<ClusterSpec.Type> exclusiveToClusterType, List<HostName> nodeHostnames, NodeResources nodeResources, Version osVersion, CloudAccount cloudAccount) { if (!hostType.isHost()) throw new IllegalArgumentException(hostType + " is not a host"); @@ -46,6 +49,7 @@ public class ProvisionedHost { this.hostHostname = Objects.requireNonNull(hostHostname, "Host hostname must be set"); this.hostFlavor = Objects.requireNonNull(hostFlavor, "Host flavor must be set"); this.hostType = Objects.requireNonNull(hostType, "Host type must be set"); + this.provisionedForApplicationId = Objects.requireNonNull(provisionedForApplicationId, "provisionedForApplicationId must be set"); this.exclusiveToApplicationId = Objects.requireNonNull(exclusiveToApplicationId, "exclusiveToApplicationId must be set"); this.exclusiveToClusterType = Objects.requireNonNull(exclusiveToClusterType, "exclusiveToClusterType must be set"); this.nodeHostnames = validateNodeAddresses(nodeHostnames); @@ -67,6 +71,7 @@ public class ProvisionedHost { Node.Builder builder = Node.create(id, IP.Config.of(List.of(), List.of(), nodeHostnames), hostHostname, hostFlavor, hostType) .status(Status.initial().withOsVersion(OsVersion.EMPTY.withCurrent(Optional.of(osVersion)))) .cloudAccount(cloudAccount); + provisionedForApplicationId.ifPresent(builder::provisionedForApplicationId); exclusiveToApplicationId.ifPresent(builder::exclusiveToApplicationId); exclusiveToClusterType.ifPresent(builder::exclusiveToClusterType); if ( ! hostTTL.isZero()) builder.hostTTL(hostTTL); @@ -84,6 +89,7 @@ public class ProvisionedHost { public String hostHostname() { return hostHostname; } public Flavor hostFlavor() { return hostFlavor; } public NodeType hostType() { return hostType; } + public Optional<ApplicationId> provisionedForApplicationId() { return provisionedForApplicationId; } public Optional<ApplicationId> exclusiveToApplicationId() { return exclusiveToApplicationId; } public Optional<ClusterSpec.Type> exclusiveToClusterType() { return exclusiveToClusterType; } public List<HostName> nodeHostnames() { return nodeHostnames; } @@ -102,6 +108,7 @@ public class ProvisionedHost { hostHostname.equals(that.hostHostname) && hostFlavor.equals(that.hostFlavor) && hostType == that.hostType && + provisionedForApplicationId.equals(that.provisionedForApplicationId) && exclusiveToApplicationId.equals(that.exclusiveToApplicationId) && exclusiveToClusterType.equals(that.exclusiveToClusterType) && nodeHostnames.equals(that.nodeHostnames) && @@ -112,7 +119,7 @@ public class ProvisionedHost { @Override public int hashCode() { - return Objects.hash(id, hostHostname, hostFlavor, hostType, exclusiveToApplicationId, exclusiveToClusterType, nodeHostnames, nodeResources, osVersion, cloudAccount); + return Objects.hash(id, hostHostname, hostFlavor, hostType, provisionedForApplicationId, exclusiveToApplicationId, exclusiveToClusterType, nodeHostnames, nodeResources, osVersion, cloudAccount); } @Override @@ -122,9 +129,10 @@ public class ProvisionedHost { ", hostHostname='" + hostHostname + '\'' + ", hostFlavor=" + hostFlavor + ", hostType=" + hostType + + ", provisionedForApplicationId=" + provisionedForApplicationId + ", exclusiveToApplicationId=" + exclusiveToApplicationId + ", exclusiveToClusterType=" + exclusiveToClusterType + - ", nodeAddresses=" + nodeHostnames + + ", nodeHostnames=" + nodeHostnames + ", nodeResources=" + nodeResources + ", osVersion=" + osVersion + ", cloudAccount=" + cloudAccount + diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottler.java index 1cb7daae33f..fa5f1ff86f0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottler.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import ai.vespa.metrics.ConfigServerMetrics; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/package-info.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/package-info.java index 561f0846d14..5657543effa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/package-info.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/package-info.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage package com.yahoo.vespa.hosted.provision.provisioning; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationPatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationPatcher.java index f4022570a9b..8f705fe30d8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationPatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationPatcher.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java index 7cf284eca3b..225eb3e4e8d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.config.provision.IntRange; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveResponse.java index 84c82d314c9..35ceb7081ea 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveResponse.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.restapi.SlimeJsonResponse; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/HostCapacityResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/HostCapacityResponse.java index 54c3daa7bd7..9f7c795cf48 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/HostCapacityResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/HostCapacityResponse.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.container.jdisc.HttpRequest; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/JobsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/JobsResponse.java index dbcdb30719e..e1dcfa0ce60 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/JobsResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/JobsResponse.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.concurrent.maintenance.JobControl; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java index 20aa7d8181e..175ea8d294e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiHandler.java index a23b1273fe3..4633af27b72 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiHandler.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.container.jdisc.HttpRequest; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java index 8e4bcd6d942..2ed07a9625e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.container.jdisc.HttpResponse; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeAclResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeAclResponse.java index 6fe14715355..c672fb74a12 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeAclResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeAclResponse.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.container.jdisc.HttpRequest; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index cad034e01aa..19b9fc26fd3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi; import com.google.common.collect.Maps; @@ -263,6 +263,8 @@ public class NodePatcher { case "exclusiveTo": case "exclusiveToApplicationId": return node.withExclusiveToApplicationId(SlimeUtils.optionalString(value).map(ApplicationId::fromSerializedForm).orElse(null)); + case "provisionedFor": + return node.withProvisionedForApplicationId(SlimeUtils.optionalString(value).map(ApplicationId::fromSerializedForm).orElse(null)); case "hostTTL": return node.withHostTTL(SlimeUtils.optionalDuration(value).orElse(null)); case "hostEmptyAt": diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeResourcesSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeResourcesSerializer.java index 0f944947440..518474c3711 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeResourcesSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeResourcesSerializer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.config.provision.NodeResources; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java index 8e5fedf69ff..55331657117 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.config.provision.NodeType; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index 05bb0a27d69..73e48d6df55 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.component.Version; @@ -134,6 +134,7 @@ class NodesResponse extends SlimeJsonResponse { object.setString("flavor", node.flavor().name()); node.reservedTo().ifPresent(reservedTo -> object.setString("reservedTo", reservedTo.value())); node.exclusiveToApplicationId().ifPresent(applicationId -> object.setString("exclusiveTo", applicationId.serializedForm())); + node.provisionedForApplicationId().ifPresent(applicationId -> object.setString("provisionedFor", applicationId.serializedForm())); node.hostTTL().ifPresent(ttl -> object.setLong("hostTTL", ttl.toMillis())); node.hostEmptyAt().ifPresent(emptyAt -> object.setLong("hostEmptyAt", emptyAt.toEpochMilli())); node.exclusiveToClusterType().ifPresent(clusterType -> object.setString("exclusiveToClusterType", clusterType.name())); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index 79dc7fe72a8..9080030f026 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.component.Version; @@ -9,6 +9,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.InfraDeployer; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; @@ -44,6 +45,7 @@ import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeOsVersionFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeTypeFilter; import com.yahoo.vespa.hosted.provision.node.filter.ParentHostFilter; +import com.yahoo.vespa.hosted.provision.maintenance.InfraApplicationRedeployer; import com.yahoo.vespa.hosted.provision.restapi.NodesResponse.ResponseType; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.yolean.Exceptions; @@ -75,13 +77,16 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { private final Orchestrator orchestrator; private final NodeRepository nodeRepository; private final NodeFlavors nodeFlavors; + private final InfraApplicationRedeployer infraApplicationRedeployer; @Inject - public NodesV2ApiHandler(Context parentCtx, Orchestrator orchestrator, NodeRepository nodeRepository, NodeFlavors flavors) { + public NodesV2ApiHandler(Context parentCtx, Orchestrator orchestrator, NodeRepository nodeRepository, + NodeFlavors flavors, InfraDeployer infraDeployer) { super(parentCtx); this.orchestrator = orchestrator; this.nodeRepository = nodeRepository; this.nodeFlavors = flavors; + this.infraApplicationRedeployer = new InfraApplicationRedeployer(infraDeployer, nodeRepository); } @Override @@ -138,7 +143,8 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { Path path = new Path(request.getUri()); // Check paths to disallow illegal state changes if (path.matches("/nodes/v2/state/ready/{hostname}")) { - nodeRepository.nodes().markNodeAvailableForNewAllocation(path.get("hostname"), agent(request), "Readied through the nodes/v2 API"); + if (nodeRepository.nodes().markNodeAvailableForNewAllocation(path.get("hostname"), agent(request), "Readied through the nodes/v2 API")) + infraApplicationRedeployer.readied(nodeRepository.nodes().node(path.get("hostname")).get().type()); return new MessageResponse("Moved " + path.get("hostname") + " to " + Node.State.ready); } else if (path.matches("/nodes/v2/state/failed/{hostname}")) { @@ -147,7 +153,7 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { " and marked " + hostnamesAsString(failedOrMarkedNodes.failing().asList()) + " as wantToFail"); } else if (path.matches("/nodes/v2/state/parked/{hostname}")) { - List<Node> parkedNodes = nodeRepository.nodes().parkRecursively(path.get("hostname"), agent(request), "Parked through the nodes/v2 API"); + List<Node> parkedNodes = nodeRepository.nodes().parkRecursively(path.get("hostname"), agent(request), false, "Parked through the nodes/v2 API"); return new MessageResponse("Moved " + hostnamesAsString(parkedNodes) + " to " + Node.State.parked); } else if (path.matches("/nodes/v2/state/dirty/{hostname}")) { @@ -299,6 +305,7 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { optionalString(inspector.field("parentHostname")).ifPresent(builder::parentHostname); optionalString(inspector.field("modelName")).ifPresent(builder::modelName); optionalString(inspector.field("reservedTo")).map(TenantName::from).ifPresent(builder::reservedTo); + optionalString(inspector.field("provisionedFor")).map(ApplicationId::fromSerializedForm).ifPresent(builder::provisionedForApplicationId); optionalString(inspector.field("exclusiveTo")).map(ApplicationId::fromSerializedForm).ifPresent(builder::exclusiveToApplicationId); optionalString(inspector.field("switchHostname")).ifPresent(builder::switchHostname); return builder.build(); @@ -507,4 +514,10 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { } } + @Override + public void destroy() { + super.destroy(); + infraApplicationRedeployer.close(); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NotFoundException.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NotFoundException.java index 5423562cfa6..08baa82a5d4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NotFoundException.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NotFoundException.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi; /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/UpgradeResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/UpgradeResponse.java index e03edecff5e..f7e014e276c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/UpgradeResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/UpgradeResponse.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.container.jdisc.HttpResponse; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/WireguardResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/WireguardResponse.java index e29c4f1b87a..3ffcb2f90c0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/WireguardResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/WireguardResponse.java @@ -1,3 +1,4 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.config.provision.NodeType; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java index 4f88a10dff0..f653416d973 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; import com.yahoo.config.provision.CloudAccount; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/InMemoryProvisionLogger.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/InMemoryProvisionLogger.java index 7ded74b7451..4b07d61b411 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/InMemoryProvisionLogger.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/InMemoryProvisionLogger.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; import com.yahoo.config.provision.ProvisionLogger; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java index 0573c6a877e..1e3c74af247 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; import com.yahoo.component.Version; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java index 06ce08df8c3..9bdfa8a4a91 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; import ai.vespa.http.DomainName; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java index a72c2fb0b9c..b5bb91af71a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; import com.yahoo.config.provision.CloudAccount; @@ -73,13 +73,13 @@ public class MockHostProvisioner implements HostProvisioner { } @Override - public void provisionHosts(HostProvisionRequest request, Predicate<NodeResources> realHostResourcesWithinLimits, Consumer<List<ProvisionedHost>> whenProvisioned) throws NodeAllocationException { + public Runnable provisionHosts(HostProvisionRequest request, Predicate<NodeResources> realHostResourcesWithinLimits, Consumer<List<ProvisionedHost>> whenProvisioned) throws NodeAllocationException { if (behaviour(Behaviour.failProvisionRequest)) throw new NodeAllocationException("No capacity for provision request", true); Flavor hostFlavor = hostFlavors.get(request.clusterType().orElse(ClusterSpec.Type.content)); if (hostFlavor == null) hostFlavor = flavors.stream() - .filter(f -> request.sharing() == HostSharing.exclusive ? compatible(f, request.resources()) - : satisfies(f, request.resources())) + .filter(f -> request.sharing().isExclusiveAllocation() ? compatible(f, request.resources()) + : satisfies(f, request.resources())) .filter(f -> realHostResourcesWithinLimits.test(f.resources())) .findFirst() .orElseThrow(() -> new NodeAllocationException("No host flavor matches " + request.resources(), true)); @@ -91,7 +91,8 @@ public class MockHostProvisioner implements HostProvisioner { hostHostname, hostFlavor, request.type(), - request.sharing() == HostSharing.exclusive ? Optional.of(request.owner()) : Optional.empty(), + request.sharing() == HostSharing.provision ? Optional.of(request.owner()) : Optional.empty(), + request.sharing().isExclusiveAllocation() ? Optional.of(request.owner()) : Optional.empty(), Optional.empty(), createHostnames(request.type(), hostFlavor, index), request.resources(), @@ -100,6 +101,7 @@ public class MockHostProvisioner implements HostProvisioner { } provisionedHosts.addAll(hosts); whenProvisioned.accept(hosts); + return () -> {}; } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java index 70c658be154..d1809321704 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockMetricsFetcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockMetricsFetcher.java index 6c6face7ba6..28ff0dcad7e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockMetricsFetcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockMetricsFetcher.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java index 722dc5ef96c..5f0ac4fbcdb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; import com.yahoo.vespa.hosted.provision.node.IP; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeFlavors.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeFlavors.java index 2a78e4081e3..835d56ca075 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeFlavors.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeFlavors.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; import com.yahoo.config.provision.Flavor; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 2fb549acc11..9a6ab16b4fc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; import com.yahoo.component.Version; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java index 81947251e64..2fb458fc04c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; import com.yahoo.component.annotation.Inject; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java index d8ad892e210..e130f53fab5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; import com.yahoo.config.provision.ActivationContext; @@ -8,7 +8,7 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; -import com.yahoo.config.provision.ProvisionLock; +import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; @@ -36,7 +36,7 @@ public class MockProvisioner implements Provisioner { public void restart(ApplicationId application, HostFilter filter) { } @Override - public ProvisionLock lock(ApplicationId application) { + public ApplicationMutex lock(ApplicationId application) { return null; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java index 4247e2bcec5..b818b54070c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; import com.yahoo.component.AbstractComponent; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/README.md b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/README.md index 59e9f4a03c8..fe7e78afa96 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/README.md +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/README.md @@ -1,4 +1,4 @@ -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<!-- Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> The test resources are used by both NodeAdmin and NodeRepository tests to verify APIs. So when modifying this test data remember to check tests for both NodeAdmin and NodeRepository. diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ServiceMonitorStub.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ServiceMonitorStub.java index e0b8fcc5feb..5409061c441 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ServiceMonitorStub.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ServiceMonitorStub.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; import com.yahoo.component.annotation.Inject; |