diff options
21 files changed, 550 insertions, 227 deletions
diff --git a/dist/vespa.spec b/dist/vespa.spec index 94ced88b9fb..26310c19750 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -53,7 +53,7 @@ BuildRequires: llvm7.0-devel BuildRequires: vespa-boost-devel >= 1.59.0-6 BuildRequires: vespa-gtest >= 1.8.1-1 BuildRequires: vespa-protobuf-devel >= 3.7.0-4 -BuildRequires: vespa-openssl-devel >= 1.1.1c-1 +BuildRequires: vespa-openssl-devel >= 1.1.1g-1 BuildRequires: vespa-icu-devel >= 65.1.0-1 %endif %if 0%{?el8} @@ -156,7 +156,7 @@ Requires: gdb Requires: net-tools %if 0%{?el7} Requires: llvm7.0 -Requires: vespa-openssl >= 1.1.1c-1 +Requires: vespa-openssl >= 1.1.1g-1 Requires: vespa-icu >= 65.1.0-1 Requires: vespa-protobuf >= 3.7.0-4 Requires: vespa-telegraf >= 1.1.1-1 @@ -235,7 +235,7 @@ Requires: xxhash-libs >= 0.7.3 Requires: lz4 Requires: libzstd %if 0%{?el7} -Requires: vespa-openssl >= 1.1.1c-1 +Requires: vespa-openssl >= 1.1.1g-1 %else Requires: openssl-libs %endif diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 40bccf22434..6044bce7610 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -124,14 +124,14 @@ public class NodeRepository extends AbstractComponent { this.firmwareChecks = new FirmwareChecks(db, clock); this.dockerImages = new DockerImages(db, dockerImage); this.jobControl = new JobControl(db); - this.applications = new Applications(); + this.applications = new Applications(db); // read and write all nodes to make sure they are stored in the latest version of the serialized format for (State state : State.values()) // TODO(mpolden): Add per-node locking. In its current state this may collide with other callers making // node state changes. Example: A redeployment on another config server which moves a node // to another state while this is constructed. - db.writeTo(state, db.getNodes(state), Agent.system, Optional.empty()); + db.writeTo(state, db.readNodes(state), Agent.system, Optional.empty()); } /** Returns the curator database client used by this */ @@ -171,7 +171,7 @@ public class NodeRepository extends AbstractComponent { * @return the node, or empty if it was not found in any of the given states */ public Optional<Node> getNode(String hostname, State ... inState) { - return db.getNode(hostname, inState); + return db.readNode(hostname, inState); } /** @@ -181,7 +181,7 @@ public class NodeRepository extends AbstractComponent { * @return the node, or empty if it was not found in any of the given states */ public List<Node> getNodes(State ... inState) { - return new ArrayList<>(db.getNodes(inState)); + return new ArrayList<>(db.readNodes(inState)); } /** * Finds and returns the nodes of the given type in any of the given states. @@ -191,7 +191,7 @@ public class NodeRepository extends AbstractComponent { * @return the node, or empty if it was not found in any of the given states */ public List<Node> getNodes(NodeType type, State ... inState) { - return db.getNodes(inState).stream().filter(node -> node.type().equals(type)).collect(Collectors.toList()); + return db.readNodes(inState).stream().filter(node -> node.type().equals(type)).collect(Collectors.toList()); } /** Returns a filterable list of all nodes in this repository */ @@ -223,9 +223,9 @@ public class NodeRepository extends AbstractComponent { return LoadBalancerList.copyOf(db.readLoadBalancers(predicate).values()); } - public List<Node> getNodes(ApplicationId id, State ... inState) { return db.getNodes(id, inState); } - public List<Node> getInactive() { return db.getNodes(State.inactive); } - public List<Node> getFailed() { return db.getNodes(State.failed); } + public List<Node> getNodes(ApplicationId id, State ... inState) { return db.readNodes(id, inState); } + public List<Node> getInactive() { return db.readNodes(State.inactive); } + public List<Node> getFailed() { return db.readNodes(State.failed); } /** * Returns the ACL for the node (trusted nodes, networks and ports) @@ -450,7 +450,8 @@ public class NodeRepository extends AbstractComponent { public void deactivate(ApplicationId application, NestedTransaction transaction) { try (Mutex lock = lock(application)) { - deactivate(db.getNodes(application, State.reserved, State.active), transaction); + deactivate(db.readNodes(application, State.reserved, State.active), transaction); + applications.remove(application, transaction, lock); } } @@ -736,7 +737,7 @@ public class NodeRepository extends AbstractComponent { ListMap<ApplicationId, Node> allocatedNodes = new ListMap<>(); // Group matching nodes by the lock needed - for (Node node : db.getNodes()) { + for (Node node : db.readNodes()) { if ( ! filter.matches(node)) continue; if (node.allocation().isPresent()) allocatedNodes.put(node.allocation().get().owner(), node); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java index e56e426b499..e90a5f132ed 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java @@ -1,13 +1,14 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.applications; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.transaction.Mutex; - +import java.util.Collection; import java.util.Map; import java.util.HashMap; import java.util.Optional; +import java.util.stream.Collectors; /** * The node repository's view of an application deployment. @@ -18,23 +19,30 @@ import java.util.Optional; */ public class Application { + private final ApplicationId id; private final Map<ClusterSpec.Id, Cluster> clusters; - public Application() { - this(Map.of()); + public Application(ApplicationId id) { + this(id, Map.of()); + } + + public Application(ApplicationId id, Collection<Cluster> clusters) { + this(id, clusters.stream().collect(Collectors.toMap(c -> c.id(), c -> c))); } - private Application(Map<ClusterSpec.Id, Cluster> clusters) { - this.clusters = Map.copyOf(clusters); + private Application(ApplicationId id, Map<ClusterSpec.Id, Cluster> clusters) { + this.id = id; + this.clusters = clusters; } - /** Returns the cluster with the given id or null if none */ - public Cluster cluster(ClusterSpec.Id id) { return clusters.get(id); } + public ApplicationId id() { return id; } - public Application with(ClusterSpec.Id id, Cluster cluster) { + public Map<ClusterSpec.Id, Cluster> clusters() { return clusters; } + + public Application with(Cluster cluster) { Map<ClusterSpec.Id, Cluster> clusters = new HashMap<>(this.clusters); - clusters.put(id, cluster); - return new Application(clusters); + clusters.put(cluster.id(), cluster); + return new Application(id, clusters); } /** @@ -43,7 +51,11 @@ public class Application { */ public Application withClusterLimits(ClusterSpec.Id id, ClusterResources min, ClusterResources max) { Cluster cluster = clusters.get(id); - return with(id, new Cluster(min, max, cluster == null ? Optional.empty() : cluster.targetResources())); + if (cluster == null) + cluster = new Cluster(id, min, max, Optional.empty()); + else + cluster = cluster.withLimits(min, max); + return with(cluster); } /** @@ -53,7 +65,24 @@ public class Application { public Application withClusterTarget(ClusterSpec.Id id, ClusterResources target) { Cluster cluster = clusters.get(id); if (cluster == null) return this; - return with(id, cluster.withTarget(target)); + return with(cluster.withTarget(target)); + } + + @Override + public int hashCode() { + return id.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == this) return true; + if ( ! (other instanceof Application)) return false; + return ((Application)other).id().equals(this.id()); + } + + @Override + public String toString() { + return "application '" + id + "'"; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java index 879fcc5f6cb..17f8d73c88f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java @@ -3,8 +3,11 @@ package com.yahoo.vespa.hosted.provision.applications; import com.yahoo.config.provision.ApplicationId; import com.yahoo.transaction.Mutex; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; -import java.util.concurrent.ConcurrentHashMap; +import java.util.List; +import java.util.Optional; /** * An (in-memory, for now) repository of the node repo's view of applications. @@ -15,15 +18,38 @@ import java.util.concurrent.ConcurrentHashMap; */ public class Applications { - private final ConcurrentHashMap<ApplicationId, Application> applications = new ConcurrentHashMap<>(); + private final CuratorDatabaseClient db; - /** Returns the application with the given id, or null if it does not exist and should not be created */ - public Application get(ApplicationId applicationId, boolean create) { - return applications.computeIfAbsent(applicationId, id -> create ? new Application() : null); + public Applications(CuratorDatabaseClient db) { + this.db = db; + // read and write all to make sure they are stored in the latest version of the serialized format + for (ApplicationId id : ids()) { + try (Mutex lock = db.lock(id)) { + get(id).ifPresent(application -> put(application, lock)); + } + } } - public void set(ApplicationId id, Application application, Mutex applicationLock) { - applications.put(id, application); + /** Returns the ids of all applications */ + public List<ApplicationId> ids() { return db.readApplicationIds(); } + + /** Returns the application with the given id, or empty if it does not exist */ + public Optional<Application> get(ApplicationId id) { + return db.readApplication(id); + } + + public void put(Application application, Mutex applicationLock) { + NestedTransaction transaction = new NestedTransaction(); + put(application, transaction, applicationLock); + transaction.commit(); + } + + public void put(Application application, NestedTransaction transaction, Mutex applicationLock) { + db.writeApplication(application, transaction); + } + + public void remove(ApplicationId application, NestedTransaction transaction, Mutex applicationLock) { + db.deleteApplication(application, transaction); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index 6ff7f41be8f..914bc6c5a48 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -2,7 +2,9 @@ package com.yahoo.vespa.hosted.provision.applications; import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import java.util.Objects; import java.util.Optional; @@ -16,10 +18,15 @@ import java.util.Optional; */ public class Cluster { + private final ClusterSpec.Id id; private final ClusterResources min, max; private final Optional<ClusterResources> target; - Cluster(ClusterResources minResources, ClusterResources maxResources, Optional<ClusterResources> targetResources) { + public Cluster(ClusterSpec.Id id, + ClusterResources minResources, + ClusterResources maxResources, + Optional<ClusterResources> targetResources) { + this.id = Objects.requireNonNull(id); this.min = Objects.requireNonNull(minResources); this.max = Objects.requireNonNull(maxResources); Objects.requireNonNull(targetResources); @@ -30,6 +37,8 @@ public class Cluster { this.target = targetResources; } + public ClusterSpec.Id id() { return id; } + /** Returns the configured minimal resources in this cluster */ public ClusterResources minResources() { return min; } @@ -42,12 +51,16 @@ public class Cluster { */ public Optional<ClusterResources> targetResources() { return target; } + public Cluster withLimits(ClusterResources min, ClusterResources max) { + return new Cluster(id, min, max, target); + } + public Cluster withTarget(ClusterResources target) { - return new Cluster(min, max, Optional.of(target)); + return new Cluster(id, min, max, Optional.of(target)); } public Cluster withoutTarget() { - return new Cluster(min, max, Optional.empty()); + return new Cluster(id, min, max, Optional.empty()); } public NodeResources capAtLimits(NodeResources resources) { @@ -63,4 +76,19 @@ public class Cluster { return value; } + @Override + public int hashCode() { return id.hashCode(); } + + @Override + public boolean equals(Object other) { + if (other == this) return true; + if ( ! (other instanceof Cluster)) return false; + return ((Cluster)other).id().equals(this.id); + } + + @Override + public String toString() { + return "cluster '" + id + "'"; + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java index 5ca09ddf51c..2414bd95b85 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java @@ -90,7 +90,7 @@ public class AllocatableClusterResources { public double cost() { return nodes * costOf(advertisedResources); } /** - * Returns the fraction measuring how well the real resources fulfils the ideal: 1 means completely fulfiled, + * Returns the fraction measuring how well the real resources fulfils the ideal: 1 means completely fulfilled, * 0 means we have zero real resources. * The real may be short of the ideal due to resource limits imposed by the system or application. */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index 0d7d10663c5..d17577af135 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -66,8 +66,8 @@ public class AutoscalingMaintainer extends Maintainer { ClusterSpec.Id clusterId, List<Node> clusterNodes, MaintenanceDeployment deployment) { - Application application = nodeRepository().applications().get(applicationId, true); - Cluster cluster = application.cluster(clusterId); + Application application = nodeRepository().applications().get(applicationId).orElse(new Application(applicationId)); + Cluster cluster = application.clusters().get(clusterId); if (cluster == null) return; // no information on limits for this cluster Optional<AllocatableClusterResources> target = autoscaler.autoscale(cluster, clusterNodes); if (target.isEmpty()) return; // current resources are fine @@ -77,17 +77,15 @@ public class AutoscalingMaintainer extends Maintainer { } else { logAutoscaling("Autoscaling ", target.get(), applicationId, clusterId, clusterNodes); - autoscaleTo(target.get(), applicationId, clusterId, application, deployment); + autoscaleTo(target.get(), clusterId, application, deployment); } } private void autoscaleTo(AllocatableClusterResources target, - ApplicationId applicationId, ClusterSpec.Id clusterId, Application application, MaintenanceDeployment deployment) { - nodeRepository().applications().set(applicationId, - application.withClusterTarget(clusterId, target.toAdvertisedClusterResources()), + nodeRepository().applications().put(application.withClusterTarget(clusterId, target.toAdvertisedClusterResources()), deployment.applicationLock().get()); deployment.activate(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index 7beb717ea8b..d7a93288d2d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -121,12 +121,10 @@ public class LoadBalancerExpirer extends Maintainer { private void withLoadBalancersIn(LoadBalancer.State state, Consumer<LoadBalancer> operation) { for (var id : db.readLoadBalancerIds()) { try (var lock = db.lockConfig(id.application())) { - try (var legacyLock = db.lockLoadBalancers(id.application())) { - var loadBalancer = db.readLoadBalancer(id); - if (loadBalancer.isEmpty()) continue; // Load balancer was removed during loop - if (loadBalancer.get().state() != state) continue; // Wrong state - operation.accept(loadBalancer.get()); - } + var loadBalancer = db.readLoadBalancer(id); + if (loadBalancer.isEmpty()) continue; // Load balancer was removed during loop + if (loadBalancer.get().state() != state) continue; // Wrong state + operation.accept(loadBalancer.get()); } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java index 6e8af120b3b..65a24ea0ec8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java @@ -91,7 +91,6 @@ public class Rebalancer extends Maintainer { for (Node node : allNodes.nodeType(NodeType.tenant).state(Node.State.active)) { if (node.parentHostname().isEmpty()) continue; if (node.allocation().get().owner().instance().isTester()) continue; - if (node.allocation().get().owner().application().value().equals("lsbe-dictionaries")) continue; // TODO: Remove for (Node toHost : allNodes.filter(nodeRepository()::canAllocateTenantNodeTo)) { if (toHost.hostname().equals(node.parentHostname().get())) continue; if ( ! capacity.freeCapacityOf(toHost).satisfies(node.flavor().resources())) continue; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java new file mode 100644 index 00000000000..f514cc20fe8 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java @@ -0,0 +1,111 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.ObjectTraverser; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.applications.Cluster; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; + +/** + * Application JSON serializer + * + * @author bratseth + */ +public class ApplicationSerializer { + + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + + private static final String idKey = "id"; + private static final String clustersKey = "clusters"; + private static final String minResourcesKey = "min"; + private static final String maxResourcesKey = "max"; + private static final String targetResourcesKey = "target"; + private static final String nodesKey = "nodes"; + private static final String groupsKey = "groups"; + private static final String nodeResourcesKey = "resources"; + + public static byte[] toJson(Application application) { + Slime slime = new Slime(); + toSlime(application, slime.setObject()); + try { + return SlimeUtils.toJsonBytes(slime); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static Application fromJson(byte[] data) { + return applicationFromSlime(SlimeUtils.jsonToSlime(data).get()); + } + + // --------------------------------------------------------------------------------------- + + private static void toSlime(Application application, Cursor object) { + object.setString(idKey, application.id().serializedForm()); + clustersToSlime(application.clusters().values(), object.setObject(clustersKey)); + } + + private static Application applicationFromSlime(Inspector applicationObject) { + ApplicationId id = ApplicationId.fromSerializedForm(applicationObject.field(idKey).asString()); + return new Application(id, clustersFromSlime(applicationObject.field(clustersKey))); + } + + private static void clustersToSlime(Collection<Cluster> clusters, Cursor clustersObject) { + clusters.forEach(cluster -> toSlime(cluster, clustersObject.setObject(cluster.id().value()))); + } + + private static Collection<Cluster> clustersFromSlime(Inspector clustersObject) { + List<Cluster> clusters = new ArrayList<>(); + clustersObject.traverse((ObjectTraverser)(id, clusterObject) -> clusters.add(clusterFromSlime(id, clusterObject))); + return clusters; + } + + private static void toSlime(Cluster cluster, Cursor clusterObject) { + toSlime(cluster.minResources(), clusterObject.setObject(minResourcesKey)); + toSlime(cluster.maxResources(), clusterObject.setObject(maxResourcesKey)); + cluster.targetResources().ifPresent(target -> toSlime(target, clusterObject.setObject(targetResourcesKey))); + } + + private static Cluster clusterFromSlime(String id, Inspector clusterObject) { + return new Cluster(ClusterSpec.Id.from(id), + clusterResourcesFromSlime(clusterObject.field(minResourcesKey)), + clusterResourcesFromSlime(clusterObject.field(maxResourcesKey)), + optionalClusterResourcesFromSlime(clusterObject.field(targetResourcesKey))); + } + + private static void toSlime(ClusterResources resources, Cursor clusterResourcesObject) { + clusterResourcesObject.setLong(nodesKey, resources.nodes()); + clusterResourcesObject.setLong(groupsKey, resources.groups()); + NodeResourcesSerializer.toSlime(resources.nodeResources(), clusterResourcesObject.setObject(nodeResourcesKey)); + } + + private static ClusterResources clusterResourcesFromSlime(Inspector clusterResourcesObject) { + return new ClusterResources((int)clusterResourcesObject.field(nodesKey).asLong(), + (int)clusterResourcesObject.field(groupsKey).asLong(), + NodeResourcesSerializer.resourcesFromSlime(clusterResourcesObject.field(nodeResourcesKey))); + } + + private static Optional<ClusterResources> optionalClusterResourcesFromSlime(Inspector clusterResourcesObject) { + return clusterResourcesObject.valid() ? Optional.of(clusterResourcesFromSlime(clusterResourcesObject)) + : Optional.empty(); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 6036cc2366f..af4e054579e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.curator.recipes.CuratorCounter; import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -61,9 +62,16 @@ public class CuratorDatabaseClient { private static final Logger log = Logger.getLogger(CuratorDatabaseClient.class.getName()); private static final Path root = Path.fromString("/provision/v1"); - private static final Path lockRoot = root.append("locks"); - private static final Path configLockRoot = Path.fromString("/config/v2/locks/"); - private static final Path loadBalancersRoot = root.append("loadBalancers"); + private static final Path lockPath = root.append("locks"); + private static final Path configLockPath = Path.fromString("/config/v2/locks/"); + private static final Path loadBalancersPath = root.append("loadBalancers"); + private static final Path applicationsPath = root.append("applications"); + private static final Path inactiveJobsPath = root.append("inactiveJobs"); + private static final Path infrastructureVersionsPath = root.append("infrastructureVersions"); + private static final Path osVersionsPath = root.append("osVersions"); + private static final Path dockerImagesPath = root.append("dockerImages"); + private static final Path firmwareCheckPath = root.append("firmwareCheck"); + private static final Duration defaultLockTimeout = Duration.ofMinutes(2); private final NodeSerializer nodeSerializer; @@ -90,12 +98,13 @@ public class CuratorDatabaseClient { curatorDatabase.create(root); for (Node.State state : Node.State.values()) curatorDatabase.create(toPath(state)); - curatorDatabase.create(inactiveJobsPath()); - curatorDatabase.create(infrastructureVersionsPath()); - curatorDatabase.create(osVersionsPath()); - curatorDatabase.create(dockerImagesPath()); - curatorDatabase.create(firmwareCheckPath()); - curatorDatabase.create(loadBalancersRoot); + curatorDatabase.create(applicationsPath); + curatorDatabase.create(inactiveJobsPath); + curatorDatabase.create(infrastructureVersionsPath); + curatorDatabase.create(osVersionsPath); + curatorDatabase.create(dockerImagesPath); + curatorDatabase.create(firmwareCheckPath); + curatorDatabase.create(loadBalancersPath); provisionIndexCounter.initialize(100); } @@ -256,14 +265,14 @@ public class CuratorDatabaseClient { * Returns all nodes which are in one of the given states. * If no states are given this returns all nodes. */ - public List<Node> getNodes(Node.State ... states) { + public List<Node> readNodes(Node.State ... states) { List<Node> nodes = new ArrayList<>(); if (states.length == 0) states = Node.State.values(); CuratorDatabase.Session session = curatorDatabase.getSession(); for (Node.State state : states) { for (String hostname : session.getChildren(toPath(state))) { - Optional<Node> node = getNode(session, hostname, state); + Optional<Node> node = readNode(session, hostname, state); node.ifPresent(nodes::add); // node might disappear between getChildren and getNode } } @@ -274,8 +283,8 @@ public class CuratorDatabaseClient { * Returns all nodes allocated to the given application which are in one of the given states * If no states are given this returns all nodes. */ - public List<Node> getNodes(ApplicationId applicationId, Node.State ... states) { - List<Node> nodes = getNodes(states); + public List<Node> readNodes(ApplicationId applicationId, Node.State ... states) { + List<Node> nodes = readNodes(states); nodes.removeIf(node -> ! node.allocation().isPresent() || ! node.allocation().get().owner().equals(applicationId)); return nodes; } @@ -284,7 +293,7 @@ public class CuratorDatabaseClient { * Returns a particular node, or empty if this noe is not in any of the given states. * If no states are given this returns the node if it is present in any state. */ - public Optional<Node> getNode(CuratorDatabase.Session session, String hostname, Node.State ... states) { + public Optional<Node> readNode(CuratorDatabase.Session session, String hostname, Node.State ... states) { if (states.length == 0) states = Node.State.values(); for (Node.State state : states) { @@ -299,8 +308,8 @@ public class CuratorDatabaseClient { * Returns a particular node, or empty if this noe is not in any of the given states. * If no states are given this returns the node if it is present in any state. */ - public Optional<Node> getNode(String hostname, Node.State ... states) { - return getNode(curatorDatabase.getSession(), hostname, states); + public Optional<Node> readNode(String hostname, Node.State ... states) { + return readNode(curatorDatabase.getSession(), hostname, states); } private Path toPath(Node.State nodeState) { return root.append(toDir(nodeState)); } @@ -316,7 +325,7 @@ public class CuratorDatabaseClient { /** Creates and returns the path to the lock for this application */ private Path lockPath(ApplicationId application) { Path lockPath = - lockRoot + CuratorDatabaseClient.lockPath .append(application.tenant().value()) .append(application.application().value()) .append(application.instance().value()); @@ -327,7 +336,7 @@ public class CuratorDatabaseClient { /** Creates and returns the path to the config server lock for this application */ private Path configLockPath(ApplicationId application) { // This must match the lock path used by com.yahoo.vespa.config.server.application.TenantApplications - Path lockPath = configLockRoot.append(application.tenant().value()).append(application.serializedForm()); + Path lockPath = configLockPath.append(application.tenant().value()).append(application.serializedForm()); curatorDatabase.create(lockPath); return lockPath; } @@ -349,7 +358,7 @@ public class CuratorDatabaseClient { /** Acquires the single cluster-global, reentrant lock for all non-active nodes */ public Lock lockInactive() { - return lock(lockRoot.append("unallocatedLock"), defaultLockTimeout); + return lock(lockPath.append("unallocatedLock"), defaultLockTimeout); } /** Acquires the single cluster-global, reentrant lock for active nodes of this application */ @@ -389,22 +398,43 @@ public class CuratorDatabaseClient { } } - private Lock lock(Path path, Duration timeout) { - return curatorDatabase.lock(path, timeout); + // Applications ----------------------------------------------------------- + + public List<ApplicationId> readApplicationIds() { + return curatorDatabase.getChildren(applicationsPath).stream() + .map(path -> ApplicationId.fromSerializedForm(path)) + .collect(Collectors.toList()); } - private <T> Optional<T> read(Path path, Function<byte[], T> mapper) { - return curatorDatabase.getData(path).filter(data -> data.length > 0).map(mapper); + public Optional<Application> readApplication(ApplicationId id) { + return read(applicationPath(id), ApplicationSerializer::fromJson); + } + + public void writeApplication(Application application, NestedTransaction transaction) { + curatorDatabase.newCuratorTransactionIn(transaction) + .add(createOrSet(applicationPath(application.id()), + ApplicationSerializer.toJson(application))); + } + + public void deleteApplication(ApplicationId application, NestedTransaction transaction) { + if (curatorDatabase.exists(applicationPath(application))) + curatorDatabase.newCuratorTransactionIn(transaction) + .add(CuratorOperations.delete(applicationPath(application).getAbsolute())); } - // Maintenance jobs + private Path applicationPath(ApplicationId id) { + return applicationsPath.append(id.serializedForm()); + } + + // Maintenance jobs ----------------------------------------------------------- + public Lock lockMaintenanceJob(String jobName) { - return lock(lockRoot.append("maintenanceJobLocks").append(jobName), defaultLockTimeout); + return lock(lockPath.append("maintenanceJobLocks").append(jobName), defaultLockTimeout); } public Set<String> readInactiveJobs() { try { - return read(inactiveJobsPath(), stringSetSerializer::fromJson).orElseGet(HashSet::new); + return read(inactiveJobsPath, stringSetSerializer::fromJson).orElseGet(HashSet::new); } catch (RuntimeException e) { log.log(Level.WARNING, "Error reading inactive jobs, deleting inactive state"); @@ -416,108 +446,88 @@ public class CuratorDatabaseClient { public void writeInactiveJobs(Set<String> inactiveJobs) { NestedTransaction transaction = new NestedTransaction(); CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); - curatorTransaction.add(CuratorOperations.setData(inactiveJobsPath().getAbsolute(), + curatorTransaction.add(CuratorOperations.setData(inactiveJobsPath.getAbsolute(), stringSetSerializer.toJson(inactiveJobs))); transaction.commit(); } public Lock lockInactiveJobs() { - return lock(lockRoot.append("inactiveJobsLock"), defaultLockTimeout); - } - - private Path inactiveJobsPath() { - return root.append("inactiveJobs"); + return lock(lockPath.append("inactiveJobsLock"), defaultLockTimeout); } + // Infrastructure versions ----------------------------------------------------------- - // Infrastructure versions public Map<NodeType, Version> readInfrastructureVersions() { - return read(infrastructureVersionsPath(), NodeTypeVersionsSerializer::fromJson).orElseGet(TreeMap::new); + return read(infrastructureVersionsPath, NodeTypeVersionsSerializer::fromJson).orElseGet(TreeMap::new); } public void writeInfrastructureVersions(Map<NodeType, Version> infrastructureVersions) { NestedTransaction transaction = new NestedTransaction(); CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); - curatorTransaction.add(CuratorOperations.setData(infrastructureVersionsPath().getAbsolute(), + curatorTransaction.add(CuratorOperations.setData(infrastructureVersionsPath.getAbsolute(), NodeTypeVersionsSerializer.toJson(infrastructureVersions))); transaction.commit(); } public Lock lockInfrastructureVersions() { - return lock(lockRoot.append("infrastructureVersionsLock"), defaultLockTimeout); - } - - private Path infrastructureVersionsPath() { - return root.append("infrastructureVersions"); + return lock(lockPath.append("infrastructureVersionsLock"), defaultLockTimeout); } + // OS versions ----------------------------------------------------------- - // OS versions public Map<NodeType, Version> readOsVersions() { - return read(osVersionsPath(), OsVersionsSerializer::fromJson).orElseGet(TreeMap::new); + return read(osVersionsPath, OsVersionsSerializer::fromJson).orElseGet(TreeMap::new); } public void writeOsVersions(Map<NodeType, Version> versions) { NestedTransaction transaction = new NestedTransaction(); CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); - curatorTransaction.add(CuratorOperations.setData(osVersionsPath().getAbsolute(), + curatorTransaction.add(CuratorOperations.setData(osVersionsPath.getAbsolute(), OsVersionsSerializer.toJson(versions))); transaction.commit(); } public Lock lockOsVersions() { - return lock(lockRoot.append("osVersionsLock"), defaultLockTimeout); + return lock(lockPath.append("osVersionsLock"), defaultLockTimeout); } - private Path osVersionsPath() { - return root.append("osVersions"); - } + // Docker images ----------------------------------------------------------- - - // Docker images public Map<NodeType, DockerImage> readDockerImages() { - return read(dockerImagesPath(), NodeTypeDockerImagesSerializer::fromJson).orElseGet(TreeMap::new); + return read(dockerImagesPath, NodeTypeDockerImagesSerializer::fromJson).orElseGet(TreeMap::new); } public void writeDockerImages(Map<NodeType, DockerImage> dockerImages) { NestedTransaction transaction = new NestedTransaction(); CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); - curatorTransaction.add(CuratorOperations.setData(dockerImagesPath().getAbsolute(), + curatorTransaction.add(CuratorOperations.setData(dockerImagesPath.getAbsolute(), NodeTypeDockerImagesSerializer.toJson(dockerImages))); transaction.commit(); } public Lock lockDockerImages() { - return lock(lockRoot.append("dockerImagesLock"), defaultLockTimeout); - } - - private Path dockerImagesPath() { - return root.append("dockerImages"); + return lock(lockPath.append("dockerImagesLock"), defaultLockTimeout); } + // Firmware checks ----------------------------------------------------------- - // Firmware checks /** Stores the instant after which a firmware check is required, or clears any outstanding ones if empty is given. */ public void writeFirmwareCheck(Optional<Instant> after) { byte[] data = after.map(instant -> Long.toString(instant.toEpochMilli()).getBytes()) .orElse(new byte[0]); NestedTransaction transaction = new NestedTransaction(); CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); - curatorTransaction.add(CuratorOperations.setData(firmwareCheckPath().getAbsolute(), data)); + curatorTransaction.add(CuratorOperations.setData(firmwareCheckPath.getAbsolute(), data)); transaction.commit(); } /** Returns the instant after which a firmware check is required, if any. */ public Optional<Instant> readFirmwareCheck() { - return read(firmwareCheckPath(), data -> Instant.ofEpochMilli(Long.parseLong(new String(data)))); - } - - private Path firmwareCheckPath() { - return root.append("firmwareCheck"); + return read(firmwareCheckPath, data -> Instant.ofEpochMilli(Long.parseLong(new String(data)))); } + // Load balancers ----------------------------------------------------------- - // Load balancers public List<LoadBalancerId> readLoadBalancerIds() { return readLoadBalancerIds((ignored) -> true); } @@ -556,29 +566,17 @@ public class CuratorDatabaseClient { transaction.commit(); } - // TODO(mpolden): Remove this and usages after April 2020 - public Lock lockLoadBalancers(ApplicationId application) { - return lock(lockRoot.append("loadBalancersLock2").append(application.serializedForm()), defaultLockTimeout); - } - private Path loadBalancerPath(LoadBalancerId id) { - return loadBalancersRoot.append(id.serializedForm()); + return loadBalancersPath.append(id.serializedForm()); } private List<LoadBalancerId> readLoadBalancerIds(Predicate<LoadBalancerId> predicate) { - return curatorDatabase.getChildren(loadBalancersRoot).stream() + return curatorDatabase.getChildren(loadBalancersPath).stream() .map(LoadBalancerId::fromSerializedForm) .filter(predicate) .collect(Collectors.toUnmodifiableList()); } - private Transaction.Operation createOrSet(Path path, byte[] data) { - if (curatorDatabase.exists(path)) { - return CuratorOperations.setData(path.getAbsolute(), data); - } - return CuratorOperations.create(path.getAbsolute(), data); - } - /** Returns a given number of unique provision indexes */ public List<Integer> getProvisionIndexes(int numIndexes) { if (numIndexes < 1) @@ -590,4 +588,17 @@ public class CuratorDatabaseClient { .collect(Collectors.toList()); } + private Lock lock(Path path, Duration timeout) { + return curatorDatabase.lock(path, timeout); + } + + private <T> Optional<T> read(Path path, Function<byte[], T> mapper) { + return curatorDatabase.getData(path).filter(data -> data.length > 0).map(mapper); + } + + private Transaction.Operation createOrSet(Path path, byte[] data) { + return curatorDatabase.exists(path) ? CuratorOperations.setData(path.getAbsolute(), data) + : CuratorOperations.create(path.getAbsolute(), data); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeResourcesSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeResourcesSerializer.java new file mode 100644 index 00000000000..8b792c8790b --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeResourcesSerializer.java @@ -0,0 +1,81 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.yahoo.config.provision.NodeResources; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; + +import java.util.Optional; + +/** + * @author bratseth + */ +public class NodeResourcesSerializer { + + private static final String vcpuKey = "vcpu"; + private static final String memoryKey = "memory"; + private static final String diskKey = "disk"; + private static final String bandwidthKey = "bandwidth"; + private static final String diskSpeedKey = "diskSpeed"; + private static final String storageTypeKey = "storageType"; + + static void toSlime(NodeResources resources, Cursor resourcesObject) { + resourcesObject.setDouble(vcpuKey, resources.vcpu()); + resourcesObject.setDouble(memoryKey, resources.memoryGb()); + resourcesObject.setDouble(diskKey, resources.diskGb()); + resourcesObject.setDouble(bandwidthKey, resources.bandwidthGbps()); + resourcesObject.setString(diskSpeedKey, diskSpeedToString(resources.diskSpeed())); + resourcesObject.setString(storageTypeKey, storageTypeToString(resources.storageType())); + } + + static NodeResources resourcesFromSlime(Inspector resources) { + return new NodeResources(resources.field(vcpuKey).asDouble(), + resources.field(memoryKey).asDouble(), + resources.field(diskKey).asDouble(), + resources.field(bandwidthKey).asDouble(), + diskSpeedFromSlime(resources.field(diskSpeedKey)), + storageTypeFromSlime(resources.field(storageTypeKey))); + } + + static Optional<NodeResources> optionalResourcesFromSlime(Inspector resources) { + return resources.valid() ? Optional.of(resourcesFromSlime(resources)) : Optional.empty(); + } + + private static NodeResources.DiskSpeed diskSpeedFromSlime(Inspector diskSpeed) { + switch (diskSpeed.asString()) { + case "fast" : return NodeResources.DiskSpeed.fast; + case "slow" : return NodeResources.DiskSpeed.slow; + case "any" : return NodeResources.DiskSpeed.any; + default: throw new IllegalStateException("Illegal disk-speed value '" + diskSpeed.asString() + "'"); + } + } + + private static String diskSpeedToString(NodeResources.DiskSpeed diskSpeed) { + switch (diskSpeed) { + case fast : return "fast"; + case slow : return "slow"; + case any : return "any"; + default: throw new IllegalStateException("Illegal disk-speed value '" + diskSpeed + "'"); + } + } + + private static NodeResources.StorageType storageTypeFromSlime(Inspector storageType) { + if ( ! storageType.valid()) return NodeResources.StorageType.getDefault(); // TODO: Remove this line after December 2019 + switch (storageType.asString()) { + case "remote" : return NodeResources.StorageType.remote; + case "local" : return NodeResources.StorageType.local; + case "any" : return NodeResources.StorageType.any; + default: throw new IllegalStateException("Illegal storage-type value '" + storageType.asString() + "'"); + } + } + + private static String storageTypeToString(NodeResources.StorageType storageType) { + switch (storageType) { + case remote : return "remote"; + case local : return "local"; + case any : return "any"; + default: throw new IllegalStateException("Illegal storage-type value '" + storageType + "'"); + } + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 2f834ab289f..81fc542afcc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -81,16 +81,9 @@ public class NodeSerializer { private static final String reservedToKey = "reservedTo"; // Node resource fields - // ...for hosts and nodes allocated by legacy flavor specs private static final String flavorKey = "flavor"; - // ...for nodes allocated by resources private static final String resourcesKey = "resources"; - private static final String vcpuKey = "vcpu"; - private static final String memoryKey = "memory"; private static final String diskKey = "disk"; - private static final String bandwidthKey = "bandwidth"; - private static final String diskSpeedKey = "diskSpeed"; - private static final String storageTypeKey = "storageType"; // Allocation fields private static final String tenantIdKey = "tenantId"; @@ -164,21 +157,12 @@ public class NodeSerializer { } } else { - toSlime(flavor.resources(), object.setObject(resourcesKey)); + NodeResourcesSerializer.toSlime(flavor.resources(), object.setObject(resourcesKey)); } } - private void toSlime(NodeResources resources, Cursor resourcesObject) { - resourcesObject.setDouble(vcpuKey, resources.vcpu()); - resourcesObject.setDouble(memoryKey, resources.memoryGb()); - resourcesObject.setDouble(diskKey, resources.diskGb()); - resourcesObject.setDouble(bandwidthKey, resources.bandwidthGbps()); - resourcesObject.setString(diskSpeedKey, diskSpeedToString(resources.diskSpeed())); - resourcesObject.setString(storageTypeKey, storageTypeToString(resources.storageType())); - } - private void toSlime(Allocation allocation, Cursor object) { - toSlime(allocation.requestedResources(), object.setObject(requestedResourcesKey)); + NodeResourcesSerializer.toSlime(allocation.requestedResources(), object.setObject(requestedResourcesKey)); object.setString(tenantIdKey, allocation.owner().tenant().value()); object.setString(applicationIdKey, allocation.owner().application().value()); object.setString(instanceIdKey, allocation.owner().instance().value()); @@ -252,26 +236,16 @@ public class NodeSerializer { return flavor.with(FlavorOverrides.ofDisk(resources.field(diskKey).asDouble())); } else { - return new Flavor(resourcesFromSlime(resources).get()); + return new Flavor(NodeResourcesSerializer.resourcesFromSlime(resources)); } } - private Optional<NodeResources> resourcesFromSlime(Inspector resources) { - if ( ! resources.valid()) return Optional.empty(); - - return Optional.of(new NodeResources(resources.field(vcpuKey).asDouble(), - resources.field(memoryKey).asDouble(), - resources.field(diskKey).asDouble(), - resources.field(bandwidthKey).asDouble(), - diskSpeedFromSlime(resources.field(diskSpeedKey)), - storageTypeFromSlime(resources.field(storageTypeKey)))); - } - private Optional<Allocation> allocationFromSlime(NodeResources assignedResources, Inspector object) { if ( ! object.valid()) return Optional.empty(); // TODO: Remove this line (and to the simplifications that follows) after November 2019 return Optional.of(new Allocation(applicationIdFromSlime(object), clusterMembershipFromSlime(object), - resourcesFromSlime(object.field(requestedResourcesKey)).orElse(assignedResources), + NodeResourcesSerializer.optionalResourcesFromSlime(object.field(requestedResourcesKey)) + .orElse(assignedResources), generationFromSlime(object, restartGenerationKey, currentRestartGenerationKey), object.field(removableKey).asBool(), NetworkPortsSerializer.fromSlime(object.field(networkPortsKey)))); @@ -470,41 +444,4 @@ public class NodeSerializer { throw new IllegalArgumentException("Serialized form of '" + type + "' not defined"); } - private static NodeResources.DiskSpeed diskSpeedFromSlime(Inspector diskSpeed) { - switch (diskSpeed.asString()) { - case "fast" : return NodeResources.DiskSpeed.fast; - case "slow" : return NodeResources.DiskSpeed.slow; - case "any" : return NodeResources.DiskSpeed.any; - default: throw new IllegalStateException("Illegal disk-speed value '" + diskSpeed.asString() + "'"); - } - } - - private static String diskSpeedToString(NodeResources.DiskSpeed diskSpeed) { - switch (diskSpeed) { - case fast : return "fast"; - case slow : return "slow"; - case any : return "any"; - default: throw new IllegalStateException("Illegal disk-speed value '" + diskSpeed + "'"); - } - } - - private static NodeResources.StorageType storageTypeFromSlime(Inspector storageType) { - if ( ! storageType.valid()) return NodeResources.StorageType.getDefault(); // TODO: Remove this line after December 2019 - switch (storageType.asString()) { - case "remote" : return NodeResources.StorageType.remote; - case "local" : return NodeResources.StorageType.local; - case "any" : return NodeResources.StorageType.any; - default: throw new IllegalStateException("Illegal storage-type value '" + storageType.asString() + "'"); - } - } - - private static String storageTypeToString(NodeResources.StorageType storageType) { - switch (storageType) { - case remote : return "remote"; - case local : return "local"; - case any : return "any"; - default: throw new IllegalStateException("Illegal storage-type value '" + storageType + "'"); - } - } - } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index b694a7c3cd4..f1fc6be837e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -53,10 +53,8 @@ public class LoadBalancerProvisioner { // Read and write all load balancers to make sure they are stored in the latest version of the serialization format for (var id : db.readLoadBalancerIds()) { try (var lock = db.lockConfig(id.application())) { - try (var legacyLock = db.lockLoadBalancers(id.application())) { - var loadBalancer = db.readLoadBalancer(id); - loadBalancer.ifPresent(db::writeLoadBalancer); - } + var loadBalancer = db.readLoadBalancer(id); + loadBalancer.ifPresent(db::writeLoadBalancer); } } } @@ -76,9 +74,7 @@ public class LoadBalancerProvisioner { if (!cluster.type().isContainer()) return; // Nothing to provision for this cluster type if (application.instance().isTester()) return; // Do not provision for tester instances try (var lock = db.lockConfig(application)) { - try (var legacyLock = db.lockLoadBalancers(application)) { - provision(application, effectiveId(cluster), false, lock); - } + provision(application, effectiveId(cluster), false, lock); } } @@ -95,16 +91,14 @@ public class LoadBalancerProvisioner { public void activate(ApplicationId application, Set<ClusterSpec> clusters, @SuppressWarnings("unused") Mutex applicationLock, NestedTransaction transaction) { try (var lock = db.lockConfig(application)) { - try (var legacyLock = db.lockLoadBalancers(application)) { - var containerClusters = containerClustersOf(clusters); - for (var clusterId : containerClusters) { - // Provision again to ensure that load balancer instance is re-configured with correct nodes - provision(application, clusterId, true, legacyLock); - } - // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed - var surplusLoadBalancers = surplusLoadBalancersOf(application, containerClusters); - deactivate(surplusLoadBalancers, transaction); + var containerClusters = containerClustersOf(clusters); + for (var clusterId : containerClusters) { + // Provision again to ensure that load balancer instance is re-configured with correct nodes + provision(application, clusterId, true, lock); } + // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed + var surplusLoadBalancers = surplusLoadBalancersOf(application, containerClusters); + deactivate(surplusLoadBalancers, transaction); } } @@ -115,9 +109,7 @@ public class LoadBalancerProvisioner { public void deactivate(ApplicationId application, NestedTransaction transaction) { try (var applicationLock = nodeRepository.lock(application)) { try (var lock = db.lockConfig(application)) { - try (var legacyLock = db.lockLoadBalancers(application)) { - deactivate(nodeRepository.loadBalancers(application).asList(), transaction); - } + deactivate(nodeRepository.loadBalancers(application).asList(), transaction); } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 6802196f80d..103543917bc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -145,10 +145,10 @@ public class NodeRepositoryProvisioner implements Provisioner { */ private ClusterResources decideTargetResources(ApplicationId applicationId, ClusterSpec.Id clusterId, Capacity requested) { try (Mutex lock = nodeRepository.lock(applicationId)) { - Application application = nodeRepository.applications().get(applicationId, true); + Application application = nodeRepository.applications().get(applicationId).orElse(new Application(applicationId)); application = application.withClusterLimits(clusterId, requested.minResources(), requested.maxResources()); - nodeRepository.applications().set(applicationId, application, lock); - return application.cluster(clusterId).targetResources() + nodeRepository.applications().put(application, lock); + return application.clusters().get(clusterId).targetResources() .orElseGet(() -> currentResources(applicationId, clusterId, requested) .orElse(requested.minResources())); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/applications/ApplicationsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/applications/ApplicationsTest.java new file mode 100644 index 00000000000..bdb14a868bd --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/applications/ApplicationsTest.java @@ -0,0 +1,55 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.applications; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.hosted.provision.NodeRepositoryTester; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author bratseth + */ +public class ApplicationsTest { + + @Test + public void testApplications() { + NodeRepositoryTester tester = new NodeRepositoryTester(); + Applications applications = new NodeRepositoryTester().nodeRepository().applications(); + ApplicationId app1 = ApplicationId.from("t1", "a1", "i1"); + ApplicationId app2 = ApplicationId.from("t1", "a2", "i1"); + ApplicationId app3 = ApplicationId.from("t1", "a2", "default"); + + assertTrue(applications.get(app1).isEmpty()); + assertEquals(List.of(), applications.ids()); + applications.put(new Application(app1), tester.nodeRepository().lock(app1)); + assertEquals(app1, applications.get(app1).get().id()); + assertEquals(List.of(app1), applications.ids()); + NestedTransaction t = new NestedTransaction(); + applications.remove(app1, t, tester.nodeRepository().lock(app1)); + t.commit(); + assertTrue(applications.get(app1).isEmpty()); + assertEquals(List.of(), applications.ids()); + + applications.put(new Application(app1), tester.nodeRepository().lock(app1)); + applications.put(new Application(app2), tester.nodeRepository().lock(app1)); + t = new NestedTransaction(); + applications.put(new Application(app3), t, tester.nodeRepository().lock(app1)); + assertEquals(List.of(app1, app2), applications.ids()); + t.commit(); + assertEquals(List.of(app1, app2, app3), applications.ids()); + t = new NestedTransaction(); + applications.remove(app1, t, tester.nodeRepository().lock(app1)); + applications.remove(app2, t, tester.nodeRepository().lock(app2)); + applications.remove(app3, t, tester.nodeRepository().lock(app3)); + assertEquals(List.of(app1, app2, app3), applications.ids()); + t.commit(); + assertTrue(applications.get(app1).isEmpty()); + assertEquals(List.of(), applications.ids()); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java index f02acdc1fca..7c6c4e2336b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java @@ -10,7 +10,6 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.applications.Application; -import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock; import org.junit.Test; @@ -53,9 +52,10 @@ public class AutoscalingIntegrationTest { ClusterResources min = new ClusterResources(2, 1, nodes); ClusterResources max = new ClusterResources(2, 1, nodes); - Application application = tester.nodeRepository().applications().get(application1, true).withClusterLimits(cluster1.id(), min, max); - tester.nodeRepository().applications().set(application1, application, tester.nodeRepository().lock(application1)); - var scaledResources = autoscaler.autoscale(application.cluster(cluster1.id()), + Application application = tester.nodeRepository().applications().get(application1).orElse(new Application(application1)) + .withClusterLimits(cluster1.id(), min, max); + tester.nodeRepository().applications().put(application, tester.nodeRepository().lock(application1)); + var scaledResources = autoscaler.autoscale(application.clusters().get(cluster1.id()), tester.nodeRepository().getNodes(application1)); assertTrue(scaledResources.isPresent()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index aed262b6c96..c250a0a23b0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -156,9 +156,10 @@ class AutoscalingTester { public Optional<AllocatableClusterResources> autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId, ClusterResources min, ClusterResources max) { - Application application = nodeRepository().applications().get(applicationId, true).withClusterLimits(clusterId, min, max); - nodeRepository().applications().set(applicationId, application, nodeRepository().lock(applicationId)); - return autoscaler.autoscale(application.cluster(clusterId), + Application application = nodeRepository().applications().get(applicationId).orElse(new Application(applicationId)) + .withClusterLimits(clusterId, min, max); + nodeRepository().applications().put(application, nodeRepository().lock(applicationId)); + return autoscaler.autoscale(application.clusters().get(clusterId), nodeRepository().getNodes(applicationId, Node.State.active)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java new file mode 100644 index 00000000000..1f07e26d045 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java @@ -0,0 +1,56 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.applications.Cluster; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; + +/** + * @author bratseth + */ +public class ApplicationSerializerTest { + + @Test + public void testApplicationSerialization() { + List<Cluster> clusters = new ArrayList<>(); + clusters.add(new Cluster(ClusterSpec.Id.from("c1"), + new ClusterResources( 8, 4, new NodeResources(1, 2, 3, 4)), + new ClusterResources(12, 6, new NodeResources(3, 6, 21, 24)), + Optional.empty())); + clusters.add(new Cluster(ClusterSpec.Id.from("c2"), + new ClusterResources( 8, 4, new NodeResources(1, 2, 3, 4)), + new ClusterResources(14, 7, new NodeResources(3, 6, 21, 24)), + Optional.of(new ClusterResources(10, 5, new NodeResources(2, 4, 14, 16))))); + Application original = new Application(ApplicationId.from("myTenant", "myApplication", "myInstance"), + clusters); + + Application serialized = ApplicationSerializer.fromJson(ApplicationSerializer.toJson(original)); + assertNotSame(original, serialized); + assertEquals(original, serialized); + assertEquals(original.id(), serialized.id()); + assertEquals(original.clusters(), serialized.clusters()); + for (Cluster originalCluster : original.clusters().values()) { + Cluster serializedCluster = serialized.clusters().get(originalCluster.id()); + assertNotNull(serializedCluster); + assertNotSame(originalCluster, serializedCluster); + assertEquals(originalCluster, serializedCluster); + assertEquals(originalCluster.id(), serializedCluster.id()); + assertEquals(originalCluster.minResources(), serializedCluster.minResources()); + assertEquals(originalCluster.maxResources(), serializedCluster.maxResources()); + assertEquals(originalCluster.targetResources(), serializedCluster.targetResources()); + } + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java index 050fb274fc7..880f5af5653 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java @@ -33,7 +33,7 @@ public class CuratorDatabaseClientTest { String zkline = "{\"hostname\":\"host1\",\"ipAddresses\":[\"127.0.0.1\"],\"openStackId\":\"7951bb9d-3989-4a60-a21c-13690637c8ea\",\"flavor\":\"default\",\"created\":1421054425159, \"type\":\"host\"}"; curator.framework().create().creatingParentsIfNeeded().forPath("/provision/v1/ready/host1", zkline.getBytes()); - List<Node> allocatedNodes = zkClient.getNodes(Node.State.ready); + List<Node> allocatedNodes = zkClient.readNodes(Node.State.ready); assertEquals(1, allocatedNodes.size()); assertEquals(NodeType.host, allocatedNodes.get(0).type()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java index 58ebfa555d6..15c73d15e8a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java @@ -50,7 +50,7 @@ import static org.junit.Assert.assertTrue; * @author bratseth * @author mpolden */ -public class SerializationTest { +public class NodeSerializerTest { private final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default", "large", "ugccloud-container"); private final NodeSerializer nodeSerializer = new NodeSerializer(nodeFlavors); |