From 572056db61169cb41141fa894ba43b6a0b11d850 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Mon, 1 Jul 2019 12:12:20 +0200 Subject: Move serialization code out of AllocatedHosts --- config-provisioning/abi-spec.json | 2 - .../com/yahoo/config/provision/AllocatedHosts.java | 162 +----------------- .../serialization/AllocatedHostsSerializer.java | 185 +++++++++++++++++++++ .../provision/serialization/package-info.java | 5 + .../yahoo/config/provision/AllocatedHostsTest.java | 84 ---------- .../AllocatedHostsSerializerTest.java | 92 ++++++++++ .../config/server/deploy/ZooKeeperClient.java | 7 +- .../server/zookeeper/ZKApplicationPackage.java | 11 +- .../config/server/deploy/ZooKeeperClientTest.java | 7 +- .../server/zookeeper/ZKApplicationPackageTest.java | 36 ++-- 10 files changed, 316 insertions(+), 275 deletions(-) create mode 100644 config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java create mode 100644 config-provisioning/src/main/java/com/yahoo/config/provision/serialization/package-info.java delete mode 100644 config-provisioning/src/test/java/com/yahoo/config/provision/AllocatedHostsTest.java create mode 100644 config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index ac0b1cf393d..64114389751 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -8,8 +8,6 @@ "methods": [ "public static com.yahoo.config.provision.AllocatedHosts withHosts(java.util.Set)", "public java.util.Set getHosts()", - "public byte[] toJson()", - "public static com.yahoo.config.provision.AllocatedHosts fromJson(byte[], java.util.Optional)", "public boolean equals(java.lang.Object)", "public int hashCode()", "public java.lang.String toString()" diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java b/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java index 96942c53a12..895325c502d 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java @@ -2,18 +2,7 @@ package com.yahoo.config.provision; import com.google.common.collect.ImmutableSet; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Optional; import java.util.Set; /** @@ -26,38 +15,9 @@ import java.util.Set; */ public class AllocatedHosts { - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one - // (and rewrite all nodes on startup), changes to the serialized format must be made - // such that what is serialized on version N+1 can be read by version N: - // - ADDING FIELDS: Always ok - // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. - // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - - private static final String mappingKey = "mapping"; - private static final String hostSpecKey = "hostSpec"; - private static final String hostSpecHostNameKey = "hostName"; - private static final String aliasesKey = "aliases"; - private static final String hostSpecMembershipKey = "membership"; - - // Flavor can be removed when all allocated nodes are docker nodes - private static final String flavorKey = "flavor"; - - private static final String resourcesKey = "resources"; - private static final String vcpuKey = "vcpu"; - private static final String memoryKey = "memory"; - private static final String diskKey = "disk"; - private static final String diskSpeedKey = "diskSpeed"; - - /** Wanted version */ - private static final String hostSpecVespaVersionKey = "vespaVersion"; - - /** Current version */ - private static final String hostSpecCurrentVespaVersionKey = "currentVespaVersion"; - private static final String hostSpecNetworkPortsKey = "ports"; - private final ImmutableSet hosts; - AllocatedHosts(Set hosts) { + private AllocatedHosts(Set hosts) { this.hosts = ImmutableSet.copyOf(hosts); } @@ -65,128 +25,8 @@ public class AllocatedHosts { return new AllocatedHosts(hosts); } - private void toSlime(Cursor cursor) { - Cursor array = cursor.setArray(mappingKey); - for (HostSpec host : hosts) - toSlime(host, array.addObject().setObject(hostSpecKey)); - } - - private void toSlime(HostSpec host, Cursor cursor) { - cursor.setString(hostSpecHostNameKey, host.hostname()); - aliasesToSlime(host, cursor); - host.membership().ifPresent(membership -> { - cursor.setString(hostSpecMembershipKey, membership.stringValue()); - cursor.setString(hostSpecVespaVersionKey, membership.cluster().vespaVersion().toFullString()); - }); - host.flavor().ifPresent(flavor -> toSlime(flavor, cursor)); - host.version().ifPresent(version -> cursor.setString(hostSpecCurrentVespaVersionKey, version.toFullString())); - host.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, cursor.setArray(hostSpecNetworkPortsKey))); - } - - private void aliasesToSlime(HostSpec spec, Cursor cursor) { - if (spec.aliases().isEmpty()) return; - Cursor aliases = cursor.setArray(aliasesKey); - for (String alias : spec.aliases()) - aliases.addString(alias); - } - - private void toSlime(Flavor flavor, Cursor object) { - if (flavor.isConfigured()) { - object.setString(flavorKey, flavor.name()); - } - else { - NodeResources resources = flavor.resources(); - Cursor resourcesObject = object.setObject(resourcesKey); - resourcesObject.setDouble(vcpuKey, resources.vcpu()); - resourcesObject.setDouble(memoryKey, resources.memoryGb()); - resourcesObject.setDouble(diskKey, resources.diskGb()); - resourcesObject.setString(diskSpeedKey, diskSpeedToString(resources.diskSpeed())); - } - } - /** Returns the hosts of this allocation */ public Set getHosts() { return hosts; } - - private static AllocatedHosts fromSlime(Inspector inspector, Optional nodeFlavors) { - Inspector array = inspector.field(mappingKey); - Set hosts = new LinkedHashSet<>(); - array.traverse((ArrayTraverser)(i, host) -> hosts.add(hostFromSlime(host.field(hostSpecKey), nodeFlavors))); - return new AllocatedHosts(hosts); - } - - static HostSpec hostFromSlime(Inspector object, Optional nodeFlavors) { - Optional membership = - object.field(hostSpecMembershipKey).valid() ? Optional.of(membershipFromSlime(object)) : Optional.empty(); - Optional flavor = flavorFromSlime(object, nodeFlavors); - Optional version = - optionalString(object.field(hostSpecCurrentVespaVersionKey)).map(com.yahoo.component.Version::new); - Optional networkPorts = - NetworkPortsSerializer.fromSlime(object.field(hostSpecNetworkPortsKey)); - return new HostSpec(object.field(hostSpecHostNameKey).asString(), aliasesFromSlime(object), flavor, membership, version, networkPorts); - } - - private static List aliasesFromSlime(Inspector object) { - if ( ! object.field(aliasesKey).valid()) return Collections.emptyList(); - List aliases = new ArrayList<>(); - object.field(aliasesKey).traverse((ArrayTraverser)(index, alias) -> aliases.add(alias.asString())); - return aliases; - } - - private static Optional flavorFromSlime(Inspector object, Optional nodeFlavors) { - if (object.field(flavorKey).valid() && nodeFlavors.isPresent() && nodeFlavors.get().exists(object.field(flavorKey).asString())) { - return nodeFlavors.get().getFlavor(object.field(flavorKey).asString()); - } - else if (object.field(resourcesKey).valid()) { - Inspector resources = object.field(resourcesKey); - return Optional.of(new Flavor(new NodeResources(resources.field(vcpuKey).asDouble(), - resources.field(memoryKey).asDouble(), - resources.field(diskKey).asDouble(), - diskSpeedFromSlime(resources.field(diskSpeedKey))))); - } - else { - return Optional.empty(); - } - } - - private static NodeResources.DiskSpeed diskSpeedFromSlime(Inspector diskSpeed) { - if ( ! diskSpeed.valid()) return NodeResources.DiskSpeed.fast; // TODO: Remove this line after June 2019 - switch (diskSpeed.asString()) { - case "fast" : return NodeResources.DiskSpeed.fast; - case "slow" : return NodeResources.DiskSpeed.slow; - case "any" : return NodeResources.DiskSpeed.any; - default: throw new IllegalStateException("Illegal disk-speed value '" + diskSpeed.asString() + "'"); - } - } - - private static String diskSpeedToString(NodeResources.DiskSpeed diskSpeed) { - switch (diskSpeed) { - case fast : return "fast"; - case slow : return "slow"; - case any : return "any"; - default: throw new IllegalStateException("Illegal disk-speed value '" + diskSpeed + "'"); - } - - } - - private static ClusterMembership membershipFromSlime(Inspector object) { - return ClusterMembership.from(object.field(hostSpecMembershipKey).asString(), - com.yahoo.component.Version.fromString(object.field(hostSpecVespaVersionKey).asString())); - } - - private static Optional optionalString(Inspector inspector) { - if ( ! inspector.valid()) return Optional.empty(); - return Optional.of(inspector.asString()); - } - - public byte[] toJson() throws IOException { - Slime slime = new Slime(); - toSlime(slime.setObject()); - return SlimeUtils.toJsonBytes(slime); - } - - public static AllocatedHosts fromJson(byte[] json, Optional nodeFlavors) { - return fromSlime(SlimeUtils.jsonToSlime(json).get(), nodeFlavors); - } @Override public boolean equals(Object other) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java new file mode 100644 index 00000000000..9e01718bfc6 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java @@ -0,0 +1,185 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision.serialization; + +import com.yahoo.config.provision.AllocatedHosts; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NetworkPorts; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +/** + * The hosts allocated to an application. + * This can be serialized to/from JSON. + * This is immutable. + * + * @author Ulf Lilleengen + * @author bratseth + */ +public class AllocatedHostsSerializer { + + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + + private static final String mappingKey = "mapping"; + private static final String hostSpecKey = "hostSpec"; + private static final String hostSpecHostNameKey = "hostName"; + private static final String aliasesKey = "aliases"; + private static final String hostSpecMembershipKey = "membership"; + + // Flavor can be removed when all allocated nodes are docker nodes + private static final String flavorKey = "flavor"; + + private static final String resourcesKey = "resources"; + private static final String vcpuKey = "vcpu"; + private static final String memoryKey = "memory"; + private static final String diskKey = "disk"; + private static final String diskSpeedKey = "diskSpeed"; + + /** Wanted version */ + private static final String hostSpecVespaVersionKey = "vespaVersion"; + + /** Current version */ + private static final String hostSpecCurrentVespaVersionKey = "currentVespaVersion"; + private static final String hostSpecNetworkPortsKey = "ports"; + + + public static byte[] toJson(AllocatedHosts allocatedHosts) throws IOException { + Slime slime = new Slime(); + toSlime(allocatedHosts, slime.setObject()); + return SlimeUtils.toJsonBytes(slime); + } + + public static void toSlime(AllocatedHosts allocatedHosts, Cursor cursor) { + Cursor array = cursor.setArray(mappingKey); + for (HostSpec host : allocatedHosts.getHosts()) + toSlime(host, array.addObject().setObject(hostSpecKey)); + } + + private static void toSlime(HostSpec host, Cursor cursor) { + cursor.setString(hostSpecHostNameKey, host.hostname()); + aliasesToSlime(host, cursor); + host.membership().ifPresent(membership -> { + cursor.setString(hostSpecMembershipKey, membership.stringValue()); + cursor.setString(hostSpecVespaVersionKey, membership.cluster().vespaVersion().toFullString()); + }); + host.flavor().ifPresent(flavor -> toSlime(flavor, cursor)); + host.version().ifPresent(version -> cursor.setString(hostSpecCurrentVespaVersionKey, version.toFullString())); + host.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, cursor.setArray(hostSpecNetworkPortsKey))); + } + + private static void aliasesToSlime(HostSpec spec, Cursor cursor) { + if (spec.aliases().isEmpty()) return; + Cursor aliases = cursor.setArray(aliasesKey); + for (String alias : spec.aliases()) + aliases.addString(alias); + } + + private static void toSlime(Flavor flavor, Cursor object) { + if (flavor.isConfigured()) { + object.setString(flavorKey, flavor.name()); + } + else { + NodeResources resources = flavor.resources(); + Cursor resourcesObject = object.setObject(resourcesKey); + resourcesObject.setDouble(vcpuKey, resources.vcpu()); + resourcesObject.setDouble(memoryKey, resources.memoryGb()); + resourcesObject.setDouble(diskKey, resources.diskGb()); + resourcesObject.setString(diskSpeedKey, diskSpeedToString(resources.diskSpeed())); + } + } + + + public static AllocatedHosts fromJson(byte[] json, Optional nodeFlavors) { + return fromSlime(SlimeUtils.jsonToSlime(json).get(), nodeFlavors); + } + + public static AllocatedHosts fromSlime(Inspector inspector, Optional nodeFlavors) { + Inspector array = inspector.field(mappingKey); + Set hosts = new LinkedHashSet<>(); + array.traverse((ArrayTraverser)(i, host) -> hosts.add(hostFromSlime(host.field(hostSpecKey), nodeFlavors))); + return AllocatedHosts.withHosts(hosts); + } + + private static HostSpec hostFromSlime(Inspector object, Optional nodeFlavors) { + Optional membership = + object.field(hostSpecMembershipKey).valid() ? Optional.of(membershipFromSlime(object)) : Optional.empty(); + Optional flavor = flavorFromSlime(object, nodeFlavors); + Optional version = + optionalString(object.field(hostSpecCurrentVespaVersionKey)).map(com.yahoo.component.Version::new); + Optional networkPorts = + NetworkPortsSerializer.fromSlime(object.field(hostSpecNetworkPortsKey)); + return new HostSpec(object.field(hostSpecHostNameKey).asString(), aliasesFromSlime(object), flavor, membership, version, networkPorts); + } + + private static List aliasesFromSlime(Inspector object) { + if ( ! object.field(aliasesKey).valid()) return Collections.emptyList(); + List aliases = new ArrayList<>(); + object.field(aliasesKey).traverse((ArrayTraverser)(index, alias) -> aliases.add(alias.asString())); + return aliases; + } + + private static Optional flavorFromSlime(Inspector object, Optional nodeFlavors) { + if (object.field(flavorKey).valid() && nodeFlavors.isPresent() && nodeFlavors.get().exists(object.field(flavorKey).asString())) { + return nodeFlavors.get().getFlavor(object.field(flavorKey).asString()); + } + else if (object.field(resourcesKey).valid()) { + Inspector resources = object.field(resourcesKey); + return Optional.of(new Flavor(new NodeResources(resources.field(vcpuKey).asDouble(), + resources.field(memoryKey).asDouble(), + resources.field(diskKey).asDouble(), + diskSpeedFromSlime(resources.field(diskSpeedKey))))); + } + else { + return Optional.empty(); + } + } + + private static NodeResources.DiskSpeed diskSpeedFromSlime(Inspector diskSpeed) { + if ( ! diskSpeed.valid()) return NodeResources.DiskSpeed.fast; // TODO: Remove this line after June 2019 + switch (diskSpeed.asString()) { + case "fast" : return NodeResources.DiskSpeed.fast; + case "slow" : return NodeResources.DiskSpeed.slow; + case "any" : return NodeResources.DiskSpeed.any; + default: throw new IllegalStateException("Illegal disk-speed value '" + diskSpeed.asString() + "'"); + } + } + + private static String diskSpeedToString(NodeResources.DiskSpeed diskSpeed) { + switch (diskSpeed) { + case fast : return "fast"; + case slow : return "slow"; + case any : return "any"; + default: throw new IllegalStateException("Illegal disk-speed value '" + diskSpeed + "'"); + } + + } + + private static ClusterMembership membershipFromSlime(Inspector object) { + return ClusterMembership.from(object.field(hostSpecMembershipKey).asString(), + com.yahoo.component.Version.fromString(object.field(hostSpecVespaVersionKey).asString())); + } + + private static Optional optionalString(Inspector inspector) { + if ( ! inspector.valid()) return Optional.empty(); + return Optional.of(inspector.asString()); + } +} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/package-info.java b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/package-info.java new file mode 100644 index 00000000000..440228b9d3e --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.config.provision.serialization; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/AllocatedHostsTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/AllocatedHostsTest.java deleted file mode 100644 index 737c1047197..00000000000 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/AllocatedHostsTest.java +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.provision; - -import com.yahoo.component.Version; -import com.yahoo.config.provisioning.FlavorsConfig; -import org.junit.Test; - -import java.io.IOException; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; - -import static org.junit.Assert.assertEquals; - -/** - * @author bratseth - */ -public class AllocatedHostsTest { - - @Test - public void testAllocatedHostsSerialization() throws IOException { - NodeFlavors configuredFlavors = configuredFlavorsFrom("C/12/45/100", 12, 45, 100, Flavor.Type.BARE_METAL); - - Set hosts = new LinkedHashSet<>(); - hosts.add(new HostSpec("empty", - Optional.empty())); - hosts.add(new HostSpec("with-aliases", - List.of("alias1", "alias2"))); - hosts.add(new HostSpec("allocated", - Optional.of(ClusterMembership.from("container/test/0/0", com.yahoo.component.Version.fromString("6.73.1"))))); - hosts.add(new HostSpec("flavor-from-resources-1", - Collections.emptyList(), new Flavor(new NodeResources(0.5, 3.1, 4)))); - hosts.add(new HostSpec("flavor-from-resources-2", - Collections.emptyList(), new Flavor(new NodeResources(0.5, 3.1, 4, NodeResources.DiskSpeed.any)))); - hosts.add(new HostSpec("configured-flavor", - Collections.emptyList(), configuredFlavors.getFlavorOrThrow("C/12/45/100"))); - hosts.add(new HostSpec("with-version", - Collections.emptyList(), Optional.empty(), Optional.empty(), Optional.of(Version.fromString("3.4.5")))); - hosts.add(new HostSpec("with-ports", - Collections.emptyList(), Optional.empty(), Optional.empty(), Optional.empty(), - Optional.of(new NetworkPorts(List.of(new NetworkPorts.Allocation(1234, "service1", "configId1", "suffix1"), - new NetworkPorts.Allocation(4567, "service2", "configId2", "suffix2")))))); - - assertAllocatedHosts(AllocatedHosts.withHosts(hosts), configuredFlavors); - } - - private void assertAllocatedHosts(AllocatedHosts expectedHosts, NodeFlavors configuredFlavors) throws IOException { - AllocatedHosts deserializedHosts = AllocatedHosts.fromJson(expectedHosts.toJson(), - Optional.of(configuredFlavors)); - - assertEquals(expectedHosts, deserializedHosts); - for (HostSpec expectedHost : expectedHosts.getHosts()) { - HostSpec deserializedHost = requireHost(expectedHost.hostname(), deserializedHosts); - assertEquals(expectedHost.hostname(), deserializedHost.hostname()); - assertEquals(expectedHost.membership(), deserializedHost.membership()); - assertEquals(expectedHost.flavor(), deserializedHost.flavor()); - assertEquals(expectedHost.version(), deserializedHost.version()); - assertEquals(expectedHost.networkPorts(), deserializedHost.networkPorts()); - assertEquals(expectedHost.aliases(), deserializedHost.aliases()); - } - } - - private HostSpec requireHost(String hostname, AllocatedHosts hosts) { - for (HostSpec host : hosts.getHosts()) - if (host.hostname().equals(hostname)) - return host; - throw new IllegalArgumentException("No host " + hostname + " is present"); - } - - public NodeFlavors configuredFlavorsFrom(String flavorName, double cpu, double mem, double disk, Flavor.Type type) { - FlavorsConfig.Builder b = new FlavorsConfig.Builder(); - FlavorsConfig.Flavor.Builder flavor = new FlavorsConfig.Flavor.Builder(); - flavor.name(flavorName); - flavor.minDiskAvailableGb(disk); - flavor.minCpuCores(cpu); - flavor.minMainMemoryAvailableGb(mem); - flavor.environment(type.name()); - b.flavor(flavor); - return new NodeFlavors(b.build()); - } - -} diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java new file mode 100644 index 00000000000..44aa6ed59b5 --- /dev/null +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java @@ -0,0 +1,92 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision.serialization; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.AllocatedHosts; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NetworkPorts; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provisioning.FlavorsConfig; +import org.junit.Test; + +import java.io.IOException; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static com.yahoo.config.provision.serialization.AllocatedHostsSerializer.fromJson; +import static com.yahoo.config.provision.serialization.AllocatedHostsSerializer.toJson; +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class AllocatedHostsSerializerTest { + + @Test + public void testAllocatedHostsSerialization() throws IOException { + NodeFlavors configuredFlavors = configuredFlavorsFrom("C/12/45/100", 12, 45, 100, Flavor.Type.BARE_METAL); + + Set hosts = new LinkedHashSet<>(); + hosts.add(new HostSpec("empty", + Optional.empty())); + hosts.add(new HostSpec("with-aliases", + List.of("alias1", "alias2"))); + hosts.add(new HostSpec("allocated", + Optional.of(ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"))))); + hosts.add(new HostSpec("flavor-from-resources-1", + Collections.emptyList(), new Flavor(new NodeResources(0.5, 3.1, 4)))); + hosts.add(new HostSpec("flavor-from-resources-2", + Collections.emptyList(), new Flavor(new NodeResources(0.5, 3.1, 4, NodeResources.DiskSpeed.any)))); + hosts.add(new HostSpec("configured-flavor", + Collections.emptyList(), configuredFlavors.getFlavorOrThrow("C/12/45/100"))); + hosts.add(new HostSpec("with-version", + Collections.emptyList(), Optional.empty(), Optional.empty(), Optional.of(Version.fromString("3.4.5")))); + hosts.add(new HostSpec("with-ports", + Collections.emptyList(), Optional.empty(), Optional.empty(), Optional.empty(), + Optional.of(new NetworkPorts(List.of(new NetworkPorts.Allocation(1234, "service1", "configId1", "suffix1"), + new NetworkPorts.Allocation(4567, "service2", "configId2", "suffix2")))))); + + assertAllocatedHosts(AllocatedHosts.withHosts(hosts), configuredFlavors); + } + + private void assertAllocatedHosts(AllocatedHosts expectedHosts, NodeFlavors configuredFlavors) throws IOException { + AllocatedHosts deserializedHosts = fromJson(toJson(expectedHosts), Optional.of(configuredFlavors)); + + assertEquals(expectedHosts, deserializedHosts); + for (HostSpec expectedHost : expectedHosts.getHosts()) { + HostSpec deserializedHost = requireHost(expectedHost.hostname(), deserializedHosts); + assertEquals(expectedHost.hostname(), deserializedHost.hostname()); + assertEquals(expectedHost.membership(), deserializedHost.membership()); + assertEquals(expectedHost.flavor(), deserializedHost.flavor()); + assertEquals(expectedHost.version(), deserializedHost.version()); + assertEquals(expectedHost.networkPorts(), deserializedHost.networkPorts()); + assertEquals(expectedHost.aliases(), deserializedHost.aliases()); + } + } + + private HostSpec requireHost(String hostname, AllocatedHosts hosts) { + for (HostSpec host : hosts.getHosts()) + if (host.hostname().equals(hostname)) + return host; + throw new IllegalArgumentException("No host " + hostname + " is present"); + } + + private NodeFlavors configuredFlavorsFrom(String flavorName, double cpu, double mem, double disk, Flavor.Type type) { + FlavorsConfig.Builder b = new FlavorsConfig.Builder(); + FlavorsConfig.Flavor.Builder flavor = new FlavorsConfig.Flavor.Builder(); + flavor.name(flavorName); + flavor.minDiskAvailableGb(disk); + flavor.minCpuCores(cpu); + flavor.minMainMemoryAvailableGb(mem); + flavor.environment(type.name()); + b.flavor(flavor); + return new NodeFlavors(b.build()); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index e5552de41f5..110e73bcdf9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.deploy; +import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ApplicationPackage; @@ -9,7 +10,7 @@ import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.model.application.provider.PreGeneratedFileRegistry; import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.component.Version; +import com.yahoo.config.provision.serialization.AllocatedHostsSerializer; import com.yahoo.io.reader.NamedReader; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; @@ -361,7 +362,9 @@ public class ZooKeeperClient { } public void write(AllocatedHosts hosts) throws IOException { - configCurator.putData(rootPath.append(ZKApplicationPackage.allocatedHostsNode).getAbsolute(), hosts.toJson()); + configCurator.putData( + rootPath.append(ZKApplicationPackage.allocatedHostsNode).getAbsolute(), + AllocatedHostsSerializer.toJson(hosts)); } public void write(Map fileRegistryMap) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java index 5c40f592a77..e013244c80c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java @@ -3,19 +3,20 @@ package com.yahoo.vespa.config.server.zookeeper; import com.google.common.base.Joiner; import com.yahoo.component.Version; +import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.codegen.DefParser; -import com.yahoo.config.application.api.ApplicationFile; -import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.application.provider.PreGeneratedFileRegistry; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.AllocatedHosts; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.serialization.AllocatedHostsSerializer; import com.yahoo.io.IOUtils; -import com.yahoo.path.Path; import com.yahoo.io.reader.NamedReader; +import com.yahoo.path.Path; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionBuilder; import com.yahoo.vespa.config.ConfigDefinitionKey; @@ -69,7 +70,7 @@ public class ZKApplicationPackage implements ApplicationPackage { */ private AllocatedHosts readAllocatedHosts(String allocatedHostsPath, Optional nodeFlavors) { try { - return AllocatedHosts.fromJson(liveApp.getBytes(allocatedHostsPath), nodeFlavors); + return AllocatedHostsSerializer.fromJson(liveApp.getBytes(allocatedHostsPath), nodeFlavors); } catch (Exception e) { throw new RuntimeException("Unable to read allocated hosts", e); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java index e01b2eccb35..918670d71f2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java @@ -13,9 +13,9 @@ import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.HostSpec; import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import org.junit.Before; import org.junit.Ignore; import org.junit.Rule; @@ -30,10 +30,11 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import static com.yahoo.config.provision.serialization.AllocatedHostsSerializer.fromJson; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * Unit tests for ZooKeeperClient. @@ -199,7 +200,7 @@ public class ZooKeeperClientTest { Path hostsPath = app.append(ZKApplicationPackage.allocatedHostsNode); assertTrue(zk.exists(hostsPath.getAbsolute())); - AllocatedHosts deserialized = AllocatedHosts.fromJson(zk.getBytes(hostsPath.getAbsolute()), Optional.empty()); + AllocatedHosts deserialized = fromJson(zk.getBytes(hostsPath.getAbsolute()), Optional.empty()); assertEquals(hosts, deserialized.getHosts()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index e8e2dd07756..8b8be1a27d7 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -1,27 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.zookeeper; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertThat; - -import java.io.File; -import java.io.IOException; -import java.io.Reader; -import java.util.Collections; -import java.util.Optional; -import java.util.regex.Pattern; - +import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.component.Version; import com.yahoo.config.provisioning.FlavorsConfig; +import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.vespa.curator.mock.MockCurator; @@ -30,7 +18,19 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import com.yahoo.io.IOUtils; +import java.io.File; +import java.io.IOException; +import java.io.Reader; +import java.util.Collections; +import java.util.Optional; +import java.util.regex.Pattern; + +import static com.yahoo.config.provision.serialization.AllocatedHostsSerializer.toJson; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; public class ZKApplicationPackageTest { @@ -77,7 +77,7 @@ public class ZKApplicationPackageTest { assertFalse(zkApp.getFileRegistries().containsKey(new Version(0, 0, 0))); assertThat(zkApp.getFileRegistries().get(goodVersion).fileSourceHost(), is("dummyfiles")); AllocatedHosts readInfo = zkApp.getAllocatedHosts().get(); - assertThat(Utf8.toString(readInfo.toJson()), is(Utf8.toString(ALLOCATED_HOSTS.toJson()))); + assertThat(Utf8.toString(toJson(readInfo)), is(Utf8.toString(toJson(ALLOCATED_HOSTS)))); assertThat(readInfo.getHosts().iterator().next().flavor(), is(TEST_FLAVOR)); assertEquals("6.0.1", readInfo.getHosts().iterator().next().version().get().toString()); assertTrue(zkApp.getDeployment().isPresent()); @@ -90,7 +90,7 @@ public class ZKApplicationPackageTest { String metaData = "{\"deploy\":{\"user\":\"foo\",\"from\":\"bar\",\"timestamp\":1},\"application\":{\"name\":\"foo\",\"checksum\":\"abc\",\"generation\":4,\"previousActiveGeneration\":3}}"; zk.putData("/0", ConfigCurator.META_ZK_PATH, metaData); zk.putData("/0/" + ZKApplicationPackage.fileRegistryNode + "/3.0.0", "dummyfiles"); - zk.putData("/0/" + ZKApplicationPackage.allocatedHostsNode, ALLOCATED_HOSTS.toJson()); + zk.putData("/0/" + ZKApplicationPackage.allocatedHostsNode, toJson(ALLOCATED_HOSTS)); } private static class MockNodeFlavors extends NodeFlavors{ -- cgit v1.2.3