diff options
author | Valerij Fredriksen <valerijf@verizonmedia.com> | 2019-07-01 12:12:20 +0200 |
---|---|---|
committer | Valerij Fredriksen <valerijf@verizonmedia.com> | 2019-07-01 12:12:20 +0200 |
commit | 572056db61169cb41141fa894ba43b6a0b11d850 (patch) | |
tree | 5ec4e5d08300e4801822a2e9b7b1afc5429ab4b5 /config-provisioning | |
parent | 2ed08d822b719c4ead307e4390c9d89e93945395 (diff) |
Move serialization code out of AllocatedHosts
Diffstat (limited to 'config-provisioning')
-rw-r--r-- | config-provisioning/abi-spec.json | 2 | ||||
-rw-r--r-- | config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java | 162 | ||||
-rw-r--r-- | config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java | 185 | ||||
-rw-r--r-- | config-provisioning/src/main/java/com/yahoo/config/provision/serialization/package-info.java | 5 | ||||
-rw-r--r-- | config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java (renamed from config-provisioning/src/test/java/com/yahoo/config/provision/AllocatedHostsTest.java) | 20 |
5 files changed, 205 insertions, 169 deletions
diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index ac0b1cf393d..64114389751 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -8,8 +8,6 @@ "methods": [ "public static com.yahoo.config.provision.AllocatedHosts withHosts(java.util.Set)", "public java.util.Set getHosts()", - "public byte[] toJson()", - "public static com.yahoo.config.provision.AllocatedHosts fromJson(byte[], java.util.Optional)", "public boolean equals(java.lang.Object)", "public int hashCode()", "public java.lang.String toString()" diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java b/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java index 96942c53a12..895325c502d 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java @@ -2,18 +2,7 @@ package com.yahoo.config.provision; import com.google.common.collect.ImmutableSet; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Optional; import java.util.Set; /** @@ -26,38 +15,9 @@ import java.util.Set; */ public class AllocatedHosts { - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one - // (and rewrite all nodes on startup), changes to the serialized format must be made - // such that what is serialized on version N+1 can be read by version N: - // - ADDING FIELDS: Always ok - // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. - // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. - - private static final String mappingKey = "mapping"; - private static final String hostSpecKey = "hostSpec"; - private static final String hostSpecHostNameKey = "hostName"; - private static final String aliasesKey = "aliases"; - private static final String hostSpecMembershipKey = "membership"; - - // Flavor can be removed when all allocated nodes are docker nodes - private static final String flavorKey = "flavor"; - - private static final String resourcesKey = "resources"; - private static final String vcpuKey = "vcpu"; - private static final String memoryKey = "memory"; - private static final String diskKey = "disk"; - private static final String diskSpeedKey = "diskSpeed"; - - /** Wanted version */ - private static final String hostSpecVespaVersionKey = "vespaVersion"; - - /** Current version */ - private static final String hostSpecCurrentVespaVersionKey = "currentVespaVersion"; - private static final String hostSpecNetworkPortsKey = "ports"; - private final ImmutableSet<HostSpec> hosts; - AllocatedHosts(Set<HostSpec> hosts) { + private AllocatedHosts(Set<HostSpec> hosts) { this.hosts = ImmutableSet.copyOf(hosts); } @@ -65,128 +25,8 @@ public class AllocatedHosts { return new AllocatedHosts(hosts); } - private void toSlime(Cursor cursor) { - Cursor array = cursor.setArray(mappingKey); - for (HostSpec host : hosts) - toSlime(host, array.addObject().setObject(hostSpecKey)); - } - - private void toSlime(HostSpec host, Cursor cursor) { - cursor.setString(hostSpecHostNameKey, host.hostname()); - aliasesToSlime(host, cursor); - host.membership().ifPresent(membership -> { - cursor.setString(hostSpecMembershipKey, membership.stringValue()); - cursor.setString(hostSpecVespaVersionKey, membership.cluster().vespaVersion().toFullString()); - }); - host.flavor().ifPresent(flavor -> toSlime(flavor, cursor)); - host.version().ifPresent(version -> cursor.setString(hostSpecCurrentVespaVersionKey, version.toFullString())); - host.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, cursor.setArray(hostSpecNetworkPortsKey))); - } - - private void aliasesToSlime(HostSpec spec, Cursor cursor) { - if (spec.aliases().isEmpty()) return; - Cursor aliases = cursor.setArray(aliasesKey); - for (String alias : spec.aliases()) - aliases.addString(alias); - } - - private void toSlime(Flavor flavor, Cursor object) { - if (flavor.isConfigured()) { - object.setString(flavorKey, flavor.name()); - } - else { - NodeResources resources = flavor.resources(); - Cursor resourcesObject = object.setObject(resourcesKey); - resourcesObject.setDouble(vcpuKey, resources.vcpu()); - resourcesObject.setDouble(memoryKey, resources.memoryGb()); - resourcesObject.setDouble(diskKey, resources.diskGb()); - resourcesObject.setString(diskSpeedKey, diskSpeedToString(resources.diskSpeed())); - } - } - /** Returns the hosts of this allocation */ public Set<HostSpec> getHosts() { return hosts; } - - private static AllocatedHosts fromSlime(Inspector inspector, Optional<NodeFlavors> nodeFlavors) { - Inspector array = inspector.field(mappingKey); - Set<HostSpec> hosts = new LinkedHashSet<>(); - array.traverse((ArrayTraverser)(i, host) -> hosts.add(hostFromSlime(host.field(hostSpecKey), nodeFlavors))); - return new AllocatedHosts(hosts); - } - - static HostSpec hostFromSlime(Inspector object, Optional<NodeFlavors> nodeFlavors) { - Optional<ClusterMembership> membership = - object.field(hostSpecMembershipKey).valid() ? Optional.of(membershipFromSlime(object)) : Optional.empty(); - Optional<Flavor> flavor = flavorFromSlime(object, nodeFlavors); - Optional<com.yahoo.component.Version> version = - optionalString(object.field(hostSpecCurrentVespaVersionKey)).map(com.yahoo.component.Version::new); - Optional<NetworkPorts> networkPorts = - NetworkPortsSerializer.fromSlime(object.field(hostSpecNetworkPortsKey)); - return new HostSpec(object.field(hostSpecHostNameKey).asString(), aliasesFromSlime(object), flavor, membership, version, networkPorts); - } - - private static List<String> aliasesFromSlime(Inspector object) { - if ( ! object.field(aliasesKey).valid()) return Collections.emptyList(); - List<String> aliases = new ArrayList<>(); - object.field(aliasesKey).traverse((ArrayTraverser)(index, alias) -> aliases.add(alias.asString())); - return aliases; - } - - private static Optional<Flavor> flavorFromSlime(Inspector object, Optional<NodeFlavors> nodeFlavors) { - if (object.field(flavorKey).valid() && nodeFlavors.isPresent() && nodeFlavors.get().exists(object.field(flavorKey).asString())) { - return nodeFlavors.get().getFlavor(object.field(flavorKey).asString()); - } - else if (object.field(resourcesKey).valid()) { - Inspector resources = object.field(resourcesKey); - return Optional.of(new Flavor(new NodeResources(resources.field(vcpuKey).asDouble(), - resources.field(memoryKey).asDouble(), - resources.field(diskKey).asDouble(), - diskSpeedFromSlime(resources.field(diskSpeedKey))))); - } - else { - return Optional.empty(); - } - } - - private static NodeResources.DiskSpeed diskSpeedFromSlime(Inspector diskSpeed) { - if ( ! diskSpeed.valid()) return NodeResources.DiskSpeed.fast; // TODO: Remove this line after June 2019 - switch (diskSpeed.asString()) { - case "fast" : return NodeResources.DiskSpeed.fast; - case "slow" : return NodeResources.DiskSpeed.slow; - case "any" : return NodeResources.DiskSpeed.any; - default: throw new IllegalStateException("Illegal disk-speed value '" + diskSpeed.asString() + "'"); - } - } - - private static String diskSpeedToString(NodeResources.DiskSpeed diskSpeed) { - switch (diskSpeed) { - case fast : return "fast"; - case slow : return "slow"; - case any : return "any"; - default: throw new IllegalStateException("Illegal disk-speed value '" + diskSpeed + "'"); - } - - } - - private static ClusterMembership membershipFromSlime(Inspector object) { - return ClusterMembership.from(object.field(hostSpecMembershipKey).asString(), - com.yahoo.component.Version.fromString(object.field(hostSpecVespaVersionKey).asString())); - } - - private static Optional<String> optionalString(Inspector inspector) { - if ( ! inspector.valid()) return Optional.empty(); - return Optional.of(inspector.asString()); - } - - public byte[] toJson() throws IOException { - Slime slime = new Slime(); - toSlime(slime.setObject()); - return SlimeUtils.toJsonBytes(slime); - } - - public static AllocatedHosts fromJson(byte[] json, Optional<NodeFlavors> nodeFlavors) { - return fromSlime(SlimeUtils.jsonToSlime(json).get(), nodeFlavors); - } @Override public boolean equals(Object other) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java new file mode 100644 index 00000000000..9e01718bfc6 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java @@ -0,0 +1,185 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision.serialization; + +import com.yahoo.config.provision.AllocatedHosts; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NetworkPorts; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +/** + * The hosts allocated to an application. + * This can be serialized to/from JSON. + * This is immutable. + * + * @author Ulf Lilleengen + * @author bratseth + */ +public class AllocatedHostsSerializer { + + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + + private static final String mappingKey = "mapping"; + private static final String hostSpecKey = "hostSpec"; + private static final String hostSpecHostNameKey = "hostName"; + private static final String aliasesKey = "aliases"; + private static final String hostSpecMembershipKey = "membership"; + + // Flavor can be removed when all allocated nodes are docker nodes + private static final String flavorKey = "flavor"; + + private static final String resourcesKey = "resources"; + private static final String vcpuKey = "vcpu"; + private static final String memoryKey = "memory"; + private static final String diskKey = "disk"; + private static final String diskSpeedKey = "diskSpeed"; + + /** Wanted version */ + private static final String hostSpecVespaVersionKey = "vespaVersion"; + + /** Current version */ + private static final String hostSpecCurrentVespaVersionKey = "currentVespaVersion"; + private static final String hostSpecNetworkPortsKey = "ports"; + + + public static byte[] toJson(AllocatedHosts allocatedHosts) throws IOException { + Slime slime = new Slime(); + toSlime(allocatedHosts, slime.setObject()); + return SlimeUtils.toJsonBytes(slime); + } + + public static void toSlime(AllocatedHosts allocatedHosts, Cursor cursor) { + Cursor array = cursor.setArray(mappingKey); + for (HostSpec host : allocatedHosts.getHosts()) + toSlime(host, array.addObject().setObject(hostSpecKey)); + } + + private static void toSlime(HostSpec host, Cursor cursor) { + cursor.setString(hostSpecHostNameKey, host.hostname()); + aliasesToSlime(host, cursor); + host.membership().ifPresent(membership -> { + cursor.setString(hostSpecMembershipKey, membership.stringValue()); + cursor.setString(hostSpecVespaVersionKey, membership.cluster().vespaVersion().toFullString()); + }); + host.flavor().ifPresent(flavor -> toSlime(flavor, cursor)); + host.version().ifPresent(version -> cursor.setString(hostSpecCurrentVespaVersionKey, version.toFullString())); + host.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, cursor.setArray(hostSpecNetworkPortsKey))); + } + + private static void aliasesToSlime(HostSpec spec, Cursor cursor) { + if (spec.aliases().isEmpty()) return; + Cursor aliases = cursor.setArray(aliasesKey); + for (String alias : spec.aliases()) + aliases.addString(alias); + } + + private static void toSlime(Flavor flavor, Cursor object) { + if (flavor.isConfigured()) { + object.setString(flavorKey, flavor.name()); + } + else { + NodeResources resources = flavor.resources(); + Cursor resourcesObject = object.setObject(resourcesKey); + resourcesObject.setDouble(vcpuKey, resources.vcpu()); + resourcesObject.setDouble(memoryKey, resources.memoryGb()); + resourcesObject.setDouble(diskKey, resources.diskGb()); + resourcesObject.setString(diskSpeedKey, diskSpeedToString(resources.diskSpeed())); + } + } + + + public static AllocatedHosts fromJson(byte[] json, Optional<NodeFlavors> nodeFlavors) { + return fromSlime(SlimeUtils.jsonToSlime(json).get(), nodeFlavors); + } + + public static AllocatedHosts fromSlime(Inspector inspector, Optional<NodeFlavors> nodeFlavors) { + Inspector array = inspector.field(mappingKey); + Set<HostSpec> hosts = new LinkedHashSet<>(); + array.traverse((ArrayTraverser)(i, host) -> hosts.add(hostFromSlime(host.field(hostSpecKey), nodeFlavors))); + return AllocatedHosts.withHosts(hosts); + } + + private static HostSpec hostFromSlime(Inspector object, Optional<NodeFlavors> nodeFlavors) { + Optional<ClusterMembership> membership = + object.field(hostSpecMembershipKey).valid() ? Optional.of(membershipFromSlime(object)) : Optional.empty(); + Optional<Flavor> flavor = flavorFromSlime(object, nodeFlavors); + Optional<com.yahoo.component.Version> version = + optionalString(object.field(hostSpecCurrentVespaVersionKey)).map(com.yahoo.component.Version::new); + Optional<NetworkPorts> networkPorts = + NetworkPortsSerializer.fromSlime(object.field(hostSpecNetworkPortsKey)); + return new HostSpec(object.field(hostSpecHostNameKey).asString(), aliasesFromSlime(object), flavor, membership, version, networkPorts); + } + + private static List<String> aliasesFromSlime(Inspector object) { + if ( ! object.field(aliasesKey).valid()) return Collections.emptyList(); + List<String> aliases = new ArrayList<>(); + object.field(aliasesKey).traverse((ArrayTraverser)(index, alias) -> aliases.add(alias.asString())); + return aliases; + } + + private static Optional<Flavor> flavorFromSlime(Inspector object, Optional<NodeFlavors> nodeFlavors) { + if (object.field(flavorKey).valid() && nodeFlavors.isPresent() && nodeFlavors.get().exists(object.field(flavorKey).asString())) { + return nodeFlavors.get().getFlavor(object.field(flavorKey).asString()); + } + else if (object.field(resourcesKey).valid()) { + Inspector resources = object.field(resourcesKey); + return Optional.of(new Flavor(new NodeResources(resources.field(vcpuKey).asDouble(), + resources.field(memoryKey).asDouble(), + resources.field(diskKey).asDouble(), + diskSpeedFromSlime(resources.field(diskSpeedKey))))); + } + else { + return Optional.empty(); + } + } + + private static NodeResources.DiskSpeed diskSpeedFromSlime(Inspector diskSpeed) { + if ( ! diskSpeed.valid()) return NodeResources.DiskSpeed.fast; // TODO: Remove this line after June 2019 + switch (diskSpeed.asString()) { + case "fast" : return NodeResources.DiskSpeed.fast; + case "slow" : return NodeResources.DiskSpeed.slow; + case "any" : return NodeResources.DiskSpeed.any; + default: throw new IllegalStateException("Illegal disk-speed value '" + diskSpeed.asString() + "'"); + } + } + + private static String diskSpeedToString(NodeResources.DiskSpeed diskSpeed) { + switch (diskSpeed) { + case fast : return "fast"; + case slow : return "slow"; + case any : return "any"; + default: throw new IllegalStateException("Illegal disk-speed value '" + diskSpeed + "'"); + } + + } + + private static ClusterMembership membershipFromSlime(Inspector object) { + return ClusterMembership.from(object.field(hostSpecMembershipKey).asString(), + com.yahoo.component.Version.fromString(object.field(hostSpecVespaVersionKey).asString())); + } + + private static Optional<String> optionalString(Inspector inspector) { + if ( ! inspector.valid()) return Optional.empty(); + return Optional.of(inspector.asString()); + } +} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/package-info.java b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/package-info.java new file mode 100644 index 00000000000..440228b9d3e --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.config.provision.serialization; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/AllocatedHostsTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java index 737c1047197..44aa6ed59b5 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/AllocatedHostsTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java @@ -1,7 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.provision; +package com.yahoo.config.provision.serialization; import com.yahoo.component.Version; +import com.yahoo.config.provision.AllocatedHosts; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NetworkPorts; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provisioning.FlavorsConfig; import org.junit.Test; @@ -12,12 +19,14 @@ import java.util.List; import java.util.Optional; import java.util.Set; +import static com.yahoo.config.provision.serialization.AllocatedHostsSerializer.fromJson; +import static com.yahoo.config.provision.serialization.AllocatedHostsSerializer.toJson; import static org.junit.Assert.assertEquals; /** * @author bratseth */ -public class AllocatedHostsTest { +public class AllocatedHostsSerializerTest { @Test public void testAllocatedHostsSerialization() throws IOException { @@ -29,7 +38,7 @@ public class AllocatedHostsTest { hosts.add(new HostSpec("with-aliases", List.of("alias1", "alias2"))); hosts.add(new HostSpec("allocated", - Optional.of(ClusterMembership.from("container/test/0/0", com.yahoo.component.Version.fromString("6.73.1"))))); + Optional.of(ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"))))); hosts.add(new HostSpec("flavor-from-resources-1", Collections.emptyList(), new Flavor(new NodeResources(0.5, 3.1, 4)))); hosts.add(new HostSpec("flavor-from-resources-2", @@ -47,8 +56,7 @@ public class AllocatedHostsTest { } private void assertAllocatedHosts(AllocatedHosts expectedHosts, NodeFlavors configuredFlavors) throws IOException { - AllocatedHosts deserializedHosts = AllocatedHosts.fromJson(expectedHosts.toJson(), - Optional.of(configuredFlavors)); + AllocatedHosts deserializedHosts = fromJson(toJson(expectedHosts), Optional.of(configuredFlavors)); assertEquals(expectedHosts, deserializedHosts); for (HostSpec expectedHost : expectedHosts.getHosts()) { @@ -69,7 +77,7 @@ public class AllocatedHostsTest { throw new IllegalArgumentException("No host " + hostname + " is present"); } - public NodeFlavors configuredFlavorsFrom(String flavorName, double cpu, double mem, double disk, Flavor.Type type) { + private NodeFlavors configuredFlavorsFrom(String flavorName, double cpu, double mem, double disk, Flavor.Type type) { FlavorsConfig.Builder b = new FlavorsConfig.Builder(); FlavorsConfig.Flavor.Builder flavor = new FlavorsConfig.Flavor.Builder(); flavor.name(flavorName); |