diff options
9 files changed, 194 insertions, 75 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index cb3308d75bf..35d82efa553 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -250,19 +250,17 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon return Optional.empty(); } - /** - * Returns the name that identifies this service for the config-sentinel. - * - * @return the name that identifies this service for the config-sentinel. - */ + /** Returns the name that identifies this service for the config-sentinel, never null */ + @Override public String getServiceName() { return getServiceType() + ((id == 1) ? "" : Integer.toString(id)); } /** * Returns the type of service. This is the class name without the - * package prefix by default. + * package prefix by default, never null */ + @Override public String getServiceType() { return toLowerCase(getShortClassName()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/NetworkPortRequestor.java b/config-model/src/main/java/com/yahoo/vespa/model/NetworkPortRequestor.java index 52319f71810..4d64ece1c77 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/NetworkPortRequestor.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/NetworkPortRequestor.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model; /** * Interface implemented by services using network ports, identifying its requirements. + * * @author arnej */ public interface NetworkPortRequestor { @@ -11,10 +12,10 @@ public interface NetworkPortRequestor { /** Returns the type of service */ String getServiceType(); - /** Returns the name that identifies this service for the config-sentinel */ + /** Returns the name that identifies this service for the config-sentinel, never null */ String getServiceName(); - /** Returns the config id */ + /** Returns the config id, never null */ String getConfigId(); /** @@ -48,6 +49,6 @@ public interface NetworkPortRequestor { /** * Return names for each port requested. * The size of the returned array must be equal to getPortCount(). - **/ + */ String[] getPortSuffixes(); } diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index 287a2e4a2a5..a44f239c1e6 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -536,7 +536,9 @@ "methods": [ "public void <init>(int, java.lang.String, java.lang.String, java.lang.String)", "public java.lang.String key()", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()", + "public boolean equals(java.lang.Object)" ], "fields": [ "public final int port", @@ -554,7 +556,9 @@ "methods": [ "public void <init>(java.util.Collection)", "public java.util.Collection allocations()", - "public int size()" + "public int size()", + "public int hashCode()", + "public boolean equals(java.lang.Object)" ], "fields": [] }, diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java b/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java index e2b2933ede3..d90f52e971b 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java @@ -9,8 +9,10 @@ import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashSet; +import java.util.List; import java.util.Optional; import java.util.Set; @@ -26,16 +28,24 @@ public class AllocatedHosts { private static final String mappingKey = "mapping"; private static final String hostSpecKey = "hostSpec"; - private static final String hostSpecHostName = "hostName"; - private static final String hostSpecMembership = "membership"; - private static final String hostSpecFlavor = "flavor"; + private static final String hostSpecHostNameKey = "hostName"; + private static final String aliasesKey = "aliases"; + private static final String hostSpecMembershipKey = "membership"; + + // Flavor can be removed when all allocated nodes are docker nodes + private static final String flavorKey = "flavor"; + + private static final String resourcesKey = "resources"; + private static final String vcpuKey = "vcpu"; + private static final String memoryKey = "memory"; + private static final String diskKey = "disk"; /** Wanted version */ - private static final String hostSpecVespaVersion = "vespaVersion"; + private static final String hostSpecVespaVersionKey = "vespaVersion"; /** Current version */ - private static final String hostSpecCurrentVespaVersion = "currentVespaVersion"; - private static final String hostSpecNetworkPorts = "ports"; + private static final String hostSpecCurrentVespaVersionKey = "currentVespaVersion"; + private static final String hostSpecNetworkPortsKey = "ports"; private final ImmutableSet<HostSpec> hosts; @@ -54,14 +64,35 @@ public class AllocatedHosts { } private void toSlime(HostSpec host, Cursor cursor) { - cursor.setString(hostSpecHostName, host.hostname()); + cursor.setString(hostSpecHostNameKey, host.hostname()); + aliasesToSlime(host, cursor); host.membership().ifPresent(membership -> { - cursor.setString(hostSpecMembership, membership.stringValue()); - cursor.setString(hostSpecVespaVersion, membership.cluster().vespaVersion().toFullString()); + cursor.setString(hostSpecMembershipKey, membership.stringValue()); + cursor.setString(hostSpecVespaVersionKey, membership.cluster().vespaVersion().toFullString()); }); - host.flavor().ifPresent(flavor -> cursor.setString(hostSpecFlavor, flavor.name())); - host.version().ifPresent(version -> cursor.setString(hostSpecCurrentVespaVersion, version.toFullString())); - host.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, cursor.setArray(hostSpecNetworkPorts))); + host.flavor().ifPresent(flavor -> toSlime(flavor, cursor)); + host.version().ifPresent(version -> cursor.setString(hostSpecCurrentVespaVersionKey, version.toFullString())); + host.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, cursor.setArray(hostSpecNetworkPortsKey))); + } + + private void aliasesToSlime(HostSpec spec, Cursor cursor) { + if (spec.aliases().isEmpty()) return; + Cursor aliases = cursor.setArray(aliasesKey); + for (String alias : spec.aliases()) + aliases.addString(alias); + } + + private void toSlime(Flavor flavor, Cursor object) { + if (flavor.isConfigured()) { + object.setString(flavorKey, flavor.name()); + } + else { + NodeResources resources = flavor.resources(); + Cursor resourcesObject = object.setObject(resourcesKey); + resourcesObject.setDouble(vcpuKey, resources.vcpu()); + resourcesObject.setDouble(memoryKey, resources.memoryGb()); + resourcesObject.setDouble(diskKey, resources.diskGb()); + } } /** Returns the hosts of this allocation */ @@ -70,34 +101,46 @@ public class AllocatedHosts { private static AllocatedHosts fromSlime(Inspector inspector, Optional<NodeFlavors> nodeFlavors) { Inspector array = inspector.field(mappingKey); Set<HostSpec> hosts = new LinkedHashSet<>(); - array.traverse(new ArrayTraverser() { - @Override - public void entry(int i, Inspector inspector) { - hosts.add(hostFromSlime(inspector.field(hostSpecKey), nodeFlavors)); - } - }); + array.traverse((ArrayTraverser)(i, host) -> hosts.add(hostFromSlime(host.field(hostSpecKey), nodeFlavors))); return new AllocatedHosts(hosts); } static HostSpec hostFromSlime(Inspector object, Optional<NodeFlavors> nodeFlavors) { Optional<ClusterMembership> membership = - object.field(hostSpecMembership).valid() ? Optional.of(membershipFromSlime(object)) : Optional.empty(); - Optional<Flavor> flavor = - object.field(hostSpecFlavor).valid() ? flavorFromSlime(object, nodeFlavors) : Optional.empty(); + object.field(hostSpecMembershipKey).valid() ? Optional.of(membershipFromSlime(object)) : Optional.empty(); + Optional<Flavor> flavor = flavorFromSlime(object, nodeFlavors); Optional<com.yahoo.component.Version> version = - optionalString(object.field(hostSpecCurrentVespaVersion)).map(com.yahoo.component.Version::new); + optionalString(object.field(hostSpecCurrentVespaVersionKey)).map(com.yahoo.component.Version::new); Optional<NetworkPorts> networkPorts = - NetworkPortsSerializer.fromSlime(object.field(hostSpecNetworkPorts)); - return new HostSpec(object.field(hostSpecHostName).asString(), Collections.emptyList(), flavor, membership, version, networkPorts); + NetworkPortsSerializer.fromSlime(object.field(hostSpecNetworkPortsKey)); + return new HostSpec(object.field(hostSpecHostNameKey).asString(), aliasesFromSlime(object), flavor, membership, version, networkPorts); } - private static ClusterMembership membershipFromSlime(Inspector object) { - return ClusterMembership.from(object.field(hostSpecMembership).asString(), - com.yahoo.component.Version.fromString(object.field(hostSpecVespaVersion).asString())); + private static List<String> aliasesFromSlime(Inspector object) { + if ( ! object.field(aliasesKey).valid()) return Collections.emptyList(); + List<String> aliases = new ArrayList<>(); + object.field(aliasesKey).traverse((ArrayTraverser)(index, alias) -> aliases.add(alias.asString())); + return aliases; } private static Optional<Flavor> flavorFromSlime(Inspector object, Optional<NodeFlavors> nodeFlavors) { - return nodeFlavors.flatMap(flavorMapper -> flavorMapper.getFlavor(object.field(hostSpecFlavor).asString())); + if (object.field(flavorKey).valid() && nodeFlavors.isPresent() && nodeFlavors.get().exists(object.field(flavorKey).asString())) { + return nodeFlavors.get().getFlavor(object.field(flavorKey).asString()); + } + else if (object.field(resourcesKey).valid()) { + Inspector resources = object.field(resourcesKey); + return Optional.of(new Flavor(new NodeResources(resources.field(vcpuKey).asDouble(), + resources.field(memoryKey).asDouble(), + resources.field(diskKey).asDouble()))); + } + else { + return Optional.empty(); + } + } + + private static ClusterMembership membershipFromSlime(Inspector object) { + return ClusterMembership.from(object.field(hostSpecMembershipKey).asString(), + com.yahoo.component.Version.fromString(object.field(hostSpecVespaVersionKey).asString())); } private static Optional<String> optionalString(Inspector inspector) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java index 760b6792006..74147a55a76 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java @@ -80,7 +80,7 @@ public class Flavor { this.resources = resources; } - /** Returns the unique identity of this flavor */ + /** Returns the unique identity of this flavor if it is configured, or the resource spec string otherwise */ public String name() { return name; } /** @@ -207,7 +207,12 @@ public class Flavor { } @Override - public String toString() { return "flavor '" + name + "'"; } + public String toString() { + if (isConfigured()) + return "flavor '" + name + "'"; + else + return name; + } public enum Type { undefined, // Default value in config (flavors.def) diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NetworkPorts.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NetworkPorts.java index 90ac3651bb2..7702edccfd4 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NetworkPorts.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NetworkPorts.java @@ -4,14 +4,17 @@ package com.yahoo.config.provision; import java.util.Collection; import java.util.List; +import java.util.Objects; /** * Models an immutable list of network port allocations + * * @author arnej */ public class NetworkPorts { public static class Allocation { + public final int port; public final String serviceType; public final String configId; @@ -19,10 +22,11 @@ public class NetworkPorts { public Allocation(int port, String serviceType, String configId, String portSuffix) { this.port = port; - this.serviceType = serviceType; - this.configId = configId; - this.portSuffix = portSuffix; + this.serviceType = Objects.requireNonNull(serviceType, "servceType cannot be null"); + this.configId = Objects.requireNonNull(configId, "configId cannot be null"); + this.portSuffix = Objects.requireNonNull(portSuffix, "portSuffix cannot be null"); } + public String key() { StringBuilder buf = new StringBuilder(); buf.append("t=").append(serviceType); @@ -30,6 +34,8 @@ public class NetworkPorts { buf.append(" suf=").append(portSuffix); return buf.toString(); } + + @Override public String toString() { StringBuilder buf = new StringBuilder(); buf.append("[port=").append(port); @@ -39,17 +45,47 @@ public class NetworkPorts { buf.append("]"); return buf.toString(); } + + @Override + public int hashCode() { + return Objects.hash(port, serviceType, configId, portSuffix); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof Allocation)) return false; + Allocation other = (Allocation)o; + if (other.port != this.port) return false; + if ( ! other.serviceType.equals(this.serviceType)) return false; + if ( ! other.configId.equals(this.configId)) return false; + if ( ! other.portSuffix.equals(this.portSuffix)) return false; + return true; + } + } - private final List<Allocation> allocations; + private final List<Allocation> allocations; // immutable list public NetworkPorts(Collection<Allocation> allocations) { this.allocations = List.copyOf(allocations); } + /** Returns a read only collection of the port allocations of this */ public Collection<Allocation> allocations() { return this.allocations; } public int size() { return allocations.size(); } + + @Override + public int hashCode() { return allocations.hashCode(); } + + @Override + public boolean equals(Object other) { + if (other == this) return true; + if ( ! (other instanceof NetworkPorts)) return false; + return ((NetworkPorts)other).allocations.equals(this.allocations); + } + } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/AllocatedHostsTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/AllocatedHostsTest.java index a39b8d3ef34..3a37c96c612 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/AllocatedHostsTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/AllocatedHostsTest.java @@ -1,52 +1,82 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; +import com.yahoo.component.Version; +import com.yahoo.config.provisioning.FlavorsConfig; import org.junit.Test; import java.io.IOException; +import java.util.Collections; import java.util.LinkedHashSet; +import java.util.List; import java.util.Optional; import java.util.Set; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; /** - * @author Ulf Lilleengen + * @author bratseth */ public class AllocatedHostsTest { - private final HostSpec h1 = new HostSpec("host1", Optional.empty()); - private final HostSpec h2 = new HostSpec("host2", Optional.empty()); - private final HostSpec h3 = new HostSpec("host3", Optional.of(ClusterMembership.from("container/test/0/0", com.yahoo.component.Version.fromString("6.73.1")))); - @Test public void testAllocatedHostsSerialization() throws IOException { + NodeFlavors configuredFlavors = configuredFlavorsFrom("C/12/45/100", 12, 45, 100, Flavor.Type.BARE_METAL); + Set<HostSpec> hosts = new LinkedHashSet<>(); - hosts.add(h1); - hosts.add(h2); - hosts.add(h3); - AllocatedHosts info = AllocatedHosts.withHosts(hosts); - assertAllocatedHosts(info); + hosts.add(new HostSpec("empty", + Optional.empty())); + hosts.add(new HostSpec("with-aliases", + List.of("alias1", "alias2"))); + hosts.add(new HostSpec("allocated", + Optional.of(ClusterMembership.from("container/test/0/0", com.yahoo.component.Version.fromString("6.73.1"))))); + hosts.add(new HostSpec("flavor-from-resources", + Collections.emptyList(), new Flavor(new NodeResources(0.5, 3.1, 4)))); + hosts.add(new HostSpec("configured-flavor", + Collections.emptyList(), configuredFlavors.getFlavorOrThrow("C/12/45/100"))); + hosts.add(new HostSpec("with-version", + Collections.emptyList(), Optional.empty(), Optional.empty(), Optional.of(Version.fromString("3.4.5")))); + hosts.add(new HostSpec("with-ports", + Collections.emptyList(), Optional.empty(), Optional.empty(), Optional.empty(), + Optional.of(new NetworkPorts(List.of(new NetworkPorts.Allocation(1234, "service1", "configId1", "suffix1"), + new NetworkPorts.Allocation(4567, "service2", "configId2", "suffix2")))))); + + assertAllocatedHosts(AllocatedHosts.withHosts(hosts), configuredFlavors); } - private void assertAllocatedHosts(AllocatedHosts info) throws IOException { - AllocatedHosts serializedAllocatedHosts = AllocatedHosts.fromJson(info.toJson(), Optional.empty()); - assertEquals(info.getHosts().size(), serializedAllocatedHosts.getHosts().size()); - assertTrue(serializedAllocatedHosts.getHosts().contains(h1)); - assertTrue(serializedAllocatedHosts.getHosts().contains(h2)); - assertTrue(serializedAllocatedHosts.getHosts().contains(h3)); - assertTrue(!getHost(h1.hostname(), serializedAllocatedHosts.getHosts()).membership().isPresent()); - assertEquals("container/test/0/0", getHost(h3.hostname(), serializedAllocatedHosts.getHosts()).membership().get().stringValue()); - assertEquals(h3.membership().get().cluster().vespaVersion(), getHost(h3.hostname(), - serializedAllocatedHosts.getHosts()).membership().get().cluster().vespaVersion()); + private void assertAllocatedHosts(AllocatedHosts expectedHosts, NodeFlavors configuredFlavors) throws IOException { + AllocatedHosts deserializedHosts = AllocatedHosts.fromJson(expectedHosts.toJson(), + Optional.of(configuredFlavors)); + + assertEquals(expectedHosts, deserializedHosts); + for (HostSpec expectedHost : expectedHosts.getHosts()) { + HostSpec deserializedHost = requireHost(expectedHost.hostname(), deserializedHosts); + assertEquals(expectedHost.hostname(), deserializedHost.hostname()); + assertEquals(expectedHost.membership(), deserializedHost.membership()); + assertEquals(expectedHost.flavor(), deserializedHost.flavor()); + assertEquals(expectedHost.version(), deserializedHost.version()); + assertEquals(expectedHost.networkPorts(), deserializedHost.networkPorts()); + assertEquals(expectedHost.aliases(), deserializedHost.aliases()); + } } - private HostSpec getHost(String hostname, Set<HostSpec> hosts) { - for (HostSpec host : hosts) + private HostSpec requireHost(String hostname, AllocatedHosts hosts) { + for (HostSpec host : hosts.getHosts()) if (host.hostname().equals(hostname)) return host; throw new IllegalArgumentException("No host " + hostname + " is present"); } + public NodeFlavors configuredFlavorsFrom(String flavorName, double cpu, double mem, double disk, Flavor.Type type) { + FlavorsConfig.Builder b = new FlavorsConfig.Builder(); + FlavorsConfig.Flavor.Builder flavor = new FlavorsConfig.Flavor.Builder(); + flavor.name(flavorName); + flavor.minDiskAvailableGb(disk); + flavor.minCpuCores(cpu); + flavor.minMainMemoryAvailableGb(mem); + flavor.environment(type.name()); + b.flavor(flavor); + return new NodeFlavors(b.build()); + } + } diff --git a/vespajlib/src/main/java/com/yahoo/slime/ArrayTraverser.java b/vespajlib/src/main/java/com/yahoo/slime/ArrayTraverser.java index 8f103ffe01d..ba7392cff5a 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/ArrayTraverser.java +++ b/vespajlib/src/main/java/com/yahoo/slime/ArrayTraverser.java @@ -5,13 +5,15 @@ package com.yahoo.slime; * Callback interface for traversing arrays. * Implement this and call Inspector.traverse() * and you will get one callback for each array entry. - **/ -public interface ArrayTraverser -{ + */ +public interface ArrayTraverser { + /** * Callback function to implement. + * * @param idx array index for the current array entry. * @param inspector accessor for the current array entry's value. - **/ - public void entry(int idx, Inspector inspector); + */ + void entry(int idx, Inspector inspector); + } diff --git a/vespajlib/src/main/java/com/yahoo/text/Lowercase.java b/vespajlib/src/main/java/com/yahoo/text/Lowercase.java index 2ee88ebe5cb..a04ba1cfe13 100644 --- a/vespajlib/src/main/java/com/yahoo/text/Lowercase.java +++ b/vespajlib/src/main/java/com/yahoo/text/Lowercase.java @@ -76,8 +76,7 @@ public final class Lowercase { * lowercasing. Vespa code should <i>never</i> do lowercasing with implicit * locale. * - * @param in - * a string to lowercase + * @param in a string to lowercase * @return a string containing only lowercase character */ public static String toLowerCase(String in) { @@ -116,4 +115,5 @@ public final class Lowercase { } return builder.toString(); } + } |