From d8b7307f8b60792d8f0028551a555764085b867f Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Wed, 31 Aug 2016 16:09:17 +0200 Subject: Allow all application hosts to talk to ZooKeeper --- .../com/yahoo/config/model/provision/Host.java | 21 ++-- .../com/yahoo/config/model/provision/Hosts.java | 108 +++++++++------------ .../model/provision/HostsXmlProvisioner.java | 21 ++-- .../model/provision/InMemoryProvisioner.java | 8 +- .../model/provision/SingleNodeProvisioner.java | 5 +- 5 files changed, 72 insertions(+), 91 deletions(-) (limited to 'config-model/src/main/java/com/yahoo/config/model') diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/Host.java b/config-model/src/main/java/com/yahoo/config/model/provision/Host.java index afe4de7ef0d..360853d0f79 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/Host.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/Host.java @@ -1,40 +1,39 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.provision; +import com.google.common.collect.ImmutableList; + import java.util.ArrayList; import java.util.List; /** - * A hostname with zero or more aliases. + * A hostname with zero or more aliases. This is immutable. * * @author hmusum */ public class Host { private final String hostname; - private final List hostAliases; + private final ImmutableList aliases; public Host(String hostname) { this.hostname = hostname; - this.hostAliases = new ArrayList<>(); + this.aliases = ImmutableList.of(); } public Host(String hostname, List hostAliases) { this.hostname = hostname; - this.hostAliases = hostAliases; + this.aliases = ImmutableList.copyOf(hostAliases); } - public String getHostname() { - return hostname; - } + public String hostname() { return hostname; } - public List getHostAliases() { - return hostAliases; - } + /** Returns an immutable list of the aliases of this node, which may be empty but never null */ + public List aliases() { return aliases; } @Override public String toString() { - return hostname + " (aliases: " + hostAliases + ")"; + return hostname + (aliases.size() > 0 ? " (aliases: " + aliases + ")" : "" ); } } diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java index bfb77612d31..1a145e6c83e 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java @@ -1,8 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.provision; +import com.google.common.collect.ImmutableMap; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.model.builder.xml.XmlHelper; -import com.yahoo.log.LogLevel; import com.yahoo.net.HostName; import com.yahoo.text.XML; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; @@ -17,26 +18,60 @@ import java.util.*; import java.util.logging.Logger; /** - * TODO: What is this? + * A collection of hosts * - * @author hmusum + * @author bratseth */ public class Hosts { public static final Logger log = Logger.getLogger(Hosts.class.getPackage().toString()); - private final HashMap hosts = new LinkedHashMap<>(); - private final Map alias2hostname = new LinkedHashMap<>(); - private final Map alias2host = new LinkedHashMap<>(); + private final ImmutableMap hosts; + public Hosts(Collection hosts) { + validateAliases(hosts); + + ImmutableMap.Builder hostsBuilder = new ImmutableMap.Builder<>(); + for (Host host : hosts) + hostsBuilder.put(host.hostname(), host); + this.hosts = hostsBuilder.build(); + + System.setProperty("zookeeper.vespa.clients", toHostnameString(hosts)); // See com.yahoo.vespa.zookeeper.ZooKeeperServer + } + + /** Throw IllegalArgumentException if host aliases breaks invariants */ + private void validateAliases(Collection hosts) { + Set aliases = new HashSet<>(); + for (Host host : hosts) { + if (host.aliases().size() > 0) { + if (host.aliases().size() < 1) + throw new IllegalArgumentException("Host '" + host.hostname() + "' must have at least one tag."); + for (String alias : host.aliases()) { + if (aliases.contains(alias)) + throw new IllegalArgumentException("Alias '" + alias + "' is used by multiple hosts."); + aliases.add(alias); + } + } + } + } + + private String toHostnameString(Collection hosts) { + StringBuilder b = new StringBuilder(); + for (Host host : hosts) + b.append(host.hostname()).append(","); + if (b.length() > 0) + b.setLength(b.length() - 1); // remove last comma + return b.toString(); + } + /** * Builds host system from a hosts.xml file * * @param hostsFile a reader for host from application package * @return the HostSystem for this application package */ - public static Hosts getHosts(Reader hostsFile) { - Hosts hosts = new Hosts(); + public static Hosts readFrom(Reader hostsFile) { + List hosts = new ArrayList<>(); Document doc; try { doc = XmlHelper.getDocumentBuilder().parse(new InputSource(hostsFile)); @@ -55,62 +90,13 @@ public class Hosts { if (hostAliases.isEmpty()) { throw new IllegalArgumentException("No host aliases defined for host '" + name + "'"); } - Host host = new Host(name, hostAliases); - hosts.addHost(host, hostAliases); - } - log.log(LogLevel.DEBUG, "Created hosts:" + hosts); - return hosts; - } - - public Collection getHosts() { - return hosts.values(); - } - - /** - * Adds one host to this host system. - * - * @param host The host to add - * @param aliases The aliases for this host. - */ - public void addHost(Host host, List aliases) { - hosts.put(host.getHostname(), host); - if ((aliases != null) && (aliases.size() > 0)) { - addHostAliases(aliases, host); + hosts.add(new Host(name, hostAliases)); } + return new Hosts(hosts); } - /** - * Add all aliases for one host - * - * @param hostAliases A list of host aliases - * @param host The Host instance to add the alias for - */ - private void addHostAliases(List hostAliases, Host host) { - if (hostAliases.size() < 1) { - throw new RuntimeException("Host '" + host.getHostname() + "' must have at least one tag."); - } - for (String alias : hostAliases) { - addHostAlias(alias, host); - } - } - - /** - * Adds an alias for the given host - * - * @param alias alias (string) for a Host - * @param host the {@link Host} to add the alias for - */ - protected void addHostAlias(String alias, Host host) { - if (alias2hostname.containsKey(alias)) { - throw new RuntimeException("Alias '" + alias + "' must be used for only one host!"); - } - alias2hostname.put(alias, host.getHostname()); - alias2host.put(alias, host); - } - - public Map getAlias2host() { - return alias2host; - } + /** Returns an immutable collection of the hosts of this */ + public Collection asCollection() { return hosts.values(); } @Override public String toString() { diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java index bf630b74272..32a7e79d278 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java @@ -22,27 +22,24 @@ public class HostsXmlProvisioner implements HostProvisioner { public static final String IMPLICIT_ADMIN_HOSTALIAS = "INTERNAL_VESPA_IMPLICIT_ADMIN"; public HostsXmlProvisioner(Reader hosts) { - this.hosts = Hosts.getHosts(hosts); + this.hosts = Hosts.readFrom(hosts); } @Override public HostSpec allocateHost(String alias) { - /** - * Some special rules to allow no admin elements as well - * as jdisc element without nodes. - */ + // Some special rules to allow no admin elements as well as jdisc element without nodes. if (alias.equals(IMPLICIT_ADMIN_HOSTALIAS)) { - if (hosts.getHosts().size() > 1) { - throw new IllegalArgumentException("More than 1 host specified (" + hosts.getHosts().size() + ") and not specified"); + if (hosts.asCollection().size() > 1) { + throw new IllegalArgumentException("More than 1 host specified (" + hosts.asCollection().size() + ") and not specified"); } else { return host2HostSpec(getFirstHost()); } } else if (alias.equals(Container.SINGLENODE_CONTAINER_SERVICESPEC)) { return host2HostSpec(getFirstHost()); } - for (Host host : hosts.getHosts()) { - if (host.getHostAliases().contains(alias)) { - return new HostSpec(host.getHostname(), host.getHostAliases()); + for (Host host : hosts.asCollection()) { + if (host.aliases().contains(alias)) { + return new HostSpec(host.hostname(), host.aliases()); } } throw new IllegalArgumentException("Unable to find host for alias '" + alias + "'"); @@ -54,11 +51,11 @@ public class HostsXmlProvisioner implements HostProvisioner { } private HostSpec host2HostSpec(Host host) { - return new HostSpec(host.getHostname(), host.getHostAliases()); + return new HostSpec(host.hostname(), host.aliases()); } private Host getFirstHost() { - return hosts.getHosts().iterator().next(); + return hosts.asCollection().iterator().next(); } } diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index b4488cef385..c0813f7ab85 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -49,12 +49,12 @@ public class InMemoryProvisioner implements HostProvisioner { /** Creates this with a set of hosts of the flavor 'default' */ public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, String ... retiredHostNames) { - this(Collections.singletonMap("default", hosts.getHosts()), failOnOutOfCapacity, 0, retiredHostNames); + this(Collections.singletonMap("default", hosts.asCollection()), failOnOutOfCapacity, 0, retiredHostNames); } /** Creates this with a set of hosts of the flavor 'default' */ public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) { - this(Collections.singletonMap("default", hosts.getHosts()), failOnOutOfCapacity, startIndexForClusters, retiredHostNames); + this(Collections.singletonMap("default", hosts.asCollection()), failOnOutOfCapacity, startIndexForClusters, retiredHostNames); } public InMemoryProvisioner(Map> hosts, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) { @@ -88,7 +88,7 @@ public class InMemoryProvisioner implements HostProvisioner { List defaultHosts = freeNodes.get("default"); if (defaultHosts.isEmpty()) throw new IllegalArgumentException("No more hosts of default flavor available"); Host newHost = freeNodes.removeValue("default", 0); - HostSpec hostSpec = new HostSpec(newHost.getHostname(), newHost.getHostAliases()); + HostSpec hostSpec = new HostSpec(newHost.hostname(), newHost.aliases()); legacyMapping.put(alias, hostSpec); return hostSpec; } @@ -141,7 +141,7 @@ public class InMemoryProvisioner implements HostProvisioner { if (freeNodes.get(flavor).isEmpty()) throw new IllegalArgumentException("No nodes of flavor '" + flavor + "' available"); Host newHost = freeNodes.removeValue(flavor, 0); ClusterMembership membership = ClusterMembership.from(clusterGroup, nextIndex++); - allocation.add(new HostSpec(newHost.getHostname(), newHost.getHostAliases(), membership)); + allocation.add(new HostSpec(newHost.hostname(), newHost.aliases(), membership)); } nextIndexInCluster.put(new Pair<>(clusterGroup.type(), clusterGroup.id()), nextIndex); diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java index 67e80ec95d6..1d5544873d9 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.provision; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.provision.*; import com.yahoo.net.HostName; @@ -31,7 +30,7 @@ public class SingleNodeProvisioner implements HostProvisioner { } catch (UnknownHostException e) { throw new RuntimeException(e); } - this.hostSpec = new HostSpec(host.getHostname(), host.getHostAliases()); + this.hostSpec = new HostSpec(host.hostname(), host.aliases()); } @Override @@ -42,7 +41,7 @@ public class SingleNodeProvisioner implements HostProvisioner { @Override public List prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { // TODO: This should fail if capacity requested is more than 1 List hosts = new ArrayList<>(); - hosts.add(new HostSpec(host.getHostname(), host.getHostAliases(), ClusterMembership.from(cluster, counter++))); + hosts.add(new HostSpec(host.hostname(), host.aliases(), ClusterMembership.from(cluster, counter++))); return hosts; } -- cgit v1.2.3