From 2aadcdb605af1a01d5a4908dc4522ca08fb26529 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Thu, 14 Dec 2017 23:58:32 +0100 Subject: Revert "Revert "Revert "Bratseth/select 1 host by index""" --- .../model/ApplicationConfigProducerRoot.java | 3 +- .../com/yahoo/vespa/model/AbstractService.java | 2 +- .../src/main/java/com/yahoo/vespa/model/Host.java | 4 +- .../java/com/yahoo/vespa/model/HostResource.java | 49 +------ .../java/com/yahoo/vespa/model/HostSystem.java | 16 +-- .../java/com/yahoo/vespa/model/admin/Admin.java | 18 ++- .../model/builder/xml/dom/DomAdminV4Builder.java | 12 +- .../model/content/cluster/ContentCluster.java | 22 ++-- .../FileDistributionConfigProvider.java | 4 +- .../model/filedistribution/FileDistributor.java | 10 +- .../com/yahoo/vespa/model/search/Dispatch.java | 4 +- .../vespa/model/search/IndexedSearchCluster.java | 2 +- .../com/yahoo/vespa/model/search/SearchNode.java | 4 +- .../model/provision/ModelProvisioningTest.java | 143 ++++++++------------- .../builder/xml/dom/DomAdminV2BuilderTest.java | 2 +- .../model/builder/xml/dom/VespaDomBuilderTest.java | 4 +- .../yahoo/vespa/model/test/VespaModelTester.java | 32 ++--- 17 files changed, 119 insertions(+), 212 deletions(-) (limited to 'config-model') diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index 912968747df..30d83a32ff9 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -29,6 +29,7 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; @@ -206,7 +207,7 @@ public class ApplicationConfigProducerRoot extends AbstractConfigProducer runsConfigServer = false; } - public String getHostname() { + public String getHostName() { return hostname; } @@ -77,7 +77,7 @@ public final class Host extends AbstractConfigProducer /** Returns the string representation of this Host object. */ public String toString() { - return "host '" + getHostname() + "'"; + return "host '" + getHostName() + "'"; } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java index 72309d4c266..5e74a2ebc8a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java @@ -7,9 +7,7 @@ import com.yahoo.config.provision.Flavor; import javax.annotation.Nullable; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; -import java.util.Comparator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -207,7 +205,7 @@ public class HostResource implements Comparable { } public HostInfo getHostInfo() { - return new HostInfo(getHostname(), services.values().stream() + return new HostInfo(getHostName(), services.values().stream() .map(Service::getServiceInfo) .collect(Collectors.toSet())); } @@ -244,11 +242,11 @@ public class HostResource implements Comparable { @Override public String toString() { - return "host '" + host.getHostname() + "'"; + return "host '" + host.getHostName() + "'"; } - public String getHostname() { - return host.getHostname(); + public String getHostName() { + return host.getHostName(); } @Override @@ -266,43 +264,4 @@ public class HostResource implements Comparable { return this.host.compareTo(other.host); } - /** - * Compares by the index of the primary membership, if both hosts are members in at least one cluster at this time. - * Compare by hostname otherwise. - */ - public int comparePrimarilyByIndexTo(HostResource other) { - Optional thisMembership = this.primaryClusterMembership(); - Optional otherMembership = other.primaryClusterMembership(); - if (thisMembership.isPresent() && otherMembership.isPresent()) - return Integer.compare(thisMembership.get().index(), otherMembership.get().index()); - else - return this.getHostname().compareTo(other.getHostname()); - } - - /** - * Picks hosts by some mixture of host name and index - * (where the mix of one or the other is decided by the last parameter). - */ - public static List pickHosts(Collection hosts, int count, int targetHostsSelectedByIndex) { - targetHostsSelectedByIndex = Math.min(Math.min(targetHostsSelectedByIndex, count), hosts.size()); - - List hostsSortedByName = new ArrayList<>(hosts); - Collections.sort(hostsSortedByName); - - List hostsSortedByIndex = new ArrayList<>(hosts); - hostsSortedByIndex.sort((a, b) -> a.comparePrimarilyByIndexTo(b)); - return pickHosts(hostsSortedByName, hostsSortedByIndex, count, targetHostsSelectedByIndex); - } - public static List pickHosts(List hostsSelectedByName, List hostsSelectedByIndex, - int count, int targetHostsSelectedByIndex) { - hostsSelectedByName = hostsSelectedByName.subList(0, Math.min(count - targetHostsSelectedByIndex, hostsSelectedByName.size())); - hostsSelectedByIndex.removeAll(hostsSelectedByName); - hostsSelectedByIndex = hostsSelectedByIndex.subList(0, Math.min(targetHostsSelectedByIndex, hostsSelectedByIndex.size())); - - List finalHosts = new ArrayList<>(); - finalHosts.addAll(hostsSelectedByName); - finalHosts.addAll(hostsSelectedByIndex); - return finalHosts; - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index 6ef6c208c0a..2db9b6acece 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -101,7 +101,7 @@ public class HostSystem extends AbstractConfigProducer { public HostResource getHost(String hostAlias) { HostSpec hostSpec = provisioner.allocateHost(hostAlias); for (HostResource resource : hostname2host.values()) { - if (resource.getHostname().equals(hostSpec.hostname())) { + if (resource.getHostName().equals(hostSpec.hostname())) { hostSpec.membership().ifPresent(resource::addClusterMembership); return resource; } @@ -114,8 +114,8 @@ public class HostSystem extends AbstractConfigProducer { HostResource hostResource = new HostResource(host); hostResource.setFlavor(hostSpec.flavor()); hostSpec.membership().ifPresent(hostResource::addClusterMembership); - hostname2host.put(host.getHostname(), hostResource); - log.log(DEBUG, () -> "Added new host resource for " + host.getHostname() + " with flavor " + hostResource.getFlavor()); + hostname2host.put(host.getHostName(), hostResource); + log.log(DEBUG, () -> "Added new host resource for " + host.getHostName() + " with flavor " + hostResource.getFlavor()); return hostResource; } @@ -136,16 +136,16 @@ public class HostSystem extends AbstractConfigProducer { retAllocatedHosts.put(host, spec.membership().orElse(null)); if (! host.getFlavor().isPresent()) { host.setFlavor(spec.flavor()); - log.log(DEBUG, () -> "Host resource " + host.getHostname() + " had no flavor, setting to " + spec.flavor()); + log.log(DEBUG, () -> "Host resource " + host.getHostName() + " had no flavor, setting to " + spec.flavor()); } } - retAllocatedHosts.keySet().forEach(host -> log.log(DEBUG, () -> "Allocated host " + host.getHostname() + " with flavor " + host.getFlavor())); + retAllocatedHosts.keySet().forEach(host -> log.log(DEBUG, () -> "Allocated host " + host.getHostName() + " with flavor " + host.getFlavor())); return retAllocatedHosts; } private Optional getExistingHost(HostSpec key) { List hosts = hostname2host.values().stream() - .filter(resource -> resource.getHostname().equals(key.hostname())) + .filter(resource -> resource.getHostName().equals(key.hostname())) .collect(Collectors.toList()); if (hosts.isEmpty()) { return Optional.empty(); @@ -156,12 +156,12 @@ public class HostSystem extends AbstractConfigProducer { } public void addBoundHost(HostResource host) { - hostname2host.put(host.getHostname(), host); + hostname2host.put(host.getHostName(), host); } Set getHostSpecs() { return getHosts().stream() - .map(host -> new HostSpec(host.getHostname(), Collections.emptyList(), + .map(host -> new HostSpec(host.getHostName(), Collections.emptyList(), host.getFlavor(), host.primaryClusterMembership())) .collect(Collectors.toCollection(LinkedHashSet::new)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index 59b7388f5bb..d892d70e0de 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -1,26 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; -import com.yahoo.cloud.config.SlobroksConfig; -import com.yahoo.cloud.config.ZookeepersConfig; import com.yahoo.cloud.config.log.LogdConfig; +import com.yahoo.cloud.config.ZookeepersConfig; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.deploy.DeployProperties; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.model.AbstractService; -import com.yahoo.vespa.model.ConfigProxy; -import com.yahoo.vespa.model.ConfigSentinel; -import com.yahoo.vespa.model.HostResource; -import com.yahoo.vespa.model.Logd; +import com.yahoo.vespa.model.*; import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.filedistribution.DummyFileDistributionConfigProducer; -import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProvider; +import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; import com.yahoo.vespa.model.filedistribution.FileDistributor; import com.yahoo.vespa.model.filedistribution.FileDistributorService; @@ -30,6 +26,8 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import com.yahoo.vespa.model.HostResource; + /** * This is the admin pseudo-plugin of the Vespa model, responsible for * creating all admin services. @@ -225,12 +223,12 @@ public class Admin extends AbstractConfigProducer implements Serializable { if (fileDistribution.getOptions().disableFiledistributor()) { DummyFileDistributionConfigProducer dummyFileDistributionConfigProducer = new DummyFileDistributionConfigProducer(fileDistribution, - host.getHost().getHostname(), + host.getHost().getHostName(), configProvider); fileDistribution.addFileDistributionConfigProducer(host.getHost(), dummyFileDistributionConfigProducer); } else { FileDistributorService fds = new FileDistributorService(fileDistribution, - host.getHost().getHostname(), + host.getHost().getHostName(), configProvider); fds.setHostResource(host); fds.initService(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index 1f7c1ba676c..a7a785cea43 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -19,7 +19,6 @@ import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; /** * Builds the admin model from a version 4 XML tag, or as a default when an admin 3 tag or no admin tag is used. @@ -120,11 +119,12 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { /** Returns the count first containers in the current model having isRetired set to the given value */ private List sortedContainerHostsFrom(ContainerModel model, int count, boolean retired) { - List hosts = model.getCluster().getContainers().stream() - .filter(container -> retired == container.isRetired()) - .map(Container::getHostResource) - .collect(Collectors.toList()); - return HostResource.pickHosts(hosts, count, 1); + List hosts = new ArrayList<>(); + for (Container container : model.getCluster().getContainers()) + if (retired == container.isRetired()) + hosts.add(container.getHostResource()); + Collections.sort(hosts); + return hosts.subList(0, Math.min(count, hosts.size())); } private void createLogserver(Admin admin, Collection hosts) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index db849d0ae50..7654fbc217b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -328,11 +328,9 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri } private List drawControllerHosts(int count, StorageGroup rootGroup, Collection containers) { - List hostsByName = drawContentHostsRecursively(count, false, rootGroup); - List hostsByIndex = drawContentHostsRecursively(count, true, rootGroup); + List hosts = drawContentHostsRecursively(count, rootGroup); // if (hosts.size() < count) // supply with containers TODO: Currently disabled due to leading to topology change problems // hosts.addAll(drawContainerHosts(count - hosts.size(), containers, new HashSet<>(hosts))); - List hosts = HostResource.pickHosts(hostsByName, hostsByIndex, count, 1); if (hosts.size() % 2 == 0) // ZK clusters of even sizes are less available (even in the size=2 case) hosts = hosts.subList(0, hosts.size()-1); return hosts; @@ -359,7 +357,7 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri // have one cluster controller List uniqueHostsWithoutClusterController = allHosts.stream() .filter(h -> ! usedHosts.contains(h)) - .filter(h -> ! hostHasClusterController(h.getHostname(), allHosts)) + .filter(h -> ! hostHasClusterController(h.getHostName(), allHosts)) .distinct() .collect(Collectors.toList()); @@ -383,7 +381,7 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri /** Returns whether any host having the given hostname has a cluster controller */ private boolean hostHasClusterController(String hostname, List hosts) { for (HostResource host : hosts) { - if ( ! host.getHostname().equals(hostname)) continue; + if ( ! host.getHostName().equals(hostname)) continue; if (hasClusterController(host)) return true; @@ -405,24 +403,20 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri */ // Note: This method cannot be changed to draw different nodes without ensuring that it will draw nodes // which overlaps with previously drawn nodes as this will prevent rolling upgrade - private List drawContentHostsRecursively(int count, boolean byIndex, StorageGroup group) { + private List drawContentHostsRecursively(int count, StorageGroup group) { Set hosts = new HashSet<>(); if (group.getNodes().isEmpty()) { int hostsPerSubgroup = (int)Math.ceil((double)count / group.getSubgroups().size()); for (StorageGroup subgroup : group.getSubgroups()) - hosts.addAll(drawContentHostsRecursively(hostsPerSubgroup, byIndex, subgroup)); + hosts.addAll(drawContentHostsRecursively(hostsPerSubgroup, subgroup)); } else { hosts.addAll(group.getNodes().stream() - .filter(node -> ! node.isRetired()) // Avoid retired controllers to avoid surprises on expiry - .map(StorageNode::getHostResource).collect(Collectors.toList())); + .filter(node -> ! node.isRetired()) // Avoid retired controllers to avoid surprises on expiry + .map(StorageNode::getHostResource).collect(Collectors.toList())); } - List sortedHosts = new ArrayList<>(hosts); - if (byIndex) - sortedHosts.sort((a, b) -> (a.comparePrimarilyByIndexTo(b))); - else // by name - Collections.sort(sortedHosts); + Collections.sort(sortedHosts); sortedHosts = sortedHosts.subList(0, Math.min(count, hosts.size())); return sortedHosts; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java index a78eb1f74f9..33cc581c3a8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java @@ -33,14 +33,14 @@ public class FileDistributionConfigProvider { fileDistributionOptions.getConfig(builder); builder.torrentport(FileDistributorService.BASEPORT + 1); builder.stateport(FileDistributorService.BASEPORT + 2); - builder.hostname(host.getHostname()); + builder.hostname(host.getHostName()); builder.filedbpath(FileDistribution.getDefaultFileDBPath().toString()); } public void getConfig(FiledistributorrpcConfig.Builder builder) { // If disabled config proxy should act as file distributor, so use config proxy port int port = (fileDistributionOptions.disableFiledistributor()) ? ConfigProxy.BASEPORT : FileDistributorService.BASEPORT; - builder.connectionspec("tcp/" + host.getHostname() + ":" + port); + builder.connectionspec("tcp/" + host.getHostName() + ":" + port); } public void getConfig(FilereferencesConfig.Builder builder) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java index f3aad88a446..8fcece3aa80 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java @@ -78,7 +78,7 @@ public class FileDistributor { } public Set getTargetHostnames() { - return getTargetHosts().stream().map(Host::getHostname).collect(Collectors.toSet()); + return getTargetHosts().stream().map(Host::getHostName).collect(Collectors.toSet()); } /** Returns the host which is the source of the files */ @@ -94,9 +94,9 @@ public class FileDistributor { public void sendDeployedFiles(FileDistribution dbHandler) { String fileSourceHost = fileSourceHost(); for (Host host : getTargetHosts()) { - if ( ! host.getHostname().equals(fileSourceHost)) { - dbHandler.sendDeployedFiles(host.getHostname(), filesToSendToHost(host)); - dbHandler.startDownload(host.getHostname(), filesToSendToHost(host)); + if ( ! host.getHostName().equals(fileSourceHost)) { + dbHandler.sendDeployedFiles(host.getHostName(), filesToSendToHost(host)); + dbHandler.startDownload(host.getHostName(), filesToSendToHost(host)); } } dbHandler.sendDeployedFiles(fileSourceHost, allFilesToSend()); @@ -108,5 +108,5 @@ public class FileDistributor { public void reloadDeployFileDistributor(FileDistribution dbHandler) { dbHandler.reloadDeployFileDistributor(); } - + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java index 87146447dd1..789c5562d9c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java @@ -96,7 +96,7 @@ public class Dispatch extends AbstractService implements SearchInterface, public int getMaxThreads() { return 500*2; } public String getHostname() { - return getHost().getHostname(); + return getHost().getHostName(); } @Override @@ -105,7 +105,7 @@ public class Dispatch extends AbstractService implements SearchInterface, } public String getDispatcherConnectSpec() { - return "tcp/" + getHost().getHostname() + ":" + getDispatchPort(); + return "tcp/" + getHost().getHostName() + ":" + getDispatchPort(); } public DispatchGroup getDispatchGroup() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 4925b88e608..8d7d289b99f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -200,7 +200,7 @@ public abstract class IndexedSearchCluster extends SearchCluster String containerClusterName = containerCluster.getName(); log.log(LogLevel.DEBUG, "Adding tld with index " + containerIndex + " for content cluster " + this.getClusterName() + ", container cluster " + containerClusterName + " (container id " + containerSubId + - ") on host " + container.getHostResource().getHostname()); + ") on host " + container.getHostResource().getHostName()); rootDispatch.addDispatcher(createTld(tldParent, container.getHostResource(), containerClusterName, containerIndex)); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index 8420a1f4b91..819d5904c25 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -183,7 +183,7 @@ public class SearchNode extends AbstractService implements * @return The connection string. */ public String getDispatcherConnectSpec() { - return "tcp/" + getHost().getHostname() + ":" + getDispatchPort(); + return "tcp/" + getHost().getHostName() + ":" + getDispatchPort(); } public int getDispatchPort() { @@ -247,7 +247,7 @@ public class SearchNode extends AbstractService implements partition(getNodeSpec().partitionId()). clustername(getClusterName()). basedir(getBaseDir()). - tlsspec("tcp/" + getHost().getHostname() + ":" + getTransactionLogServer().getTlsPort()). + tlsspec("tcp/" + getHost().getHostName() + ":" + getTransactionLogServer().getTlsPort()). tlsconfigid(getConfigId()). slobrokconfigid(getClusterConfigId()). routingconfigid(getClusterConfigId()). diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 6ea15788cf3..63d5d37598b 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -391,16 +391,16 @@ public class ModelProvisioningTest { ContainerCluster clusterControllers = cluster.getClusterControllers(); assertEquals(3, clusterControllers.getContainers().size()); assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("default28", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("default31", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("default54", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("default10", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("default13", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("default16", clusterControllers.getContainers().get(2).getHostName()); assertEquals(0, cluster.getRootGroup().getNodes().size()); assertEquals(9, cluster.getRootGroup().getSubgroups().size()); assertThat(cluster.getRootGroup().getSubgroups().get(0).getIndex(), is("0")); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().size(), is(3)); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey(), is(0)); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getConfigId(), is("bar/storage/0")); - assertEquals("default54", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); + assertEquals("default10", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getDistributionKey(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getConfigId(), is("bar/storage/1")); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(2).getDistributionKey(), is(2)); @@ -409,13 +409,13 @@ public class ModelProvisioningTest { assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().size(), is(3)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getDistributionKey(), is(3)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getConfigId(), is("bar/storage/3")); - assertEquals("default51", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); + assertEquals("default13", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getDistributionKey(), is(4)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getConfigId(), is("bar/storage/4")); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(2).getDistributionKey(), is(5)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(2).getConfigId(), is("bar/storage/5")); // ... - assertEquals("default48", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName()); + assertEquals("default16", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName()); // ... assertThat(cluster.getRootGroup().getSubgroups().get(8).getIndex(), is("8")); assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().size(), is(3)); @@ -430,23 +430,23 @@ public class ModelProvisioningTest { clusterControllers = cluster.getClusterControllers(); assertEquals(3, clusterControllers.getContainers().size()); assertEquals("baz-controllers", clusterControllers.getName()); - assertEquals("default01", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("default02", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("default27", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("default37", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("default38", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("default39", clusterControllers.getContainers().get(2).getHostName()); assertEquals(0, cluster.getRootGroup().getNodes().size()); assertEquals(27, cluster.getRootGroup().getSubgroups().size()); assertThat(cluster.getRootGroup().getSubgroups().get(0).getIndex(), is("0")); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().size(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey(), is(0)); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getConfigId(), is("baz/storage/0")); - assertEquals("default27", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); + assertEquals("default37", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); assertThat(cluster.getRootGroup().getSubgroups().get(1).getIndex(), is("1")); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().size(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getDistributionKey(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getConfigId(), is("baz/storage/1")); - assertEquals("default26", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); + assertEquals("default38", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); // ... - assertEquals("default25", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName()); + assertEquals("default39", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName()); // ... assertThat(cluster.getRootGroup().getSubgroups().get(26).getIndex(), is("26")); assertThat(cluster.getRootGroup().getSubgroups().get(26).getNodes().size(), is(1)); @@ -483,9 +483,9 @@ public class ModelProvisioningTest { ContainerCluster clusterControllers = cluster.getClusterControllers(); assertEquals(3, clusterControllers.getContainers().size()); assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("default01", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("default02", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("default08", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("default10", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("default11", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("default12", clusterControllers.getContainers().get(2).getHostName()); assertEquals(0, cluster.getRootGroup().getNodes().size()); assertEquals(8, cluster.getRootGroup().getSubgroups().size()); assertEquals(8, cluster.distributionBits()); @@ -494,19 +494,19 @@ public class ModelProvisioningTest { assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().size(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey(), is(0)); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getConfigId(), is("bar/storage/0")); - assertEquals("default08", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); + assertEquals("default10", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); // second group assertThat(cluster.getRootGroup().getSubgroups().get(1).getIndex(), is("1")); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().size(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getDistributionKey(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getConfigId(), is("bar/storage/1")); - assertEquals("default07", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); + assertEquals("default11", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); // ... last group assertThat(cluster.getRootGroup().getSubgroups().get(7).getIndex(), is("7")); assertThat(cluster.getRootGroup().getSubgroups().get(7).getNodes().size(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(7).getNodes().get(0).getDistributionKey(), is(7)); assertThat(cluster.getRootGroup().getSubgroups().get(7).getNodes().get(0).getConfigId(), is("bar/storage/7")); - assertEquals("default01", cluster.getRootGroup().getSubgroups().get(7).getNodes().get(0).getHostName()); + assertEquals("default17", cluster.getRootGroup().getSubgroups().get(7).getNodes().get(0).getHostName()); } @Test @@ -538,51 +538,16 @@ public class ModelProvisioningTest { ContentCluster cluster = model.getContentClusters().get("bar"); ContainerCluster clusterControllers = cluster.getClusterControllers(); assertEquals( 8, cluster.distributionBits()); - assertEquals("We get the closest odd number", 5, clusterControllers.getContainers().size()); + assertEquals("We get the closest odd numer", 5, clusterControllers.getContainers().size()); assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("default01", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("default02", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("default04", clusterControllers.getContainers().get(2).getHostName()); - assertEquals("default05", clusterControllers.getContainers().get(3).getHostName()); - assertEquals("default07", clusterControllers.getContainers().get(4).getHostName()); - assertEquals("default09", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); - assertEquals("default08", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getHostName()); - assertEquals("default06", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); - assertEquals("default03", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName()); - } - - @Test - public void testClusterControllersWithGroupSize2() { - String services = - "\n" + - "" + - " " + - " " + - " " + - " " + - " " + - " 2" + - " " + - " " + - " " + - " " + - " " + - ""; - - int numberOfHosts = 18; - VespaModelTester tester = new VespaModelTester(); - tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); - - // Check content clusters - ContentCluster cluster = model.getContentClusters().get("bar"); - ContainerCluster clusterControllers = cluster.getClusterControllers(); - assertEquals("We get the closest odd number", 3, clusterControllers.getContainers().size()); - assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("default01", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("default03", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("default08", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("default10", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("default11", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("default13", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("default14", clusterControllers.getContainers().get(3).getHostName()); // Should be 16 for perfect distribution ... + assertEquals("default10", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); + assertEquals("default11", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getHostName()); + assertEquals("default13", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); + assertEquals("default16", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName()); } @Test @@ -639,7 +604,7 @@ public class ModelProvisioningTest { int numberOfHosts = 19; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true, "default09", "default06", "default03"); + VespaModel model = tester.createModel(services, true, "default10", "default13", "default16"); assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); // Check content clusters @@ -647,9 +612,9 @@ public class ModelProvisioningTest { ContainerCluster clusterControllers = cluster.getClusterControllers(); assertEquals(3, clusterControllers.getContainers().size()); assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("Skipping retired default09", "default01", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("Skipping retired default03", "default04", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("Skipping retired default06", "default08", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("Skipping retired default10", "default11", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("Skipping retired default13", "default14", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("Skipping retired default16", "default17", clusterControllers.getContainers().get(2).getHostName()); } @Test @@ -666,15 +631,15 @@ public class ModelProvisioningTest { int numberOfHosts = 10; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true, "default09"); + VespaModel model = tester.createModel(services, true, "default0"); assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); // Check slobroks clusters assertEquals("Includes retired node", 1+3, model.getAdmin().getSlobroks().size()); - assertEquals("default01", model.getAdmin().getSlobroks().get(0).getHostName()); - assertEquals("default02", model.getAdmin().getSlobroks().get(1).getHostName()); - assertEquals("default10", model.getAdmin().getSlobroks().get(2).getHostName()); - assertEquals("Included in addition because it is retired", "default09", model.getAdmin().getSlobroks().get(3).getHostName()); + assertEquals("default1", model.getAdmin().getSlobroks().get(0).getHostName()); + assertEquals("default2", model.getAdmin().getSlobroks().get(1).getHostName()); + assertEquals("default3", model.getAdmin().getSlobroks().get(2).getHostName()); + assertEquals("Included in addition because it is retired", "default0", model.getAdmin().getSlobroks().get(3).getHostName()); } @Test @@ -691,16 +656,16 @@ public class ModelProvisioningTest { int numberOfHosts = 10; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true, "default09", "default08"); + VespaModel model = tester.createModel(services, true, "default3", "default4"); assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); // Check slobroks clusters assertEquals("Includes retired node", 3+2, model.getAdmin().getSlobroks().size()); - assertEquals("default01", model.getAdmin().getSlobroks().get(0).getHostName()); - assertEquals("default02", model.getAdmin().getSlobroks().get(1).getHostName()); - assertEquals("default10", model.getAdmin().getSlobroks().get(2).getHostName()); - assertEquals("Included in addition because it is retired", "default08", model.getAdmin().getSlobroks().get(3).getHostName()); - assertEquals("Included in addition because it is retired", "default09", model.getAdmin().getSlobroks().get(4).getHostName()); + assertEquals("default0", model.getAdmin().getSlobroks().get(0).getHostName()); + assertEquals("default1", model.getAdmin().getSlobroks().get(1).getHostName()); + assertEquals("default2", model.getAdmin().getSlobroks().get(2).getHostName()); + assertEquals("Included in addition because it is retired", "default3", model.getAdmin().getSlobroks().get(3).getHostName()); + assertEquals("Included in addition because it is retired", "default4", model.getAdmin().getSlobroks().get(4).getHostName()); } @Test @@ -720,19 +685,19 @@ public class ModelProvisioningTest { int numberOfHosts = 13; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true, "default12", "default03", "default02"); + VespaModel model = tester.createModel(services, true, "default0", "default10", "default11"); assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); // Check slobroks clusters // ... from cluster default assertEquals("Includes retired node", 3+3, model.getAdmin().getSlobroks().size()); - assertEquals("default04", model.getAdmin().getSlobroks().get(0).getHostName()); - assertEquals("default13", model.getAdmin().getSlobroks().get(1).getHostName()); - assertEquals("Included in addition because it is retired", "default12", model.getAdmin().getSlobroks().get(2).getHostName()); + assertEquals("default1", model.getAdmin().getSlobroks().get(0).getHostName()); + assertEquals("default2", model.getAdmin().getSlobroks().get(1).getHostName()); + assertEquals("Included in addition because it is retired", "default0", model.getAdmin().getSlobroks().get(2).getHostName()); // ... from cluster bar - assertEquals("default01", model.getAdmin().getSlobroks().get(3).getHostName()); - assertEquals("Included in addition because it is retired", "default02", model.getAdmin().getSlobroks().get(4).getHostName()); - assertEquals("Included in addition because it is retired", "default03", model.getAdmin().getSlobroks().get(5).getHostName()); + assertEquals("default12", model.getAdmin().getSlobroks().get(3).getHostName()); + assertEquals("Included in addition because it is retired", "default10", model.getAdmin().getSlobroks().get(4).getHostName()); + assertEquals("Included in addition because it is retired", "default11", model.getAdmin().getSlobroks().get(5).getHostName()); } @Test @@ -863,10 +828,10 @@ public class ModelProvisioningTest { ContainerCluster clusterControllers = cluster.getClusterControllers(); assertEquals(4, clusterControllers.getContainers().size()); assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("default04", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("default03", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("default02", clusterControllers.getContainers().get(2).getHostName()); - assertEquals("default01", clusterControllers.getContainers().get(3).getHostName()); + assertEquals("default19", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("default20", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("default21", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("default22", clusterControllers.getContainers().get(3).getHostName()); } @Test @@ -992,7 +957,7 @@ public class ModelProvisioningTest { ContainerCluster clusterControllers = cluster.getClusterControllers(); assertEquals(1, clusterControllers.getContainers().size()); assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("default01", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("default0", clusterControllers.getContainers().get(0).getHostName()); assertEquals(1, cluster.redundancy().effectiveInitialRedundancy()); // Reduced from 3*3 assertEquals(1, cluster.redundancy().effectiveFinalRedundancy()); // Reduced from 3*4 assertEquals(1, cluster.redundancy().effectiveReadyCopies()); // Reduced from 3*3 diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java index 446ceb686e8..91f6dd65bda 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java @@ -124,7 +124,7 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { assertThat(admin.getClusterControllerHosts().size(), is(1)); assertNotNull(admin.getHostSystem().getHostByHostname("test1")); for (Configserver configserver : admin.getConfigservers()) { - assertThat(configserver.getHostName(), is(not(admin.getClusterControllerHosts().get(0).getHost().getHostname()))); + assertThat(configserver.getHostName(), is(not(admin.getClusterControllerHosts().get(0).getHost().getHostName()))); for (Slobrok slobrok : admin.getSlobroks()) { assertThat(slobrok.getHostName(), is(not(configserver.getHostName()))); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilderTest.java index 77cc881e0d9..51e982f8a34 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilderTest.java @@ -122,9 +122,9 @@ public class VespaDomBuilderTest { System.out.println(hostSystem); assertThat(hostSystem.getHosts().size(), is(1)); HostResource host = hostSystem.getHosts().get(0); - assertThat(host, is(hostSystem.getHostByHostname(host.getHostname()))); + assertThat(host, is(hostSystem.getHostByHostname(host.getHostName()))); assertNotNull(hostSystem.getHost("node1")); - assertThat(hostSystem.toString(), is("host '" + host.getHostname() + "'")); + assertThat(hostSystem.toString(), is("host '" + host.getHostName() + "'")); } private VespaModel createModel(String hosts, String services) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index b88ba0276c4..901bd7f55b5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -18,6 +18,7 @@ import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; @@ -28,7 +29,7 @@ import java.util.Optional; * Helper class which sets up a system with multiple hosts. * Usage: * - * VespaModelteser tester = new VespaModelTester(); + * VespaModelteser teser = new VespaModelTester(); * tester.addHosts(count, flavor); * ... add more nodes * VesoaModel model = tester.createModel(servicesString); @@ -42,7 +43,7 @@ public class VespaModelTester { private final ConfigModelRegistry configModelRegistry; private boolean hosted = true; - private Map> hostsByFlavor = new HashMap<>(); + private Map> hosts = new HashMap<>(); public VespaModelTester() { this(new NullConfigModelRegistry()); @@ -54,31 +55,20 @@ public class VespaModelTester { /** Adds some hosts of the 'default' flavor to this system */ public Hosts addHosts(int count) { return addHosts("default", count); } - /** Adds some hosts to this system */ public Hosts addHosts(String flavor, int count) { - return addHosts(Optional.empty(), flavor, count); + List hosts = new ArrayList<>(); + for (int i = 0; i < count; i++) + hosts.add(new com.yahoo.config.model.provision.Host(flavor + i)); + this.hosts.put(flavor.isEmpty() ? "default" : flavor, hosts); + return new Hosts(hosts); } - public void addHosts(Flavor flavor, int count) { - addHosts(Optional.of(flavor), flavor.name(), count); - } - - private Hosts addHosts(Optional flavor, String flavorName, int count) { List hosts = new ArrayList<>(); - for (int i = 0; i < count; ++i) { - // Let host names sort in the opposite order of the order the hosts are added - // This allows us to test index vs. name order selection when subsets of hosts are selected from a cluster - // (for e.g cluster controllers and slobrok nodes) - String hostname = String.format("%s%02d", flavorName, count - i); - hosts.add(new Host(hostname, ImmutableList.of(), flavor)); + hosts.add(new Host(flavor.name() + i, ImmutableList.of(), Optional.of(flavor))); } - this.hostsByFlavor.put(flavorName, hosts); - - if (hosts.size() > 100) - throw new IllegalStateException("The host naming scheme is nameNN. To test more than 100 hosts, change to nameNNN"); - return new Hosts(hosts); + this.hosts.put(flavor.name(), hosts); } /** Sets whether this sets up a model for a hosted system. Default: true */ @@ -105,7 +95,7 @@ public class VespaModelTester { ApplicationPackage appPkg = modelCreatorWithMockPkg.appPkg; HostProvisioner provisioner = hosted ? - new InMemoryProvisioner(hostsByFlavor, failOnOutOfCapacity, startIndexForClusters, retiredHostNames) : + new InMemoryProvisioner(hosts, failOnOutOfCapacity, startIndexForClusters, retiredHostNames) : new SingleNodeProvisioner(); DeployState deployState = new DeployState.Builder() -- cgit v1.2.3