diff options
Diffstat (limited to 'config-model/src/main/java/com/yahoo')
13 files changed, 99 insertions, 51 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index 30d83a32ff9..912968747df 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -29,7 +29,6 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; @@ -207,7 +206,7 @@ public class ApplicationConfigProducerRoot extends AbstractConfigProducer<Abstra builder.vespaVersion(vespaVersion.toSerializedForm()); for (HostResource modelHost : getHostSystem().getHosts()) { builder.hosts(new Hosts.Builder() - .name(modelHost.getHostName()) + .name(modelHost.getHostname()) .services(getServices(modelHost)) ); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index 659d5b65a4c..3b75be5167d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -315,7 +315,7 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon * @return The hostname on which this service runs. */ public String getHostName() { - return hostResource.getHostName(); + return hostResource.getHostname(); } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Host.java b/config-model/src/main/java/com/yahoo/vespa/model/Host.java index e4e1e029a1a..0adfe9e4bdb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/Host.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/Host.java @@ -67,7 +67,7 @@ public final class Host extends AbstractConfigProducer<AbstractConfigProducer<?> runsConfigServer = false; } - public String getHostName() { + public String getHostname() { return hostname; } @@ -77,7 +77,7 @@ public final class Host extends AbstractConfigProducer<AbstractConfigProducer<?> /** Returns the string representation of this Host object. */ public String toString() { - return "host '" + getHostName() + "'"; + return "host '" + getHostname() + "'"; } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java index 5e74a2ebc8a..72309d4c266 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java @@ -7,7 +7,9 @@ import com.yahoo.config.provision.Flavor; import javax.annotation.Nullable; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -205,7 +207,7 @@ public class HostResource implements Comparable<HostResource> { } public HostInfo getHostInfo() { - return new HostInfo(getHostName(), services.values().stream() + return new HostInfo(getHostname(), services.values().stream() .map(Service::getServiceInfo) .collect(Collectors.toSet())); } @@ -242,11 +244,11 @@ public class HostResource implements Comparable<HostResource> { @Override public String toString() { - return "host '" + host.getHostName() + "'"; + return "host '" + host.getHostname() + "'"; } - public String getHostName() { - return host.getHostName(); + public String getHostname() { + return host.getHostname(); } @Override @@ -264,4 +266,43 @@ public class HostResource implements Comparable<HostResource> { return this.host.compareTo(other.host); } + /** + * Compares by the index of the primary membership, if both hosts are members in at least one cluster at this time. + * Compare by hostname otherwise. + */ + public int comparePrimarilyByIndexTo(HostResource other) { + Optional<ClusterMembership> thisMembership = this.primaryClusterMembership(); + Optional<ClusterMembership> otherMembership = other.primaryClusterMembership(); + if (thisMembership.isPresent() && otherMembership.isPresent()) + return Integer.compare(thisMembership.get().index(), otherMembership.get().index()); + else + return this.getHostname().compareTo(other.getHostname()); + } + + /** + * Picks hosts by some mixture of host name and index + * (where the mix of one or the other is decided by the last parameter). + */ + public static List<HostResource> pickHosts(Collection<HostResource> hosts, int count, int targetHostsSelectedByIndex) { + targetHostsSelectedByIndex = Math.min(Math.min(targetHostsSelectedByIndex, count), hosts.size()); + + List<HostResource> hostsSortedByName = new ArrayList<>(hosts); + Collections.sort(hostsSortedByName); + + List<HostResource> hostsSortedByIndex = new ArrayList<>(hosts); + hostsSortedByIndex.sort((a, b) -> a.comparePrimarilyByIndexTo(b)); + return pickHosts(hostsSortedByName, hostsSortedByIndex, count, targetHostsSelectedByIndex); + } + public static List<HostResource> pickHosts(List<HostResource> hostsSelectedByName, List<HostResource> hostsSelectedByIndex, + int count, int targetHostsSelectedByIndex) { + hostsSelectedByName = hostsSelectedByName.subList(0, Math.min(count - targetHostsSelectedByIndex, hostsSelectedByName.size())); + hostsSelectedByIndex.removeAll(hostsSelectedByName); + hostsSelectedByIndex = hostsSelectedByIndex.subList(0, Math.min(targetHostsSelectedByIndex, hostsSelectedByIndex.size())); + + List<HostResource> finalHosts = new ArrayList<>(); + finalHosts.addAll(hostsSelectedByName); + finalHosts.addAll(hostsSelectedByIndex); + return finalHosts; + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index 2db9b6acece..6ef6c208c0a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -101,7 +101,7 @@ public class HostSystem extends AbstractConfigProducer<Host> { public HostResource getHost(String hostAlias) { HostSpec hostSpec = provisioner.allocateHost(hostAlias); for (HostResource resource : hostname2host.values()) { - if (resource.getHostName().equals(hostSpec.hostname())) { + if (resource.getHostname().equals(hostSpec.hostname())) { hostSpec.membership().ifPresent(resource::addClusterMembership); return resource; } @@ -114,8 +114,8 @@ public class HostSystem extends AbstractConfigProducer<Host> { HostResource hostResource = new HostResource(host); hostResource.setFlavor(hostSpec.flavor()); hostSpec.membership().ifPresent(hostResource::addClusterMembership); - hostname2host.put(host.getHostName(), hostResource); - log.log(DEBUG, () -> "Added new host resource for " + host.getHostName() + " with flavor " + hostResource.getFlavor()); + hostname2host.put(host.getHostname(), hostResource); + log.log(DEBUG, () -> "Added new host resource for " + host.getHostname() + " with flavor " + hostResource.getFlavor()); return hostResource; } @@ -136,16 +136,16 @@ public class HostSystem extends AbstractConfigProducer<Host> { retAllocatedHosts.put(host, spec.membership().orElse(null)); if (! host.getFlavor().isPresent()) { host.setFlavor(spec.flavor()); - log.log(DEBUG, () -> "Host resource " + host.getHostName() + " had no flavor, setting to " + spec.flavor()); + log.log(DEBUG, () -> "Host resource " + host.getHostname() + " had no flavor, setting to " + spec.flavor()); } } - retAllocatedHosts.keySet().forEach(host -> log.log(DEBUG, () -> "Allocated host " + host.getHostName() + " with flavor " + host.getFlavor())); + retAllocatedHosts.keySet().forEach(host -> log.log(DEBUG, () -> "Allocated host " + host.getHostname() + " with flavor " + host.getFlavor())); return retAllocatedHosts; } private Optional<HostResource> getExistingHost(HostSpec key) { List<HostResource> hosts = hostname2host.values().stream() - .filter(resource -> resource.getHostName().equals(key.hostname())) + .filter(resource -> resource.getHostname().equals(key.hostname())) .collect(Collectors.toList()); if (hosts.isEmpty()) { return Optional.empty(); @@ -156,12 +156,12 @@ public class HostSystem extends AbstractConfigProducer<Host> { } public void addBoundHost(HostResource host) { - hostname2host.put(host.getHostName(), host); + hostname2host.put(host.getHostname(), host); } Set<HostSpec> getHostSpecs() { return getHosts().stream() - .map(host -> new HostSpec(host.getHostName(), Collections.emptyList(), + .map(host -> new HostSpec(host.getHostname(), Collections.emptyList(), host.getFlavor(), host.primaryClusterMembership())) .collect(Collectors.toCollection(LinkedHashSet::new)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index d892d70e0de..59b7388f5bb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -1,22 +1,26 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; -import com.yahoo.cloud.config.log.LogdConfig; +import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.cloud.config.ZookeepersConfig; +import com.yahoo.cloud.config.log.LogdConfig; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.deploy.DeployProperties; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.model.*; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.ConfigProxy; +import com.yahoo.vespa.model.ConfigSentinel; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.Logd; import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.filedistribution.DummyFileDistributionConfigProducer; -import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProvider; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; +import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProvider; import com.yahoo.vespa.model.filedistribution.FileDistributor; import com.yahoo.vespa.model.filedistribution.FileDistributorService; @@ -26,8 +30,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import com.yahoo.vespa.model.HostResource; - /** * This is the admin pseudo-plugin of the Vespa model, responsible for * creating all admin services. @@ -223,12 +225,12 @@ public class Admin extends AbstractConfigProducer implements Serializable { if (fileDistribution.getOptions().disableFiledistributor()) { DummyFileDistributionConfigProducer dummyFileDistributionConfigProducer = new DummyFileDistributionConfigProducer(fileDistribution, - host.getHost().getHostName(), + host.getHost().getHostname(), configProvider); fileDistribution.addFileDistributionConfigProducer(host.getHost(), dummyFileDistributionConfigProducer); } else { FileDistributorService fds = new FileDistributorService(fileDistribution, - host.getHost().getHostName(), + host.getHost().getHostname(), configProvider); fds.setHostResource(host); fds.initService(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index a7a785cea43..1f7c1ba676c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -19,6 +19,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; /** * Builds the admin model from a version 4 XML tag, or as a default when an admin 3 tag or no admin tag is used. @@ -119,12 +120,11 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { /** Returns the count first containers in the current model having isRetired set to the given value */ private List<HostResource> sortedContainerHostsFrom(ContainerModel model, int count, boolean retired) { - List<HostResource> hosts = new ArrayList<>(); - for (Container container : model.getCluster().getContainers()) - if (retired == container.isRetired()) - hosts.add(container.getHostResource()); - Collections.sort(hosts); - return hosts.subList(0, Math.min(count, hosts.size())); + List<HostResource> hosts = model.getCluster().getContainers().stream() + .filter(container -> retired == container.isRetired()) + .map(Container::getHostResource) + .collect(Collectors.toList()); + return HostResource.pickHosts(hosts, count, 1); } private void createLogserver(Admin admin, Collection<HostResource> hosts) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 7654fbc217b..db849d0ae50 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -328,9 +328,11 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri } private List<HostResource> drawControllerHosts(int count, StorageGroup rootGroup, Collection<ContainerModel> containers) { - List<HostResource> hosts = drawContentHostsRecursively(count, rootGroup); + List<HostResource> hostsByName = drawContentHostsRecursively(count, false, rootGroup); + List<HostResource> hostsByIndex = drawContentHostsRecursively(count, true, rootGroup); // if (hosts.size() < count) // supply with containers TODO: Currently disabled due to leading to topology change problems // hosts.addAll(drawContainerHosts(count - hosts.size(), containers, new HashSet<>(hosts))); + List<HostResource> hosts = HostResource.pickHosts(hostsByName, hostsByIndex, count, 1); if (hosts.size() % 2 == 0) // ZK clusters of even sizes are less available (even in the size=2 case) hosts = hosts.subList(0, hosts.size()-1); return hosts; @@ -357,7 +359,7 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri // have one cluster controller List<HostResource> uniqueHostsWithoutClusterController = allHosts.stream() .filter(h -> ! usedHosts.contains(h)) - .filter(h -> ! hostHasClusterController(h.getHostName(), allHosts)) + .filter(h -> ! hostHasClusterController(h.getHostname(), allHosts)) .distinct() .collect(Collectors.toList()); @@ -381,7 +383,7 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri /** Returns whether any host having the given hostname has a cluster controller */ private boolean hostHasClusterController(String hostname, List<HostResource> hosts) { for (HostResource host : hosts) { - if ( ! host.getHostName().equals(hostname)) continue; + if ( ! host.getHostname().equals(hostname)) continue; if (hasClusterController(host)) return true; @@ -403,20 +405,24 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri */ // Note: This method cannot be changed to draw different nodes without ensuring that it will draw nodes // which overlaps with previously drawn nodes as this will prevent rolling upgrade - private List<HostResource> drawContentHostsRecursively(int count, StorageGroup group) { + private List<HostResource> drawContentHostsRecursively(int count, boolean byIndex, StorageGroup group) { Set<HostResource> hosts = new HashSet<>(); if (group.getNodes().isEmpty()) { int hostsPerSubgroup = (int)Math.ceil((double)count / group.getSubgroups().size()); for (StorageGroup subgroup : group.getSubgroups()) - hosts.addAll(drawContentHostsRecursively(hostsPerSubgroup, subgroup)); + hosts.addAll(drawContentHostsRecursively(hostsPerSubgroup, byIndex, subgroup)); } else { hosts.addAll(group.getNodes().stream() - .filter(node -> ! node.isRetired()) // Avoid retired controllers to avoid surprises on expiry - .map(StorageNode::getHostResource).collect(Collectors.toList())); + .filter(node -> ! node.isRetired()) // Avoid retired controllers to avoid surprises on expiry + .map(StorageNode::getHostResource).collect(Collectors.toList())); } + List<HostResource> sortedHosts = new ArrayList<>(hosts); - Collections.sort(sortedHosts); + if (byIndex) + sortedHosts.sort((a, b) -> (a.comparePrimarilyByIndexTo(b))); + else // by name + Collections.sort(sortedHosts); sortedHosts = sortedHosts.subList(0, Math.min(count, hosts.size())); return sortedHosts; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java index 33cc581c3a8..a78eb1f74f9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java @@ -33,14 +33,14 @@ public class FileDistributionConfigProvider { fileDistributionOptions.getConfig(builder); builder.torrentport(FileDistributorService.BASEPORT + 1); builder.stateport(FileDistributorService.BASEPORT + 2); - builder.hostname(host.getHostName()); + builder.hostname(host.getHostname()); builder.filedbpath(FileDistribution.getDefaultFileDBPath().toString()); } public void getConfig(FiledistributorrpcConfig.Builder builder) { // If disabled config proxy should act as file distributor, so use config proxy port int port = (fileDistributionOptions.disableFiledistributor()) ? ConfigProxy.BASEPORT : FileDistributorService.BASEPORT; - builder.connectionspec("tcp/" + host.getHostName() + ":" + port); + builder.connectionspec("tcp/" + host.getHostname() + ":" + port); } public void getConfig(FilereferencesConfig.Builder builder) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java index 8fcece3aa80..f3aad88a446 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java @@ -78,7 +78,7 @@ public class FileDistributor { } public Set<String> getTargetHostnames() { - return getTargetHosts().stream().map(Host::getHostName).collect(Collectors.toSet()); + return getTargetHosts().stream().map(Host::getHostname).collect(Collectors.toSet()); } /** Returns the host which is the source of the files */ @@ -94,9 +94,9 @@ public class FileDistributor { public void sendDeployedFiles(FileDistribution dbHandler) { String fileSourceHost = fileSourceHost(); for (Host host : getTargetHosts()) { - if ( ! host.getHostName().equals(fileSourceHost)) { - dbHandler.sendDeployedFiles(host.getHostName(), filesToSendToHost(host)); - dbHandler.startDownload(host.getHostName(), filesToSendToHost(host)); + if ( ! host.getHostname().equals(fileSourceHost)) { + dbHandler.sendDeployedFiles(host.getHostname(), filesToSendToHost(host)); + dbHandler.startDownload(host.getHostname(), filesToSendToHost(host)); } } dbHandler.sendDeployedFiles(fileSourceHost, allFilesToSend()); @@ -108,5 +108,5 @@ public class FileDistributor { public void reloadDeployFileDistributor(FileDistribution dbHandler) { dbHandler.reloadDeployFileDistributor(); } - + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java index 789c5562d9c..87146447dd1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java @@ -96,7 +96,7 @@ public class Dispatch extends AbstractService implements SearchInterface, public int getMaxThreads() { return 500*2; } public String getHostname() { - return getHost().getHostName(); + return getHost().getHostname(); } @Override @@ -105,7 +105,7 @@ public class Dispatch extends AbstractService implements SearchInterface, } public String getDispatcherConnectSpec() { - return "tcp/" + getHost().getHostName() + ":" + getDispatchPort(); + return "tcp/" + getHost().getHostname() + ":" + getDispatchPort(); } public DispatchGroup getDispatchGroup() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 8d7d289b99f..4925b88e608 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -200,7 +200,7 @@ public abstract class IndexedSearchCluster extends SearchCluster String containerClusterName = containerCluster.getName(); log.log(LogLevel.DEBUG, "Adding tld with index " + containerIndex + " for content cluster " + this.getClusterName() + ", container cluster " + containerClusterName + " (container id " + containerSubId + - ") on host " + container.getHostResource().getHostName()); + ") on host " + container.getHostResource().getHostname()); rootDispatch.addDispatcher(createTld(tldParent, container.getHostResource(), containerClusterName, containerIndex)); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index 819d5904c25..8420a1f4b91 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -183,7 +183,7 @@ public class SearchNode extends AbstractService implements * @return The connection string. */ public String getDispatcherConnectSpec() { - return "tcp/" + getHost().getHostName() + ":" + getDispatchPort(); + return "tcp/" + getHost().getHostname() + ":" + getDispatchPort(); } public int getDispatchPort() { @@ -247,7 +247,7 @@ public class SearchNode extends AbstractService implements partition(getNodeSpec().partitionId()). clustername(getClusterName()). basedir(getBaseDir()). - tlsspec("tcp/" + getHost().getHostName() + ":" + getTransactionLogServer().getTlsPort()). + tlsspec("tcp/" + getHost().getHostname() + ":" + getTransactionLogServer().getTlsPort()). tlsconfigid(getConfigId()). slobrokconfigid(getClusterConfigId()). routingconfigid(getClusterConfigId()). |