diff options
97 files changed, 1004 insertions, 605 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 378439a4fbb..45de080dd1c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,9 +11,9 @@ vespa_detect_build_platform() message("-- Vespa build platform is ${VESPA_OS_DISTRO} ${VESPA_OS_DISTRO_VERSION}") vespa_use_default_cxx_compiler() vespa_use_default_java_home() -vespa_use_default_build_settings() project(vespa CXX C) +vespa_use_default_build_settings() # allows import of project in CLion on OSX if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index aede68a1dd1..1cc5c93c28a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -223,7 +223,7 @@ public class ApplicationConfigProducerRoot extends AbstractConfigProducer<Abstra @Override public void getConfig(ModelConfig.Builder builder) { builder.vespaVersion(vespaVersion.toFullString()); - for (HostResource modelHost : getHostSystem().getHosts()) { + for (HostResource modelHost : hostSystem().getHosts()) { builder.hosts(new Hosts.Builder() .name(modelHost.getHostname()) .services(getServices(modelHost)) @@ -278,7 +278,7 @@ public class ApplicationConfigProducerRoot extends AbstractConfigProducer<Abstra } @Override - public HostSystem getHostSystem() { + public HostSystem hostSystem() { return hostSystem; } diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java index c2834847423..48c21f370f4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java @@ -265,13 +265,8 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce } } - /** - * Returns the one and only HostSystem of the root node - * Must be overridden by root node. - */ - public HostSystem getHostSystem() { - return getRoot().getHostSystem(); - } + /** Returns the one and only HostSystem of the root node. Must be overridden by root node. */ + public HostSystem hostSystem() { return getRoot().hostSystem(); } public AbstractConfigProducerRoot getRoot() { return parent == null ? null : parent.getRoot(); diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java index 74ab4504136..13f271ebe9d 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java @@ -123,7 +123,7 @@ public class MockRoot extends AbstractConfigProducerRoot { return fileDistributor; } - public HostSystem getHostSystem() { + public HostSystem hostSystem() { return hostSystem; } diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java index 25c97cba04b..81f5d303d56 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java @@ -61,6 +61,6 @@ public class TestRoot { * @return A list of hosts. */ public List<HostResource> getHosts() { - return model.getHostSystem().getHosts(); + return model.hostSystem().getHosts(); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java index 55f101a4877..9b734cacede 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java @@ -25,9 +25,20 @@ public class AttributesImplicitWord extends Processor { @Override public void process(boolean validate, boolean documentsOnly) { for (ImmutableSDField field : search.allConcreteFields()) { - if (fieldImplicitlyWordMatch(field)) { - field.getMatching().setType(Matching.Type.WORD); - } + processFieldRecursive(field); + } + } + + private void processFieldRecursive(ImmutableSDField field) { + processField(field); + for (ImmutableSDField structField : field.getStructFields()) { + processFieldRecursive(structField); + } + } + + private void processField(ImmutableSDField field) { + if (fieldImplicitlyWordMatch(field)) { + field.getMatching().setType(Matching.Type.WORD); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java index 13fe3f24d69..9338ff300ea 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java @@ -27,12 +27,25 @@ public class WordMatch extends Processor { public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { - if ( ! field.getMatching().getType().equals(Matching.Type.WORD)) continue; + processFieldRecursive(field); + } + } + + private void processFieldRecursive(SDField field) { + processField(field); + for (SDField structField : field.getStructFields()) { + processField(structField); + } + } - field.setStemming(Stemming.NONE); - field.getNormalizing().inferLowercase(); - field.addQueryCommand("word"); + private void processField(SDField field) { + if (!field.getMatching().getType().equals(Matching.Type.WORD)) { + return; } + field.setStemming(Stemming.NONE); + field.getNormalizing().inferLowercase(); + field.addQueryCommand("word"); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java index dc1fb0506f0..bf86bc4a453 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java @@ -23,7 +23,7 @@ public interface ConfigProducer extends com.yahoo.config.ConfigInstance.Producer String getConfigId(); /** Returns the one and only HostSystem of the root node */ - HostSystem getHostSystem(); + HostSystem hostSystem(); /** Returns the user configs of this */ UserConfigRepo getUserConfigs(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index 3ac5f794426..eda562bea5a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -63,8 +63,8 @@ public class HostSystem extends AbstractConfigProducer<Host> { /** * Returns the host with the given hostname. * - * @param name the hostname of the host. - * @return the host with the given hostname. + * @param name the hostname of the host + * @return the host with the given hostname, or null if no such host */ public HostResource getHostByHostname(String name) { // TODO: please eliminate the following ugly hack @@ -106,12 +106,10 @@ public class HostSystem extends AbstractConfigProducer<Host> { @Override public String toString() { - StringBuilder sb = new StringBuilder(); - for (HostResource host : hostname2host.values()) { - sb.append(host).append(","); - } - if (sb.length() > 0) sb.deleteCharAt(sb.length() - 1); - return sb.toString(); + return "hosts [" + hostname2host.values().stream() + .map(host -> host.getHostname()) + .collect(Collectors.joining(", ")) + + "]"; } public HostResource getHost(String hostAlias) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 08956b272f6..ad61e0f48a3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -176,7 +176,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri - HostSystem hostSystem = root.getHostSystem(); + HostSystem hostSystem = root.hostSystem(); if (complete) { // create a a completed, frozen model configModelRepo.readConfigModels(deployState, this, builder, root, configModelRegistry); addServiceClusters(deployState, builder); @@ -261,16 +261,16 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri } /** Returns the one and only HostSystem of this VespaModel */ - public HostSystem getHostSystem() { - return root.getHostSystem(); + public HostSystem hostSystem() { + return root.hostSystem(); } /** Return a collection of all hostnames used in this application */ @Override public Set<HostInfo> getHosts() { - return getHostSystem().getHosts().stream() - .map(HostResource::getHostInfo) - .collect(Collectors.toCollection(LinkedHashSet::new)); + return hostSystem().getHosts().stream() + .map(HostResource::getHostInfo) + .collect(Collectors.toCollection(LinkedHashSet::new)); } public FileDistributor getFileDistributor() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index d31c411a39e..7484e0cd9a0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -262,27 +262,21 @@ public class Admin extends AbstractConfigProducer implements Serializable { private void addFileDistribution(HostResource host) { FileDistributor fileDistributor = fileDistribution.getFileDistributor(); - HostResource deployHost = getHostSystem().getHostByHostname(fileDistributor.fileSourceHost()); - if (deployHostIsMissing(deployHost)) { - throw new RuntimeException("Could not find host in the application's host system: '" + - fileDistributor.fileSourceHost() + "'. Hostsystem=" + getHostSystem()); - } + HostResource hostResource = hostSystem().getHostByHostname(fileDistributor.fileSourceHost()); + if (hostResource == null && ! multitenant) + throw new IllegalArgumentException("Could not find " + host + " in the application's " + hostSystem()); FileDistributionConfigProvider configProvider = new FileDistributionConfigProvider(fileDistribution, fileDistributor, - host == deployHost, + host == hostResource, host.getHost()); fileDistribution.addFileDistributionConfigProducer(host.getHost(), configProvider); } - private boolean deployHostIsMissing(HostResource deployHost) { - return !multitenant && deployHost == null; - } - // If not configured by user: Use default setup: max 3 slobroks, 1 on the default configserver host private List<Slobrok> createDefaultSlobrokSetup(DeployLogger deployLogger) { - List<HostResource> hosts = getHostSystem().getHosts(); + List<HostResource> hosts = hostSystem().getHosts(); List<Slobrok> slobs = new ArrayList<>(); if (logserver != null) { Slobrok slobrok = new Slobrok(this, 0); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index 9de777a96ef..a115b25cbd9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -55,7 +55,7 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu List<Configserver> getConfigServersFromSpec(DeployLogger deployLogger, AbstractConfigProducer parent) { List<Configserver> configservers = new ArrayList<>(); for (ConfigServerSpec spec : configServerSpecs) { - HostSystem hostSystem = parent.getHostSystem(); + HostSystem hostSystem = parent.hostSystem(); HostResource host = new HostResource(Host.createConfigServerHost(hostSystem, spec.getHostName())); hostSystem.addBoundHost(host); Configserver configserver = new Configserver(parent, spec.getHostName(), spec.getConfigServerPort()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index fbd3f353d95..61d0cd7cd1e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -62,7 +62,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { private void assignSlobroks(DeployLogger deployLogger, NodesSpecification nodesSpecification, Admin admin) { if (nodesSpecification.isDedicated()) { - createSlobroks(deployLogger, admin, allocateHosts(admin.getHostSystem(), "slobroks", nodesSpecification)); + createSlobroks(deployLogger, admin, allocateHosts(admin.hostSystem(), "slobroks", nodesSpecification)); } else { createSlobroks(deployLogger, admin, pickContainerHostsForSlobrok(nodesSpecification.count(), 2)); @@ -73,7 +73,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { if (nodesSpecification.count() > 1) throw new IllegalArgumentException("You can only request a single log server"); if (deployState.getProperties().applicationId().instance().isTester()) return; // No logserver is needed on tester applications if (nodesSpecification.isDedicated()) { - Collection<HostResource> hosts = allocateHosts(admin.getHostSystem(), "logserver", nodesSpecification); + Collection<HostResource> hosts = allocateHosts(admin.hostSystem(), "logserver", nodesSpecification); if (hosts.isEmpty()) return; // No log server can be created (and none is needed) Logserver logserver = createLogserver(deployState.getDeployLogger(), admin, hosts); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java index 2718db46be9..97b78e1b9b1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java @@ -122,7 +122,7 @@ public class VespaDomBuilder extends VespaModelBuilder { T t = doBuild(deployState, ancestor, producerSpec); if (t instanceof AbstractService) { - initializeService((AbstractService)t, deployState, ancestor.getHostSystem(), producerSpec); + initializeService((AbstractService)t, deployState, ancestor.hostSystem(), producerSpec); } else { initializeProducer(t, deployState, producerSpec); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 16ecf5f761c..3e9ebaeccd4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -607,7 +607,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { /** Creates a single host when there is no nodes tag */ private HostResource allocateSingleNodeHost(ApplicationContainerCluster cluster, DeployLogger logger, Element containerElement, ConfigModelContext context) { DeployState deployState = context.getDeployState(); - HostSystem hostSystem = cluster.getHostSystem(); + HostSystem hostSystem = cluster.hostSystem(); if (deployState.isHosted()) { Optional<HostResource> singleContentHost = getHostResourceFromContentClusters(cluster, containerElement, context); if (singleContentHost.isPresent()) { // there is a content cluster; put the container on its first node @@ -631,7 +631,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private List<ApplicationContainer> createNodesFromNodeCount(ApplicationContainerCluster cluster, Element nodesElement, ConfigModelContext context) { NodesSpecification nodesSpecification = NodesSpecification.from(new ModelElement(nodesElement), context); - Map<HostResource, ClusterMembership> hosts = nodesSpecification.provision(cluster.getRoot().getHostSystem(), + Map<HostResource, ClusterMembership> hosts = nodesSpecification.provision(cluster.getRoot().hostSystem(), ClusterSpec.Type.container, ClusterSpec.Id.from(cluster.getName()), log); @@ -645,8 +645,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { context.getDeployState().getWantedNodeVespaVersion(), false); Map<HostResource, ClusterMembership> hosts = - cluster.getRoot().getHostSystem().allocateHosts(clusterSpec, - Capacity.fromRequiredNodeType(type), 1, log); + cluster.getRoot().hostSystem().allocateHosts(clusterSpec, + Capacity.fromRequiredNodeType(type), 1, log); return createNodesFromHosts(context.getDeployLogger(), hosts, cluster); } @@ -663,7 +663,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Map<HostResource, ClusterMembership> hosts = StorageGroup.provisionHosts(nodeSpecification, referenceId, - cluster.getRoot().getHostSystem(), + cluster.getRoot().hostSystem(), context.getDeployLogger()); return createNodesFromHosts(context.getDeployLogger(), hosts, cluster); } @@ -690,7 +690,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Map<HostResource, ClusterMembership> hosts = StorageGroup.provisionHosts(nodesSpec, contentServices.get(0).getAttribute("id"), - cluster.getRoot().getHostSystem(), + cluster.getRoot().hostSystem(), context.getDeployLogger()); return Optional.of(hosts.keySet().iterator().next()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index 8533c8d430f..b3f2b81014b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -5,7 +5,6 @@ import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.chain.model.ChainSpecification; import com.yahoo.component.provider.ComponentRegistry; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.ConfigModel; import com.yahoo.config.model.ConfigModelContext; @@ -15,12 +14,10 @@ import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.SimpleConfigProducer; import com.yahoo.vespa.model.admin.Admin; -import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ApplicationContainer; @@ -56,7 +53,6 @@ public class Content extends ConfigModel { private ContentCluster cluster; private Optional<ApplicationContainerCluster> ownedIndexingCluster = Optional.empty(); - private final boolean isHosted; // Dependencies to other models private final AdminModel adminModel; @@ -68,7 +64,6 @@ public class Content extends ConfigModel { public Content(ConfigModelContext modelContext, AdminModel adminModel, Collection<ContainerModel> containers) { super(modelContext); modelContext.getParentProducer().getRoot(); - isHosted = modelContext.getDeployState().isHosted(); this.adminModel = adminModel; this.containers = containers; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java index 7ab5ee9fd80..56945264789 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java @@ -1,9 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; -import java.util.List; -import java.util.stream.Collectors; - /** * Class used to validate that hierarchic distribution is correctly setup when having an indexed content cluster. * @@ -30,7 +27,7 @@ public class IndexedHierarchicDistributionValidator { this.dispatchPolicy = dispatchPolicy; } - public void validate() throws Exception { + public void validate() { validateThatWeHaveOneGroupLevel(); validateThatLeafGroupsHasEqualNumberOfNodes(); validateThatLeafGroupsCountIsAFactorOfRedundancy(clusterName, redundancy.effectiveFinalRedundancy(), rootGroup.getSubgroups().size()); @@ -84,10 +81,6 @@ public class IndexedHierarchicDistributionValidator { } } - private List<StorageNode> nonRetired(List<StorageNode> nodes) { - return nodes.stream().filter((node) -> { return !node.isRetired(); } ).collect(Collectors.toList()); - } - private String createDistributionPartitions(int redundancyPerGroup, int numGroups) { StringBuilder sb = new StringBuilder(); for (int i = 0; i < numGroups - 1; ++i) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index 228d972b839..adfc703f747 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -275,7 +275,7 @@ public class StorageGroup { storageGroup.nodes.add(nodeBuilder.build(deployState, owner, storageGroup)); } - if ( ! parent.isPresent() && subGroups.isEmpty() && nodeBuilders.isEmpty()) // no nodes or groups: create single node + if (parent.isEmpty() && subGroups.isEmpty() && nodeBuilders.isEmpty()) // no nodes or groups: create single node storageGroup.nodes.add(buildSingleNode(deployState, owner)); return storageGroup; @@ -284,7 +284,7 @@ public class StorageGroup { private StorageNode buildSingleNode(DeployState deployState, ContentCluster parent) { int distributionKey = 0; StorageNode sNode = new StorageNode(parent.getStorageNodes(), 1.0, distributionKey , false); - sNode.setHostResource(parent.getHostSystem().getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC)); + sNode.setHostResource(parent.hostSystem().getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC)); PersistenceEngine provider = parent.getPersistence().create(deployState, sNode, storageGroup, null); new Distributor(parent.getDistributorNodes(), distributionKey, null, provider); return sNode; @@ -302,7 +302,7 @@ public class StorageGroup { throw new IllegalArgumentException("Specifying individual groups is not supported on hosted applications"); Map<HostResource, ClusterMembership> hostMapping = nodeRequirement.isPresent() ? - provisionHosts(nodeRequirement.get(), owner.getStorageNodes().getClusterName(), owner.getRoot().getHostSystem(), deployLogger) : + provisionHosts(nodeRequirement.get(), owner.getStorageNodes().getClusterName(), owner.getRoot().hostSystem(), deployLogger) : Collections.emptyMap(); Map<Optional<ClusterSpec.Group>, Map<HostResource, ClusterMembership>> hostGroups = collectAllocatedSubgroups(hostMapping); @@ -412,21 +412,20 @@ public class StorageGroup { } private Optional<String> childAsString(Optional<ModelElement> element, String childTagName) { - if ( ! element.isPresent()) return Optional.empty(); + if (element.isEmpty()) return Optional.empty(); return Optional.ofNullable(element.get().childAsString(childTagName)); } private Optional<Long> childAsLong(Optional<ModelElement> element, String childTagName) { - if ( ! element.isPresent()) return Optional.empty(); + if (element.isEmpty()) return Optional.empty(); return Optional.ofNullable(element.get().childAsLong(childTagName)); } private Optional<Boolean> childAsBoolean(Optional<ModelElement> element, String childTagName) { - if ( ! element.isPresent()) return Optional.empty(); + if (element.isEmpty()) return Optional.empty(); return Optional.ofNullable(element.get().childAsBoolean(childTagName)); } private boolean booleanAttributeOr(Optional<ModelElement> element, String attributeName, boolean defaultValue) { - if ( ! element.isPresent()) return defaultValue; - return element.get().booleanAttribute(attributeName, defaultValue); + return element.map(modelElement -> modelElement.booleanAttribute(attributeName, defaultValue)).orElse(defaultValue); } private Optional<ModelElement> getNodes(ModelElement groupOrNodesElement) { @@ -435,7 +434,7 @@ public class StorageGroup { } private List<XmlNodeBuilder> collectExplicitNodes(Optional<ModelElement> groupOrNodesElement) { - if ( ! groupOrNodesElement.isPresent()) return Collections.emptyList(); + if (groupOrNodesElement.isEmpty()) return Collections.emptyList(); List<XmlNodeBuilder> nodes = new ArrayList<>(); for (ModelElement n : groupOrNodesElement.get().subElements("node")) nodes.add(new XmlNodeBuilder(clusterElement, n)); @@ -444,7 +443,7 @@ public class StorageGroup { private List<GroupBuilder> collectSubGroups(boolean isHosted, StorageGroup parentGroup, ModelElement parentGroupElement) { List<ModelElement> subGroupElements = parentGroupElement.subElements("group"); - if (subGroupElements.size() > 1 && ! parentGroup.getPartitions().isPresent()) + if (subGroupElements.size() > 1 && parentGroup.getPartitions().isEmpty()) throw new IllegalArgumentException("'distribution' attribute is required with multiple subgroups"); List<GroupBuilder> subGroups = new ArrayList<>(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/TuningDispatch.java b/config-model/src/main/java/com/yahoo/vespa/model/content/TuningDispatch.java index ec958589483..9e5a3428726 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/TuningDispatch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/TuningDispatch.java @@ -9,14 +9,12 @@ public class TuningDispatch { private final Integer maxHitsPerPartition; public enum DispatchPolicy { ROUNDROBIN, ADAPTIVE}; private final DispatchPolicy dispatchPolicy; - private final Boolean useLocalNode; private final Double minGroupCoverage; private final Double minActiveDocsCoverage; private TuningDispatch(Builder builder) { maxHitsPerPartition = builder.maxHitsPerPartition; dispatchPolicy = builder.dispatchPolicy; - useLocalNode = builder.useLocalNode; minGroupCoverage = builder.minGroupCoverage; minActiveDocsCoverage = builder.minActiveDocsCoverage; } @@ -26,7 +24,6 @@ public class TuningDispatch { } public DispatchPolicy getDispatchPolicy() { return dispatchPolicy; } - public Boolean getUseLocalNode() { return useLocalNode; } public Double getMinGroupCoverage() { return minGroupCoverage; } public Double getMinActiveDocsCoverage() { return minActiveDocsCoverage; } @@ -34,7 +31,6 @@ public class TuningDispatch { private Integer maxHitsPerPartition; private DispatchPolicy dispatchPolicy; - private Boolean useLocalNode; private Double minGroupCoverage; private Double minActiveDocsCoverage; @@ -47,28 +43,28 @@ public class TuningDispatch { return this; } public Builder setDispatchPolicy(String policy) { - if (policy == null) { - } else if ("random".equals(policy.toLowerCase())) { - dispatchPolicy = DispatchPolicy.ADAPTIVE; - } else if ("round-robin".equals(policy.toLowerCase())) { - dispatchPolicy = DispatchPolicy.ROUNDROBIN; - } else { - dispatchPolicy = DispatchPolicy.valueOf(policy.toUpperCase()); - } + if (policy != null) + dispatchPolicy = toDispatchPolicy(policy); return this; } - public Builder setUseLocalNode(Boolean useLocalNode) { - this.useLocalNode = useLocalNode; - return this; + private DispatchPolicy toDispatchPolicy(String policy) { + switch (policy.toLowerCase()) { + case "adaptive": case "random": return DispatchPolicy.ADAPTIVE; // TODO: Deprecate 'random' on Java 8 + case "round-robin": return DispatchPolicy.ROUNDROBIN; + default: throw new IllegalArgumentException("Unknown dispatch policy '" + policy + "'"); + } } + public Builder setMinGroupCoverage(Double minGroupCoverage) { this.minGroupCoverage = minGroupCoverage; return this; } + public Builder setMinActiveDocsCoverage(Double minCoverage) { this.minActiveDocsCoverage = minCoverage; return this; } } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 4925a13c4d2..04e997110a6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.content.cluster; import com.google.common.base.Preconditions; import com.google.common.collect.Sets; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; @@ -115,7 +116,6 @@ public class ContentCluster extends AbstractConfigProducer implements } public ContentCluster build(Collection<ContainerModel> containers, ConfigModelContext context, Element w3cContentElement) { - ModelElement contentElement = new ModelElement(w3cContentElement); DeployState deployState = context.getDeployState(); ModelElement documentsElement = contentElement.child("documents"); @@ -139,31 +139,27 @@ public class ContentCluster extends AbstractConfigProducer implements c.search.handleRedundancy(c.redundancy); IndexedSearchCluster index = c.search.getIndexed(); - if (index != null) { - setupIndexedCluster(index, contentElement); - } + if (index != null) + setupIndexedCluster(index, contentElement, deployState.getDeployLogger()); - if (c.search.hasIndexedCluster() && !(c.persistenceFactory instanceof ProtonEngine.Factory) ) { - throw new RuntimeException("If you have indexed search you need to have proton as engine"); - } + if (c.search.hasIndexedCluster() && !(c.persistenceFactory instanceof ProtonEngine.Factory) ) + throw new RuntimeException("Indexed search requires proton as engine"); if (documentsElement != null) { ModelElement e = documentsElement.child("document-processing"); - if (e != null) { + if (e != null) setupDocumentProcessing(c, e); - } } else if (c.persistenceFactory != null) { throw new IllegalArgumentException("The specified content engine requires the <documents> element to be specified."); } ModelElement tuning = contentElement.child("tuning"); - if (tuning != null) { + if (tuning != null) setupTuning(c, tuning); - } + ModelElement experimental = contentElement.child("experimental"); - if (experimental != null) { + if (experimental != null) setupExperimental(c, experimental); - } if (context.getParentProducer().getRoot() == null) return c; @@ -171,7 +167,7 @@ public class ContentCluster extends AbstractConfigProducer implements return c; } - private void setupIndexedCluster(IndexedSearchCluster index, ModelElement element) { + private void setupIndexedCluster(IndexedSearchCluster index, ModelElement element, DeployLogger logger) { ContentSearch search = DomContentSearchBuilder.build(element); Double queryTimeout = search.getQueryTimeout(); if (queryTimeout != null) { @@ -188,22 +184,15 @@ public class ContentCluster extends AbstractConfigProducer implements // TODO: This should be cleaned up to avoid having to change code in 100 places // every time we add a dispatch option. - TuningDispatch tuningDispatch = DomTuningDispatchBuilder.build(element); + TuningDispatch tuningDispatch = DomTuningDispatchBuilder.build(element, logger); Integer maxHitsPerPartition = tuningDispatch.getMaxHitsPerPartition(); - Boolean useLocalNode = tuningDispatch.getUseLocalNode(); - if (index.getTuning() == null) { + if (index.getTuning() == null) index.setTuning(new Tuning(index)); - } - if (index.getTuning().dispatch == null) { + if (index.getTuning().dispatch == null) index.getTuning().dispatch = new Tuning.Dispatch(); - } - if (maxHitsPerPartition != null) { + if (maxHitsPerPartition != null) index.getTuning().dispatch.maxHitsPerPartition = maxHitsPerPartition; - } - if (useLocalNode != null) { - index.getTuning().dispatch.useLocalNode = useLocalNode; - } index.getTuning().dispatch.minGroupCoverage = tuningDispatch.getMinGroupCoverage(); index.getTuning().dispatch.minActiveDocsCoverage = tuningDispatch.getMinActiveDocsCoverage(); index.getTuning().dispatch.policy = tuningDispatch.getDispatchPolicy(); @@ -345,7 +334,7 @@ public class ContentCluster extends AbstractConfigProducer implements } private Collection<HostResource> getControllerHosts(NodesSpecification nodesSpecification, Admin admin, String clusterName, ConfigModelContext context) { - return nodesSpecification.provision(admin.getHostSystem(), ClusterSpec.Type.admin, ClusterSpec.Id.from(clusterName), context.getDeployLogger()).keySet(); + return nodesSpecification.provision(admin.hostSystem(), ClusterSpec.Type.admin, ClusterSpec.Id.from(clusterName), context.getDeployLogger()).keySet(); } private List<HostResource> drawControllerHosts(int count, StorageGroup rootGroup, Collection<ContainerModel> containers) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java index 5f26b0628c2..536793e848f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java @@ -1,15 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content.cluster; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.vespa.model.content.TuningDispatch; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import java.util.logging.Level; + /** * @author Simon Thoresen Hult */ public class DomTuningDispatchBuilder { - public static TuningDispatch build(ModelElement contentXml) { + public static TuningDispatch build(ModelElement contentXml, DeployLogger logger) { TuningDispatch.Builder builder = new TuningDispatch.Builder(); ModelElement tuningElement = contentXml.child("tuning"); if (tuningElement == null) { @@ -21,10 +24,12 @@ public class DomTuningDispatchBuilder { } builder.setMaxHitsPerPartition(dispatchElement.childAsInteger("max-hits-per-partition")); builder.setDispatchPolicy(dispatchElement.childAsString("dispatch-policy")); - builder.setUseLocalNode(dispatchElement.childAsBoolean("use-local-node")); builder.setMinGroupCoverage(dispatchElement.childAsDouble("min-group-coverage")); builder.setMinActiveDocsCoverage(dispatchElement.childAsDouble("min-active-docs-coverage")); + if (dispatchElement.child("use-local-node") != null) + logger.log(Level.WARNING, "Attribute 'use-local-node' is deprecated and ignored: " + + "The local node will automatically be preferred when appropriate."); return builder.build(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java index 44cb9cb96dd..2e316bfd2c1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java @@ -9,11 +9,7 @@ import java.util.Map; import java.util.TreeMap; /** -* Created with IntelliJ IDEA. -* User: thomasg -* Date: 9/28/12 -* Time: 1:20 PM -* To change this template use File | Settings | File Templates. +* @author Thomas Gundersen */ public class SearchDefinitionBuilder { public Map<String, NewDocumentType> build(DocumentTypeRepo repo, ModelElement elem) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/service/ServiceCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/ServiceCluster.java index 76f3415a666..7d304968bc7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/generic/service/ServiceCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/ServiceCluster.java @@ -42,9 +42,9 @@ public class ServiceCluster extends AbstractConfigProducer { } @Override - public HostSystem getHostSystem() { + public HostSystem hostSystem() { if (hostSystem!=null) return hostSystem; - return super.getHostSystem(); + return super.hostSystem(); } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 64ac00020f7..53b4d085351 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -325,7 +325,6 @@ public class IndexedSearchCluster extends SearchCluster builder.maxHitsPerNode(tuning.dispatch.maxHitsPerPartition); builder.maxNodesDownPerGroup(rootDispatch.getMaxNodesDownPerFixedRow()); - builder.useLocalNode(tuning.dispatch.useLocalNode); builder.searchableCopies(rootDispatch.getSearchableCopies()); if (searchCoverage != null) { if (searchCoverage.getMinimum() != null) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java index fac641bd714..95498a2a902 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java @@ -19,7 +19,6 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ public Integer maxHitsPerPartition = null; public TuningDispatch.DispatchPolicy policy = null; - public boolean useLocalNode = false; public Double minGroupCoverage = null; public Double minActiveDocsCoverage = null; } diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc index 15ebd03d9e2..8b3868c132e 100644 --- a/config-model/src/main/resources/schema/content.rnc +++ b/config-model/src/main/resources/schema/content.rnc @@ -79,7 +79,7 @@ ClusterControllerTuning = element cluster-controller { DispatchTuning = element dispatch { element max-hits-per-partition { xsd:nonNegativeInteger }? & - element dispatch-policy { string "round-robin" | string "random" }? & + element dispatch-policy { string "round-robin" | string "adaptive" | string "random" }? & element min-group-coverage { xsd:double }? & element min-active-docs-coverage { xsd:double }? & element use-local-node { string "true" | string "false" }? diff --git a/config-model/src/test/derived/array_of_struct_attribute/index-info.cfg b/config-model/src/test/derived/array_of_struct_attribute/index-info.cfg index 99942d83f91..ebe87df9852 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/index-info.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/index-info.cfg @@ -1,21 +1,23 @@ -indexinfo[0].name "test" -indexinfo[0].command[0].indexname "sddocname" -indexinfo[0].command[0].command "index" -indexinfo[0].command[1].indexname "sddocname" -indexinfo[0].command[1].command "word" -indexinfo[0].command[2].indexname "elem_array.name" -indexinfo[0].command[2].command "index" -indexinfo[0].command[3].indexname "elem_array.name" -indexinfo[0].command[3].command "attribute" -indexinfo[0].command[4].indexname "elem_array.name" -indexinfo[0].command[4].command "fast-search" -indexinfo[0].command[5].indexname "elem_array.weight" -indexinfo[0].command[5].command "index" -indexinfo[0].command[6].indexname "elem_array.weight" -indexinfo[0].command[6].command "attribute" -indexinfo[0].command[7].indexname "elem_array.weight" -indexinfo[0].command[7].command "numerical" -indexinfo[0].command[8].indexname "elem_array" -indexinfo[0].command[8].command "index" -indexinfo[0].command[9].indexname "elem_array" -indexinfo[0].command[9].command "multivalue" +indexinfo[].name "test" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "elem_array.name" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "elem_array.name" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "elem_array.name" +indexinfo[].command[].command "fast-search" +indexinfo[].command[].indexname "elem_array.name" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "elem_array.weight" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "elem_array.weight" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "elem_array.weight" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "elem_array" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "elem_array" +indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/imported_struct_fields/index-info.cfg b/config-model/src/test/derived/imported_struct_fields/index-info.cfg index 3723b06fcbe..4aa54dac933 100644 --- a/config-model/src/test/derived/imported_struct_fields/index-info.cfg +++ b/config-model/src/test/derived/imported_struct_fields/index-info.cfg @@ -17,6 +17,8 @@ indexinfo[].command[].indexname "my_elem_array.name" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "my_elem_array.name" indexinfo[].command[].command "fast-search" +indexinfo[].command[].indexname "my_elem_array.name" +indexinfo[].command[].command "word" indexinfo[].command[].indexname "my_elem_array.weight" indexinfo[].command[].command "index" indexinfo[].command[].indexname "my_elem_array.weight" @@ -33,6 +35,8 @@ indexinfo[].command[].indexname "my_elem_map.value.name" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "my_elem_map.value.name" indexinfo[].command[].command "fast-search" +indexinfo[].command[].indexname "my_elem_map.value.name" +indexinfo[].command[].command "word" indexinfo[].command[].indexname "my_elem_map.value.weight" indexinfo[].command[].command "index" indexinfo[].command[].indexname "my_elem_map.value.weight" @@ -47,6 +51,8 @@ indexinfo[].command[].indexname "my_elem_map.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "my_elem_map.key" indexinfo[].command[].command "fast-search" +indexinfo[].command[].indexname "my_elem_map.key" +indexinfo[].command[].command "word" indexinfo[].command[].indexname "my_elem_map" indexinfo[].command[].command "index" indexinfo[].command[].indexname "my_elem_map" @@ -57,6 +63,8 @@ indexinfo[].command[].indexname "my_str_int_map.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "my_str_int_map.key" indexinfo[].command[].command "fast-search" +indexinfo[].command[].indexname "my_str_int_map.key" +indexinfo[].command[].command "word" indexinfo[].command[].indexname "my_str_int_map.value" indexinfo[].command[].command "index" indexinfo[].command[].indexname "my_str_int_map.value" diff --git a/config-model/src/test/derived/map_attribute/index-info.cfg b/config-model/src/test/derived/map_attribute/index-info.cfg index 1d44648ef6f..67c6bbf4d1b 100644 --- a/config-model/src/test/derived/map_attribute/index-info.cfg +++ b/config-model/src/test/derived/map_attribute/index-info.cfg @@ -1,33 +1,37 @@ -indexinfo[0].name "test" -indexinfo[0].command[0].indexname "sddocname" -indexinfo[0].command[0].command "index" -indexinfo[0].command[1].indexname "sddocname" -indexinfo[0].command[1].command "word" -indexinfo[0].command[2].indexname "str_map.key" -indexinfo[0].command[2].command "index" -indexinfo[0].command[3].indexname "str_map.key" -indexinfo[0].command[3].command "attribute" -indexinfo[0].command[4].indexname "str_map.key" -indexinfo[0].command[4].command "fast-search" -indexinfo[0].command[5].indexname "str_map.value" -indexinfo[0].command[5].command "index" -indexinfo[0].command[6].indexname "str_map.value" -indexinfo[0].command[6].command "attribute" -indexinfo[0].command[7].indexname "str_map" -indexinfo[0].command[7].command "index" -indexinfo[0].command[8].indexname "str_map" -indexinfo[0].command[8].command "multivalue" -indexinfo[0].command[9].indexname "int_map.key" -indexinfo[0].command[9].command "index" -indexinfo[0].command[10].indexname "int_map.key" -indexinfo[0].command[10].command "attribute" -indexinfo[0].command[11].indexname "int_map.key" -indexinfo[0].command[11].command "numerical" -indexinfo[0].command[12].indexname "int_map.value" -indexinfo[0].command[12].command "index" -indexinfo[0].command[13].indexname "int_map.value" -indexinfo[0].command[13].command "numerical" -indexinfo[0].command[14].indexname "int_map" -indexinfo[0].command[14].command "index" -indexinfo[0].command[15].indexname "int_map" -indexinfo[0].command[15].command "multivalue" +indexinfo[].name "test" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "str_map.key" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "str_map.key" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "str_map.key" +indexinfo[].command[].command "fast-search" +indexinfo[].command[].indexname "str_map.key" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "str_map.value" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "str_map.value" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "str_map.value" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "str_map" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "str_map" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "int_map.key" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "int_map.key" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "int_map.key" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "int_map.value" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "int_map.value" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "int_map" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "int_map" +indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/map_of_struct_attribute/index-info.cfg b/config-model/src/test/derived/map_of_struct_attribute/index-info.cfg index 659bd86b9f0..84296b00a5d 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/index-info.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/index-info.cfg @@ -1,49 +1,55 @@ -indexinfo[0].name "test" -indexinfo[0].command[0].indexname "sddocname" -indexinfo[0].command[0].command "index" -indexinfo[0].command[1].indexname "sddocname" -indexinfo[0].command[1].command "word" -indexinfo[0].command[2].indexname "str_elem_map.key" -indexinfo[0].command[2].command "index" -indexinfo[0].command[3].indexname "str_elem_map.key" -indexinfo[0].command[3].command "attribute" -indexinfo[0].command[4].indexname "str_elem_map.key" -indexinfo[0].command[4].command "fast-search" -indexinfo[0].command[5].indexname "str_elem_map.value.name" -indexinfo[0].command[5].command "index" -indexinfo[0].command[6].indexname "str_elem_map.value.name" -indexinfo[0].command[6].command "attribute" -indexinfo[0].command[7].indexname "str_elem_map.value.weight" -indexinfo[0].command[7].command "index" -indexinfo[0].command[8].indexname "str_elem_map.value.weight" -indexinfo[0].command[8].command "attribute" -indexinfo[0].command[9].indexname "str_elem_map.value.weight" -indexinfo[0].command[9].command "numerical" -indexinfo[0].command[10].indexname "str_elem_map.value" -indexinfo[0].command[10].command "index" -indexinfo[0].command[11].indexname "str_elem_map" -indexinfo[0].command[11].command "index" -indexinfo[0].command[12].indexname "str_elem_map" -indexinfo[0].command[12].command "multivalue" -indexinfo[0].command[13].indexname "int_elem_map.key" -indexinfo[0].command[13].command "index" -indexinfo[0].command[14].indexname "int_elem_map.key" -indexinfo[0].command[14].command "attribute" -indexinfo[0].command[15].indexname "int_elem_map.key" -indexinfo[0].command[15].command "numerical" -indexinfo[0].command[16].indexname "int_elem_map.value.name" -indexinfo[0].command[16].command "index" -indexinfo[0].command[17].indexname "int_elem_map.value.name" -indexinfo[0].command[17].command "attribute" -indexinfo[0].command[18].indexname "int_elem_map.value.name" -indexinfo[0].command[18].command "fast-search" -indexinfo[0].command[19].indexname "int_elem_map.value.weight" -indexinfo[0].command[19].command "index" -indexinfo[0].command[20].indexname "int_elem_map.value.weight" -indexinfo[0].command[20].command "numerical" -indexinfo[0].command[21].indexname "int_elem_map.value" -indexinfo[0].command[21].command "index" -indexinfo[0].command[22].indexname "int_elem_map" -indexinfo[0].command[22].command "index" -indexinfo[0].command[23].indexname "int_elem_map" -indexinfo[0].command[23].command "multivalue" +indexinfo[].name "test" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "str_elem_map.key" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "str_elem_map.key" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "str_elem_map.key" +indexinfo[].command[].command "fast-search" +indexinfo[].command[].indexname "str_elem_map.key" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "str_elem_map.value.name" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "str_elem_map.value.name" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "str_elem_map.value.name" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "str_elem_map.value.weight" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "str_elem_map.value.weight" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "str_elem_map.value.weight" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "str_elem_map.value" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "str_elem_map" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "str_elem_map" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "int_elem_map.key" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "int_elem_map.key" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "int_elem_map.key" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "int_elem_map.value.name" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "int_elem_map.value.name" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "int_elem_map.value.name" +indexinfo[].command[].command "fast-search" +indexinfo[].command[].indexname "int_elem_map.value.name" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "int_elem_map.value.weight" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "int_elem_map.value.weight" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "int_elem_map.value" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "int_elem_map" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "int_elem_map" +indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/streamingstruct/vsmfields.cfg b/config-model/src/test/derived/streamingstruct/vsmfields.cfg index a99a704be3a..7178f9d41ea 100644 --- a/config-model/src/test/derived/streamingstruct/vsmfields.cfg +++ b/config-model/src/test/derived/streamingstruct/vsmfields.cfg @@ -62,7 +62,7 @@ fieldspec[].maxlength 1048576 fieldspec[].fieldtype INDEX fieldspec[].name "c.f1" fieldspec[].searchmethod AUTOUTF8 -fieldspec[].arg1 "" +fieldspec[].arg1 "word" fieldspec[].maxlength 1048576 fieldspec[].fieldtype ATTRIBUTE fieldspec[].name "c.f1s" @@ -77,7 +77,7 @@ fieldspec[].maxlength 1048576 fieldspec[].fieldtype INDEX fieldspec[].name "c2.f1" fieldspec[].searchmethod AUTOUTF8 -fieldspec[].arg1 "" +fieldspec[].arg1 "word" fieldspec[].maxlength 1048576 fieldspec[].fieldtype ATTRIBUTE fieldspec[].name "c2.f1s" @@ -97,7 +97,7 @@ fieldspec[].maxlength 1048576 fieldspec[].fieldtype INDEX fieldspec[].name "c3.f1" fieldspec[].searchmethod AUTOUTF8 -fieldspec[].arg1 "" +fieldspec[].arg1 "word" fieldspec[].maxlength 1048576 fieldspec[].fieldtype ATTRIBUTE fieldspec[].name "c3.f1s" @@ -197,7 +197,7 @@ fieldspec[].maxlength 1048576 fieldspec[].fieldtype INDEX fieldspec[].name "array3.f1" fieldspec[].searchmethod AUTOUTF8 -fieldspec[].arg1 "" +fieldspec[].arg1 "word" fieldspec[].maxlength 1048576 fieldspec[].fieldtype ATTRIBUTE fieldspec[].name "array3.f1s" diff --git a/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java b/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java index 074ff610e9a..0b10b2b13ba 100644 --- a/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java @@ -105,7 +105,7 @@ public class SystemModelTestCase { @Test public void testHostSystem() { VespaModel vespaModel = getVespaModelDoNotValidateXml(TESTDIR + "simpleconfig/"); - HostSystem hostSystem = vespaModel.getHostSystem(); + HostSystem hostSystem = vespaModel.hostSystem(); HostResource host1 = hostSystem.getHost("host1"); HostResource host2 = hostSystem.getHost("host2"); diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 6011b138a61..d9b4693921c 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -139,7 +139,7 @@ public class ModelProvisioningTest { QrStartConfig qrsStartConfig = new QrStartConfig(qrStartBuilder); assertEquals(45, qrsStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory()); - HostSystem hostSystem = model.getHostSystem(); + HostSystem hostSystem = model.hostSystem(); assertNotNull(hostSystem.getHostByHostname("myhost0")); assertNotNull(hostSystem.getHostByHostname("myhost1")); assertNotNull(hostSystem.getHostByHostname("myhost2")); @@ -169,7 +169,7 @@ public class ModelProvisioningTest { tester.addHosts(numberOfHosts); int numberOfContentNodes = 2; VespaModel model = tester.createModel(xmlWithNodes, true); - assertEquals(numberOfHosts, model.getRoot().getHostSystem().getHosts().size()); + assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); Map<String, ContentCluster> contentClusters = model.getContentClusters(); ContentCluster cluster = contentClusters.get("bar"); assertEquals(numberOfContentNodes, cluster.getRootGroup().getNodes().size()); @@ -217,8 +217,8 @@ public class ModelProvisioningTest { tester.addHosts(1); VespaModel model = tester.createModel(xmlWithNodes, true); - assertEquals(1, model.getHostSystem().getHosts().size()); - HostResource host = model.getHostSystem().getHosts().iterator().next(); + assertEquals(1, model.hostSystem().getHosts().size()); + HostResource host = model.hostSystem().getHosts().iterator().next(); assertTrue(host.spec().membership().isPresent()); assertEquals("container", host.spec().membership().get().cluster().type().name()); @@ -250,7 +250,7 @@ public class ModelProvisioningTest { assertEquals("Nodes in container1", 2, model.getContainerClusters().get("container1").getContainers().size()); assertEquals("Heap size is lowered with combined clusters", 17, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); - assertEquals(2, model.getHostSystem().getHosts().stream() + assertEquals(2, model.hostSystem().getHosts().stream() .filter(hostResource -> hostResource.spec().membership().get().cluster().type() == ClusterSpec.Type.combined) .count()); @@ -391,7 +391,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, true); - assertEquals(numberOfHosts, model.getRoot().getHostSystem().getHosts().size()); + assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); // Check container cluster assertEquals(1, model.getContainerClusters().size()); @@ -497,7 +497,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, true); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); // Check content cluster ContentCluster cluster = model.getContentClusters().get("bar"); @@ -553,7 +553,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, true); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); // Check content clusters ContentCluster cluster = model.getContentClusters().get("bar"); @@ -594,7 +594,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, true); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); // Check content clusters ContentCluster cluster = model.getContentClusters().get("bar"); @@ -632,7 +632,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, true); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); ContentCluster cluster = model.getContentClusters().get("bar"); ContainerCluster clusterControllers = cluster.getClusterControllers(); @@ -661,7 +661,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, true, "node-1-3-9-09", "node-1-3-9-06", "node-1-3-9-03"); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); // Check content clusters ContentCluster cluster = model.getContentClusters().get("bar"); @@ -691,7 +691,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, true, "node-1-3-9-09"); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); // Check slobroks clusters assertEquals("Includes retired node", 1+3, model.getAdmin().getSlobroks().size()); @@ -716,7 +716,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, true, "node-1-3-9-01", "node-1-3-9-02"); - assertEquals(numberOfHosts, model.getRoot().getHostSystem().getHosts().size()); + assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); // Check slobroks clusters assertEquals("Includes retired node", 3+2, model.getAdmin().getSlobroks().size()); @@ -745,7 +745,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, true, "node-1-3-9-12", "node-1-3-9-03", "node-1-3-9-02"); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); // Check slobroks clusters // ... from cluster default @@ -777,7 +777,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, true); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); ContentCluster cluster = model.getContentClusters().get("bar"); ContainerCluster clusterControllers = cluster.getClusterControllers(); @@ -886,7 +886,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, true); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); // Check content clusters ContentCluster cluster = model.getContentClusters().get("bar"); @@ -952,7 +952,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, false); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); ContentCluster cluster = model.getContentClusters().get("bar"); assertEquals(2*3, cluster.redundancy().effectiveInitialRedundancy()); // Reduced from 3*3 @@ -1008,7 +1008,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, false); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); ContentCluster cluster = model.getContentClusters().get("bar"); assertEquals(4, cluster.redundancy().effectiveInitialRedundancy()); @@ -1052,7 +1052,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, false); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); ContentCluster cluster = model.getContentClusters().get("bar"); ClusterControllerContainerCluster clusterControllers = cluster.getClusterControllers(); @@ -1113,7 +1113,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, false); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); ContentCluster cluster = model.getContentClusters().get("bar"); assertEquals(1, cluster.redundancy().effectiveInitialRedundancy()); @@ -1189,7 +1189,7 @@ public class ModelProvisioningTest { tester.addHosts(new NodeResources(0.7, 2, 2.5, 0.3), 3); // Controller-bar tester.addHosts(new NodeResources(10, 64, 200, 0.3), 6); // Content-bar VespaModel model = tester.createModel(services, true, 0); - assertEquals(totalHosts, model.getRoot().getHostSystem().getHosts().size()); + assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size()); } @Test @@ -1204,7 +1204,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, true); - assertEquals(numberOfHosts, model.getRoot().getHostSystem().getHosts().size()); + assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); assertEquals(3, model.getContainerClusters().get("container").getContainers().size()); assertNotNull(model.getAdmin().getLogserver()); assertEquals(3, model.getAdmin().getSlobroks().size()); @@ -1222,7 +1222,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, true); - assertEquals(numberOfHosts, model.getRoot().getHostSystem().getHosts().size()); + assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); assertEquals("xyz", model.getContainerClusters().get("container").getContainers().get(0).getAssignedJvmOptions()); } @@ -1238,7 +1238,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, true); - assertEquals(numberOfHosts, model.getRoot().getHostSystem().getHosts().size()); + assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); assertEquals("xyz", model.getContainerClusters().get("container").getContainers().get(0).getAssignedJvmOptions()); } @@ -1282,7 +1282,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(1); VespaModel model = tester.createModel(services, true); - assertEquals(1, model.getRoot().getHostSystem().getHosts().size()); + assertEquals(1, model.getRoot().hostSystem().getHosts().size()); assertEquals(1, model.getAdmin().getSlobroks().size()); } @Test @@ -1356,7 +1356,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(1); VespaModel model = tester.createModel(services, true); - assertEquals(1, model.getRoot().getHostSystem().getHosts().size()); + assertEquals(1, model.getRoot().hostSystem().getHosts().size()); assertEquals(1, model.getAdmin().getSlobroks().size()); assertEquals(1, model.getContainerClusters().get("foo").getContainers().size()); assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes()); @@ -1375,7 +1375,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(1); VespaModel model = tester.createModel(services, true); - assertEquals(1, model.getRoot().getHostSystem().getHosts().size()); + assertEquals(1, model.getRoot().hostSystem().getHosts().size()); assertEquals(1, model.getAdmin().getSlobroks().size()); assertEquals(1, model.getContainerClusters().get("foo").getContainers().size()); } @@ -1399,7 +1399,7 @@ public class ModelProvisioningTest { tester.setHosted(false); tester.addHosts(1); VespaModel model = tester.createModel(services, true); - assertEquals(1, model.getRoot().getHostSystem().getHosts().size()); + assertEquals(1, model.getRoot().hostSystem().getHosts().size()); assertEquals(1, model.getAdmin().getSlobroks().size()); assertEquals(1, model.getContainerClusters().get("foo").getContainers().size()); assertEquals(1, model.getContentClusters().get("bar").getRootGroup().recursiveGetNodes().size()); @@ -1426,7 +1426,7 @@ public class ModelProvisioningTest { tester.setHosted(false); tester.addHosts(1); VespaModel model = tester.createModel(services, true); - assertEquals(1, model.getRoot().getHostSystem().getHosts().size()); + assertEquals(1, model.getRoot().hostSystem().getHosts().size()); assertEquals(1, model.getAdmin().getSlobroks().size()); assertEquals(1, model.getContainerClusters().get("foo").getContainers().size()); assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes()); @@ -1468,7 +1468,7 @@ public class ModelProvisioningTest { " </services>"; VespaModel model = createNonProvisionedMultitenantModel(services); - assertEquals(1, model.getRoot().getHostSystem().getHosts().size()); + assertEquals(1, model.getRoot().hostSystem().getHosts().size()); ContentCluster content = model.getContentClusters().get("storage"); assertEquals(2, content.getRootGroup().getNodes().size()); ContainerCluster controller = content.getClusterControllers(); @@ -1514,7 +1514,7 @@ public class ModelProvisioningTest { "</services>"; VespaModel model = createNonProvisionedMultitenantModel(services); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(1)); + assertThat(model.getRoot().hostSystem().getHosts().size(), is(1)); ContentCluster content = model.getContentClusters().get("storage"); assertEquals(1, content.getRootGroup().getNodes().size()); ContainerCluster controller = content.getClusterControllers(); @@ -1577,7 +1577,7 @@ public class ModelProvisioningTest { "</services>"; VespaModel model = createNonProvisionedModel(false, hosts, services); - assertEquals(3, model.getRoot().getHostSystem().getHosts().size()); + assertEquals(3, model.getRoot().hostSystem().getHosts().size()); ContentCluster content = model.getContentClusters().get("storage"); assertEquals(3, content.getRootGroup().getNodes().size()); } @@ -1626,7 +1626,7 @@ public class ModelProvisioningTest { " </services>"; VespaModel model = createNonProvisionedMultitenantModel(services); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(1)); + assertThat(model.getRoot().hostSystem().getHosts().size(), is(1)); ContentCluster content = model.getContentClusters().get("storage"); assertEquals(2, content.getRootGroup().getNodes().size()); ContainerCluster controller = content.getClusterControllers(); @@ -1747,7 +1747,7 @@ public class ModelProvisioningTest { tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(Zone.defaultZone(), services, true); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); Admin admin = model.getAdmin(); Logserver logserver = admin.getLogserver(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java index e0d2e165772..cd67e432b8f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java @@ -274,7 +274,7 @@ public class ClusterControllerTestCase extends DomBuilderTest { private boolean existsHostsWithClusterControllerConfigId(VespaModel model) { boolean found = false; - for (HostResource h : model.getHostSystem().getHosts()) { + for (HostResource h : model.hostSystem().getHosts()) { for (Service s : h.getServices()) { if (s.getConfigId().equals("admin/cluster-controllers/0")) { found = true; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java index 854b708797e..89248d2469d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java @@ -15,12 +15,10 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.C import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.configId; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getNodeDimensionsConfig; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getRpcConnectorConfig; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getVespaServicesConfig; -import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.notNullValue; import static org.junit.Assert.assertEquals; @@ -39,9 +37,9 @@ public class MetricsProxyContainerTest { tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(servicesWithManyNodes(), true); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); - for (var host : model.getHostSystem().getHosts()) { + for (var host : model.hostSystem().getHosts()) { assertThat(host.getService(METRICS_PROXY_CONTAINER.serviceName), notNullValue()); long metricsProxies = host.getServices().stream() diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index a7ab9e02a79..31a2a6496d7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -168,8 +168,8 @@ public class ContentBuilderTest extends DomBuilderTest { assertEquals("clu", c.getConfigId()); //assertEquals("content/a/0", a.getRootGroup().getNodes().get(0).getConfigId()); // This is how it should look like in an ideal world. assertEquals("clu/storage/0", c.getRootGroup().getNodes().get(0).getConfigId()); // Due to reuse. - assertEquals(1, c.getRoot().getHostSystem().getHosts().size()); - HostResource h = c.getRoot().getHostSystem().getHost("mockhost"); + assertEquals(1, c.getRoot().hostSystem().getHosts().size()); + HostResource h = c.getRoot().hostSystem().getHost("mockhost"); String [] expectedServices = {"configserver", "logserver", "logd", "container-clustercontroller", "metricsproxy-container", "slobrok", "configproxy","config-sentinel", "qrserver", "storagenode", "searchnode", "distributor", "transactionlogserver"}; assertServices(h, expectedServices); assertEquals("clu/storage/0", h.getService("storagenode").getConfigId()); @@ -216,8 +216,8 @@ public class ContentBuilderTest extends DomBuilderTest { assertEquals(musicClusterId, cluster.getConfigId()); //assertEquals("content/a/0", a.getRootGroup().getNodes().get(0).getConfigId()); assertEquals(musicClusterId + "/storage/0", cluster.getRootGroup().getNodes().get(0).getConfigId()); // Due to reuse. - assertEquals(1, cluster.getRoot().getHostSystem().getHosts().size()); - HostResource h = cluster.getRoot().getHostSystem().getHost("mockhost"); + assertEquals(1, cluster.getRoot().hostSystem().getHosts().size()); + HostResource h = cluster.getRoot().hostSystem().getHost("mockhost"); String [] expectedServices = { "logd", "configproxy", "config-sentinel", "configserver", "logserver", "slobrok", "storagenode", "distributor","searchnode","transactionlogserver", @@ -301,8 +301,8 @@ public class ContentBuilderTest extends DomBuilderTest { assertEquals("b", b.getConfigId()); //assertEquals("content/a/0", a.getRootGroup().getNodes().get(0).getConfigId()); assertEquals("b/storage/0", b.getRootGroup().getNodes().get(0).getConfigId()); // Due to reuse. - assertEquals(1, b.getRoot().getHostSystem().getHosts().size()); - HostResource h = b.getRoot().getHostSystem().getHost("mockhost"); + assertEquals(1, b.getRoot().hostSystem().getHosts().size()); + HostResource h = b.getRoot().hostSystem().getHost("mockhost"); assertEquals("b/storage/0", h.getService("storagenode").getConfigId()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java index 964f8ed4818..18063bff16b 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java @@ -123,7 +123,7 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { assertThat(admin.getConfigservers().size(), is(3)); assertThat(admin.getSlobroks().size(), is(1)); assertThat(admin.getClusterControllerHosts().size(), is(1)); - assertNotNull(admin.getHostSystem().getHostByHostname("test1")); + assertNotNull(admin.hostSystem().getHostByHostname("test1")); for (Configserver configserver : admin.getConfigservers()) { assertThat(configserver.getHostName(), is(not(admin.getClusterControllerHosts().get(0).getHost().getHostname()))); for (Slobrok slobrok : admin.getSlobroks()) { @@ -214,7 +214,7 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { deployState.getFileRegistry(), multitenant, configServerSpecs); Admin admin = domAdminBuilder.build(deployState, root, xml); - admin.addPerHostServices(root.getHostSystem().getHosts(), deployState); + admin.addPerHostServices(root.hostSystem().getHosts(), deployState); return admin; } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilderTest.java index b8d9ad59ae0..d3d5ba90488 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilderTest.java @@ -15,10 +15,8 @@ import org.w3c.dom.Element; import java.io.StringReader; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; /** * @author gjoranv @@ -94,12 +92,12 @@ public class VespaDomBuilderTest { @Test public void testHostSystem() { VespaModel model = createModel(hosts, services); - HostSystem hostSystem = model.getHostSystem(); - assertThat(hostSystem.getHosts().size(), is(1)); + HostSystem hostSystem = model.hostSystem(); + assertEquals(1, hostSystem.getHosts().size()); HostResource host = hostSystem.getHosts().get(0); - assertThat(host, is(hostSystem.getHostByHostname(host.getHostname()))); + assertEquals(hostSystem.getHostByHostname(host.getHostname()), host); assertNotNull(hostSystem.getHost("node1")); - assertThat(hostSystem.toString(), is("host '" + host.getHostname() + "'")); + assertEquals("hosts [" + host.getHostname() + "]", hostSystem.toString()); } private VespaModel createModel(String hosts, String services) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 37612f86d9e..5a1befd3eb0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -472,7 +472,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .withServices(servicesXml) .build(); VespaModel model = new VespaModel(applicationPackage); - assertThat(model.getHostSystem().getHosts().size(), is(1)); + assertThat(model.hostSystem().getHosts().size(), is(1)); } @Test @@ -575,7 +575,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .setMultitenant(true) .setHostedVespa(true)) .build()); - assertEquals(1, model.getHostSystem().getHosts().size()); + assertEquals(1, model.hostSystem().getHosts().size()); } @Test(expected = IllegalArgumentException.class) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java index 319c871f996..62221c206fd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java @@ -76,8 +76,7 @@ public class ClusterTest { "<max-hits-per-partition>77</max-hits-per-partition>", "<dispatch-policy>adaptive</dispatch-policy>", "<min-group-coverage>13</min-group-coverage>", - "<min-active-docs-coverage>93</min-active-docs-coverage>", - "<use-local-node>true</use-local-node>"), + "<min-active-docs-coverage>93</min-active-docs-coverage>"), false); DispatchConfig.Builder builder = new DispatchConfig.Builder(); cluster.getSearch().getConfig(builder); @@ -85,7 +84,6 @@ public class ClusterTest { assertEquals(2, config.searchableCopies()); assertEquals(93.0, config.minActivedocsPercentage(), DELTA); assertEquals(13.0, config.minGroupCoverage(), DELTA); - assertTrue(config.useLocalNode()); assertEquals(DispatchConfig.DistributionPolicy.ADAPTIVE, config.distributionPolicy()); assertEquals(77, config.maxHitsPerNode()); } @@ -107,7 +105,6 @@ public class ClusterTest { assertEquals(100.0, config.minSearchCoverage(), DELTA); assertEquals(97.0, config.minActivedocsPercentage(), DELTA); assertEquals(100.0, config.minGroupCoverage(), DELTA); - assertFalse(config.useLocalNode()); assertEquals(3, config.node().size()); assertEquals(0, config.node(0).key()); assertEquals(1, config.node(1).key()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilderTest.java index 0f909b7a8eb..0c7c16c15c7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilderTest.java @@ -1,8 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content.cluster; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.content.TuningDispatch; +import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import org.junit.Test; import javax.xml.parsers.DocumentBuilderFactory; @@ -94,6 +96,7 @@ public class DomTuningDispatchBuilderTest { new ModelElement(DocumentBuilderFactory.newInstance() .newDocumentBuilder() .parse(new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))) - .getDocumentElement())); + .getDocumentElement()), + new DeployLoggerStub()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java index 7ef89ded733..8c4552ba117 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java @@ -77,7 +77,7 @@ public class ModelAmendingTestCase { VespaModel model = tester.createModel(services); // Check that all hosts are amended - for (HostResource host : model.getAdmin().getHostSystem().getHosts()) { + for (HostResource host : model.getAdmin().hostSystem().getHosts()) { assertFalse(host + " is amended", host.getHost().getChildrenByTypeRecursive(AmendedService.class).isEmpty()); } @@ -108,7 +108,7 @@ public class ModelAmendingTestCase { } private void amend(DeployLogger deployLogger, AdminModel adminModel) { - for (HostResource host : adminModel.getAdmin().getHostSystem().getHosts()) { + for (HostResource host : adminModel.getAdmin().hostSystem().getHosts()) { if ( ! host.getHost().getChildrenByTypeRecursive(AmendedService.class).isEmpty()) continue; // already amended adminModel.getAdmin().addAndInitializeService(deployLogger, host, new AmendedService(host.getHost())); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java index ac81c1a7b7e..c985ebabbf9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java @@ -291,7 +291,7 @@ public class VespaModelTestCase { VespaModel model = new VespaModel(new MockApplicationPackage.Builder() .withServices("<services version='1.0'><container version='1.0'><search /></container></services>") .build()); - assertThat(model.getHostSystem().getHosts().size(), is(1)); + assertThat(model.hostSystem().getHosts().size(), is(1)); assertThat(model.getContainerClusters().size(), is(1)); } diff --git a/configserver/src/main/sh/start-configserver b/configserver/src/main/sh/start-configserver index 4849b993340..958d71d7f5e 100755 --- a/configserver/src/main/sh/start-configserver +++ b/configserver/src/main/sh/start-configserver @@ -163,8 +163,10 @@ fixddir $bundlecachedir vespa-run-as-vespa-user vespa-runserver -s configserver -r 30 -p $pidfile -- \ java \ -Xms128m -Xmx2048m \ - -XX:+PreserveFramePointer \ - -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${VESPA_HOME}/var/crash \ + -XX:+PreserveFramePointer \ + -XX:+HeapDumpOnOutOfMemoryError \ + -XX:HeapDumpPath="${VESPA_HOME}/var/crash" \ + -XX:ErrorFile="${VESPA_HOME}/var/crash/hs_err_pid%p.log" \ -XX:+ExitOnOutOfMemoryError \ $jvmargs \ --illegal-access=warn \ diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh index e64d6a9bac1..228f731fd5c 100755 --- a/container-disc/src/main/sh/vespa-start-container-daemon.sh +++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh @@ -204,6 +204,7 @@ exec $numactlcmd $envcmd java \ -XX:MaxJavaStackTraceDepth=1000000 \ -XX:+HeapDumpOnOutOfMemoryError \ -XX:HeapDumpPath="${VESPA_HOME}/var/crash" \ + -XX:ErrorFile="${VESPA_HOME}/var/crash/hs_err_pid%p.log" \ -XX:+ExitOnOutOfMemoryError \ --illegal-access=warn \ --add-opens=java.base/java.io=ALL-UNNAMED \ diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java index c04553ae2f5..14604d61c0a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -85,7 +85,7 @@ public class FastSearcher extends VespaBackEndSearcher { public Result doSearch2(Query query, Execution execution) { if (dispatcher.searchCluster().groupSize() == 1) forceSinglePassGrouping(query); - try(SearchInvoker invoker = getSearchInvoker(query)) { + try (SearchInvoker invoker = getSearchInvoker(query)) { Result result = invoker.search(query, execution); injectSource(result.hits()); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 224facd0c5b..03b51fbaf70 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -66,6 +66,8 @@ public class Dispatcher extends AbstractComponent { private final Metric metric; private final Metric.Context metricContext; + private final int maxHitsPerNode; + private static final QueryProfileType argumentType; static { @@ -120,6 +122,7 @@ public class Dispatcher extends AbstractComponent { this.invokerFactory = invokerFactory; this.metric = metric; this.metricContext = metric.createContext(null); + this.maxHitsPerNode = dispatchConfig.maxHitsPerNode(); searchCluster.startClusterMonitoring(pingFactory); } @@ -161,7 +164,11 @@ public class Dispatcher extends AbstractComponent { if (nodes.isEmpty()) return Optional.empty(); query.trace(false, 2, "Dispatching with search path ", searchPath); - return invokerFactory.createSearchInvoker(searcher, query, OptionalInt.empty(), nodes, true); + return invokerFactory.createSearchInvoker(searcher, query, + OptionalInt.empty(), + nodes, + true, + maxHitsPerNode); } catch (InvalidSearchPathException e) { return Optional.of(new SearchErrorInvoker(ErrorMessage.createIllegalQuery(e.getMessage()))); } @@ -172,7 +179,12 @@ public class Dispatcher extends AbstractComponent { if (directNode.isPresent()) { Node node = directNode.get(); query.trace(false, 2, "Dispatching to ", node); - return invokerFactory.createSearchInvoker(searcher, query, OptionalInt.empty(), Arrays.asList(node), true) + return invokerFactory.createSearchInvoker(searcher, + query, + OptionalInt.empty(), + Arrays.asList(node), + true, + maxHitsPerNode) .orElseThrow(() -> new IllegalStateException("Could not dispatch directly to " + node)); } @@ -190,7 +202,8 @@ public class Dispatcher extends AbstractComponent { query, OptionalInt.of(group.id()), group.nodes(), - acceptIncompleteCoverage); + acceptIncompleteCoverage, + maxHitsPerNode); if (invoker.isPresent()) { query.trace(false, 2, "Dispatching to group ", group.id()); query.getModel().setSearchPath("/" + group.id()); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java index 84e5e7d747f..cec3e94d551 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java @@ -36,6 +36,7 @@ import static com.yahoo.container.handler.Coverage.DEGRADED_BY_TIMEOUT; * @author ollivir */ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseMonitor<SearchInvoker> { + private static final Logger log = Logger.getLogger(InterleavedSearchInvoker.class.getName()); private final Set<SearchInvoker> invokers; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java index 6030e989595..1c3a90ac6ab 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java @@ -27,7 +27,10 @@ public abstract class InvokerFactory { this.searchCluster = searchCluster; } - protected abstract Optional<SearchInvoker> createNodeSearchInvoker(VespaBackEndSearcher searcher, Query query, Node node); + protected abstract Optional<SearchInvoker> createNodeSearchInvoker(VespaBackEndSearcher searcher, + Query query, + int maxHits, + Node node); public abstract FillInvoker createFillInvoker(VespaBackEndSearcher searcher, Result result); @@ -47,13 +50,14 @@ public abstract class InvokerFactory { Query query, OptionalInt groupId, List<Node> nodes, - boolean acceptIncompleteCoverage) { + boolean acceptIncompleteCoverage, + int maxHits) { List<SearchInvoker> invokers = new ArrayList<>(nodes.size()); Set<Integer> failed = null; for (Node node : nodes) { boolean nodeAdded = false; if (node.isWorking() != Boolean.FALSE) { - Optional<SearchInvoker> invoker = createNodeSearchInvoker(searcher, query, node); + Optional<SearchInvoker> invoker = createNodeSearchInvoker(searcher, query, maxHits, node); if (invoker.isPresent()) { invokers.add(invoker.get()); nodeAdded = true; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java index 52a45dc421c..a32931c43c8 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java @@ -18,6 +18,7 @@ import java.util.Optional; * @author ollivir */ public class SearchErrorInvoker extends SearchInvoker { + private final ErrorMessage message; private Query query; private final Coverage coverage; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java index 77b3df7c83a..45ed1b87746 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java @@ -18,6 +18,7 @@ import java.util.Optional; * @author ollivir */ public abstract class SearchInvoker extends CloseableInvoker { + private final Optional<Node> node; private ResponseMonitor<SearchInvoker> monitor; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java index 58d7035c5e8..ae2258c4546 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java @@ -38,12 +38,12 @@ public class ProtobufSerialization { private static final int INITIAL_SERIALIZATION_BUFFER_SIZE = 10 * 1024; - static byte[] serializeSearchRequest(Query query, String serverId) { - return convertFromQuery(query, serverId).toByteArray(); + static byte[] serializeSearchRequest(Query query, int hits, String serverId) { + return convertFromQuery(query, hits, serverId).toByteArray(); } - private static SearchProtocol.SearchRequest convertFromQuery(Query query, String serverId) { - var builder = SearchProtocol.SearchRequest.newBuilder().setHits(query.getHits()).setOffset(query.getOffset()) + private static SearchProtocol.SearchRequest convertFromQuery(Query query, int hits, String serverId) { + var builder = SearchProtocol.SearchRequest.newBuilder().setHits(hits).setOffset(query.getOffset()) .setTimeout((int) query.getTimeLeft()); var documentDb = query.getModel().getDocumentDb(); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java index 870f7aef9c5..5c9928de924 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java @@ -35,8 +35,11 @@ public class RpcInvokerFactory extends InvokerFactory implements PingFactory { } @Override - protected Optional<SearchInvoker> createNodeSearchInvoker(VespaBackEndSearcher searcher, Query query, Node node) { - return Optional.of(new RpcSearchInvoker(searcher, node, rpcResourcePool)); + protected Optional<SearchInvoker> createNodeSearchInvoker(VespaBackEndSearcher searcher, + Query query, + int maxHits, + Node node) { + return Optional.of(new RpcSearchInvoker(searcher, node, rpcResourcePool, maxHits)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java index 59f17501c32..07d8439ff46 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java @@ -25,25 +25,28 @@ import java.util.concurrent.TimeUnit; * @author ollivir */ public class RpcSearchInvoker extends SearchInvoker implements Client.ResponseReceiver { + private static final String RPC_METHOD = "vespa.searchprotocol.search"; private final VespaBackEndSearcher searcher; private final Node node; private final RpcResourcePool resourcePool; private final BlockingQueue<Client.ResponseOrError<ProtobufResponse>> responses; + private final int maxHits; private Query query; - RpcSearchInvoker(VespaBackEndSearcher searcher, Node node, RpcResourcePool resourcePool) { + RpcSearchInvoker(VespaBackEndSearcher searcher, Node node, RpcResourcePool resourcePool, int maxHits) { super(Optional.of(node)); this.searcher = searcher; this.node = node; this.resourcePool = resourcePool; this.responses = new LinkedBlockingQueue<>(1); + this.maxHits = maxHits; } @Override - protected void sendSearchRequest(Query query) throws IOException { + protected void sendSearchRequest(Query query) { this.query = query; Client.NodeConnection nodeConnection = resourcePool.getConnection(node.key()); @@ -54,7 +57,7 @@ public class RpcSearchInvoker extends SearchInvoker implements Client.ResponseRe } query.trace(false, 5, "Sending search request with jrt/protobuf to node with dist key ", node.key()); - var payload = ProtobufSerialization.serializeSearchRequest(query, searcher.getServerId()); + var payload = ProtobufSerialization.serializeSearchRequest(query, Math.min(query.getHits(), maxHits), searcher.getServerId()); double timeoutSeconds = ((double) query.getTimeLeft() - 3.0) / 1000.0; Compressor.Compression compressionResult = resourcePool.compress(query, payload); nodeConnection.request(RPC_METHOD, compressionResult.type(), payload.length, compressionResult.data(), this, timeoutSeconds); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index fb55e330ebe..eff6ae26816 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -137,18 +137,8 @@ public class SearchCluster implements NodeManager<Node> { private static ImmutableList<Node> toNodes(DispatchConfig dispatchConfig) { ImmutableList.Builder<Node> nodesBuilder = new ImmutableList.Builder<>(); - Predicate<DispatchConfig.Node> filter; - if (dispatchConfig.useLocalNode()) { - final String hostName = HostName.getLocalhost(); - filter = node -> node.host().equals(hostName); - } else { - filter = node -> true; - } - for (DispatchConfig.Node node : dispatchConfig.node()) { - if (filter.test(node)) { - nodesBuilder.add(new Node(node.key(), node.host(), node.group())); - } - } + for (DispatchConfig.Node node : dispatchConfig.node()) + nodesBuilder.add(new Node(node.key(), node.host(), node.group())); return nodesBuilder.build(); } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java index 291b0f4890a..de6bafa267a 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java @@ -108,7 +108,8 @@ public class DispatcherTest { Query query, OptionalInt groupId, List<Node> nodes, - boolean acceptIncompleteCoverage) { + boolean acceptIncompleteCoverage, + int maxHitsPerNode) { if (step >= events.length) { throw new RuntimeException("Was not expecting more calls to getSearchInvoker"); } @@ -126,7 +127,10 @@ public class DispatcherTest { } @Override - protected Optional<SearchInvoker> createNodeSearchInvoker(VespaBackEndSearcher searcher, Query query, Node node) { + protected Optional<SearchInvoker> createNodeSearchInvoker(VespaBackEndSearcher searcher, + Query query, + int maxHitsPerNode, + Node node) { fail("Unexpected call to createNodeSearchInvoker"); return null; } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java index c07bf119782..ce19224b35f 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java @@ -19,15 +19,15 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** * @author ollivir */ public class RpcSearchInvokerTest { + @Test public void testProtobufSerialization() throws IOException { var compressionTypeHolder = new AtomicReference<CompressionType>(); @@ -35,8 +35,7 @@ public class RpcSearchInvokerTest { var lengthHolder = new AtomicInteger(); var mockClient = parameterCollectorClient(compressionTypeHolder, payloadHolder, lengthHolder); var mockPool = new RpcResourcePool(ImmutableMap.of(7, mockClient.createConnection("foo", 123))); - @SuppressWarnings("resource") - var invoker = new RpcSearchInvoker(mockSearcher(), new Node(7, "seven", 1), mockPool); + var invoker = new RpcSearchInvoker(mockSearcher(), new Node(7, "seven", 1), mockPool, 1000); Query q = new Query("search/?query=test&hits=10&offset=3"); invoker.sendSearchRequest(q); @@ -44,9 +43,28 @@ public class RpcSearchInvokerTest { var bytes = mockPool.compressor().decompress(payloadHolder.get(), compressionTypeHolder.get(), lengthHolder.get()); var request = SearchProtocol.SearchRequest.newBuilder().mergeFrom(bytes).build(); - assertThat(request.getHits(), equalTo(10)); - assertThat(request.getOffset(), equalTo(3)); - assertThat(request.getQueryTreeBlob().size(), greaterThan(0)); + assertEquals(10, request.getHits()); + assertEquals(3, request.getOffset()); + assertTrue(request.getQueryTreeBlob().size() > 0); + } + + @Test + public void testProtobufSerializationWithMaxHitsSet() throws IOException { + int maxHits = 5; + var compressionTypeHolder = new AtomicReference<CompressionType>(); + var payloadHolder = new AtomicReference<byte[]>(); + var lengthHolder = new AtomicInteger(); + var mockClient = parameterCollectorClient(compressionTypeHolder, payloadHolder, lengthHolder); + var mockPool = new RpcResourcePool(ImmutableMap.of(7, mockClient.createConnection("foo", 123))); + var invoker = new RpcSearchInvoker(mockSearcher(), new Node(7, "seven", 1), mockPool, maxHits); + + Query q = new Query("search/?query=test&hits=10&offset=3"); + invoker.sendSearchRequest(q); + + var bytes = mockPool.compressor().decompress(payloadHolder.get(), compressionTypeHolder.get(), lengthHolder.get()); + var request = SearchProtocol.SearchRequest.newBuilder().mergeFrom(bytes).build(); + + assertEquals(maxHits, request.getHits()); } private Client parameterCollectorClient(AtomicReference<CompressionType> compressionTypeHolder, AtomicReference<byte[]> payloadHolder, @@ -91,4 +109,5 @@ public class RpcSearchInvokerTest { } }; } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ClusterMetrics.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ClusterMetrics.java index 9296e144bf7..f16e2e403ed 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ClusterMetrics.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ClusterMetrics.java @@ -18,10 +18,10 @@ public class ClusterMetrics { public static final String QUERY_LATENCY = "queryLatency"; private final String clusterId; - private final ClusterType clusterType; + private final String clusterType; private final Map<String, Double> metrics; - public ClusterMetrics(String clusterId, ClusterType clusterType) { + public ClusterMetrics(String clusterId, String clusterType) { this.clusterId = clusterId; this.clusterType = clusterType; this.metrics = new HashMap<>(); @@ -31,7 +31,7 @@ public class ClusterMetrics { return clusterId; } - public ClusterType getClusterType() { + public String getClusterType() { return clusterType; } @@ -60,9 +60,4 @@ public class ClusterMetrics { return this; } - public enum ClusterType { - content, - container - } - } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/CloudEvent.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/CloudEvent.java index a7c8a680b73..defcda28a0f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/CloudEvent.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/CloudEvent.java @@ -14,10 +14,10 @@ public final class CloudEvent { public final Optional<Date> notAfter; public String awsRegionName; - public Set<String> affectedHostnames; + public Set<String> affectedInstances; public CloudEvent(String instanceEventId, String code, String description, Date notAfter, Date notBefore, Date notBeforeDeadline, - String awsRegionName, Set<String> affectedHostnames) { + String awsRegionName, Set<String> affectedInstances) { this.instanceEventId = instanceEventId; this.code = code; this.description = description; @@ -26,6 +26,6 @@ public final class CloudEvent { this.notAfter = Optional.ofNullable(notAfter); this.awsRegionName = awsRegionName; - this.affectedHostnames = affectedHostnames; + this.affectedInstances = affectedInstances; } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockAwsEventFetcher.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockAwsEventFetcher.java index 79b332c093a..baf248fc31c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockAwsEventFetcher.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockAwsEventFetcher.java @@ -2,18 +2,29 @@ package com.yahoo.vespa.hosted.controller.api.integration.aws; import com.yahoo.vespa.hosted.controller.api.integration.organization.Issue; +import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import java.util.List; +import java.util.Map; +import java.util.HashMap; +import java.util.ArrayList; import java.util.Optional; public class MockAwsEventFetcher implements AwsEventFetcher { + + private Map<String, List<CloudEvent>> mockedEvents = new HashMap<>(); + @Override public List<CloudEvent> getEvents(String awsRegionName) { - return List.of(); + return mockedEvents.getOrDefault(awsRegionName, new ArrayList<>()); } @Override public Issue createIssue(CloudEvent event) { - return new Issue("summary", "description", "VESPA", Optional.empty()); + return new Issue("summary", event.affectedInstances.toString(), "VESPA", Optional.empty()).with(User.from(event.awsRegionName)); + } + + public void addEvent(String awsRegionName, CloudEvent cloudEvent) { + mockedEvents.computeIfAbsent(awsRegionName, i -> new ArrayList<>()).add(cloudEvent); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java index 43fea2b76fd..8bb5775566a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java @@ -37,11 +37,13 @@ public class Node { private final String flavor; private final String clusterId; private final ClusterType clusterType; + private final boolean wantToRetire; + private final boolean wantToDeprovision; public Node(HostName hostname, Optional<HostName> parentHostname, State state, NodeType type, NodeResources resources, Optional<ApplicationId> owner, Version currentVersion, Version wantedVersion, Version currentOsVersion, Version wantedOsVersion, ServiceState serviceState, long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration, - int cost, String flavor, String clusterId, ClusterType clusterType) { + int cost, String flavor, String clusterId, ClusterType clusterType, boolean wantToRetire, boolean wantToDeprovision) { this.hostname = hostname; this.parentHostname = parentHostname; this.state = state; @@ -61,6 +63,8 @@ public class Node { this.flavor = flavor; this.clusterId = clusterId; this.clusterType = clusterType; + this.wantToRetire = wantToRetire; + this.wantToDeprovision = wantToDeprovision; } public HostName hostname() { @@ -137,6 +141,14 @@ public class Node { return clusterType; } + public boolean wantToRetire() { + return wantToRetire; + } + + public boolean wantToDeprovision() { + return wantToDeprovision; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -198,6 +210,8 @@ public class Node { private String flavor; private String clusterId; private ClusterType clusterType; + private boolean wantToRetire; + private boolean wantToDeprovision; public Builder() { } @@ -221,6 +235,8 @@ public class Node { this.flavor = node.flavor; this.clusterId = node.clusterId; this.clusterType = node.clusterType; + this.wantToRetire = node.wantToRetire; + this.wantToDeprovision = node.wantToDeprovision; } public Builder hostname(HostName hostname) { @@ -318,10 +334,20 @@ public class Node { return this; } + public Builder wantToRetire(boolean wantToRetire) { + this.wantToRetire = wantToRetire; + return this; + } + + public Builder wantToDeprovision(boolean wantToDeprovision) { + this.wantToDeprovision = wantToDeprovision; + return this; + } + public Node build() { return new Node(hostname, parentHostname, state, type, resources, owner, currentVersion, wantedVersion, currentOsVersion, wantedOsVersion, serviceState, restartGeneration, wantedRestartGeneration, rebootGeneration, wantedRebootGeneration, - cost, flavor, clusterId, clusterType); + cost, flavor, clusterId, clusterType, wantToRetire, wantToDeprovision); } } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java index 94616fd27b2..dd99bef5ee2 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java @@ -73,6 +73,8 @@ public interface NodeRepository { /** Cancels firmware checks on all hosts in the given zone. */ void cancelFirmwareCheck(ZoneId zone); + void retireAndDeprovision(ZoneId zoneId, String hostName); + private static Node toNode(NodeRepositoryNode node) { var application = Optional.ofNullable(node.getOwner()) .map(owner -> ApplicationId.from(owner.getTenant(), owner.getApplication(), @@ -103,7 +105,9 @@ public interface NodeRepository { toInt(node.getCost()), node.getFlavor(), clusterIdOf(node.getMembership()), - clusterTypeOf(node.getMembership())); + clusterTypeOf(node.getMembership()), + node.getWantToRetire(), + node.getWantToDeprovision()); } private static String clusterIdOf(NodeMembership nodeMembership) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index 5ff564f7ad3..66015c76b06 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -64,7 +64,7 @@ public enum SystemApplication { } /** Returns whether this should receive OS upgrades */ - public boolean isEligibleForOsUpgrades() { + public boolean shouldUpgradeOs() { return nodeType.isDockerHost(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java index eda3d0fc571..bd8faaed2e2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java @@ -2,23 +2,28 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.aws.AwsEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.aws.CloudEvent; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.organization.Issue; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueHandler; import java.time.Duration; import java.util.List; -import java.util.Set; +import java.util.Map; +import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; /** - * Automatically fetches scheduled events from AWS and submits issues detailing them. - * + * Automatically fetches and handles scheduled events from AWS: + * 1. Deprovisions the affected hosts if applicable + * 2. Submits an issue detailing the event if some hosts are not processed by 1. * @author mgimle */ public class CloudEventReporter extends Maintainer { @@ -27,33 +32,70 @@ public class CloudEventReporter extends Maintainer { private final IssueHandler issueHandler; private final AwsEventFetcher eventFetcher; - private final Set<String> awsRegions; + private final Map<String, List<ZoneApi>> zonesByCloudNativeRegion; + private final NodeRepository nodeRepository; CloudEventReporter(Controller controller, Duration interval, JobControl jobControl) { super(controller, interval, jobControl); this.issueHandler = controller.serviceRegistry().issueHandler(); this.eventFetcher = controller.serviceRegistry().eventFetcherService(); - this.awsRegions = controller.zoneRegistry().zones() - .ofCloud(CloudName.from("aws")) - .reachable() - .zones().stream() - .map(ZoneApi::getCloudNativeRegionName) - .collect(Collectors.toSet()); + this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); + this.zonesByCloudNativeRegion = getZonesByCloudNativeRegion(); } @Override protected void maintain() { log.log(Level.INFO, "Fetching events for cloud hosts."); - for (var awsRegion : awsRegions) { + for (var awsRegion : zonesByCloudNativeRegion.keySet()) { List<CloudEvent> events = eventFetcher.getEvents(awsRegion); for (var event : events) { - Issue issue = eventFetcher.createIssue(event); - if (!issueHandler.issueExists(issue)) { - issueHandler.file(issue); - log.log(Level.INFO, String.format("Filed an issue with the title '%s'", issue.summary())); - } + List<String> deprovisionedHosts = deprovisionHosts(awsRegion, event); + submitIssue(event, deprovisionedHosts); } } } + private List<String> deprovisionHosts(String awsRegion, CloudEvent event) { + return zonesByCloudNativeRegion.get(awsRegion) + .stream() + .flatMap(zone -> + nodeRepository.list(zone.getId()) + .stream() + .filter(shouldDeprovisionHost(event)) + .map(node -> { + if (!node.wantToDeprovision() || !node.wantToRetire()) + log.info(String.format("Setting host %s to wantToRetire and wantToDeprovision", node.hostname().value())); + nodeRepository.retireAndDeprovision(zone.getId(), node.hostname().value()); + return node.hostname().value(); + }) + ) + .collect(Collectors.toList()); + } + + private void submitIssue(CloudEvent event, List<String> deprovisionedHosts) { + if (event.affectedInstances.size() == deprovisionedHosts.size()) + return; + Issue issue = eventFetcher.createIssue(event); + if (!issueHandler.issueExists(issue)) { + issueHandler.file(issue); + log.log(Level.INFO, String.format("Filed an issue with the title '%s'", issue.summary())); + } + } + + private Predicate<Node> shouldDeprovisionHost(CloudEvent event) { + return node -> + node.type() == NodeType.host && + event.affectedInstances.stream() + .anyMatch(instance -> node.hostname().value().contains(instance)); + } + + private Map<String, List<ZoneApi>> getZonesByCloudNativeRegion() { + return controller().zoneRegistry().zones() + .ofCloud(CloudName.from("aws")) + .reachable() + .zones().stream() + .collect(Collectors.groupingBy( + ZoneApi::getCloudNativeRegionName + )); + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java index 0dd6f0782bf..0a890834c6e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java @@ -49,10 +49,12 @@ public abstract class InfrastructureUpgrader extends Maintainer { converged &= upgradeAll(target, applications, zone); } catch (UnreachableNodeRepositoryException e) { converged = false; - log.warning(String.format("%s: Failed to communicate with node repository in %s, continuing with next parallel zone: %s", this, zone, Exceptions.toMessageString(e))); + log.warning(String.format("%s: Failed to communicate with node repository in %s, continuing with next parallel zone: %s", + this, zone, Exceptions.toMessageString(e))); } catch (Exception e) { converged = false; - log.warning(String.format("%s: Failed to upgrade zone: %s, continuing with next parallel zone: %s", this, zone, Exceptions.toMessageString(e))); + log.warning(String.format("%s: Failed to upgrade zone: %s, continuing with next parallel zone: %s", + this, zone, Exceptions.toMessageString(e))); } } if (!converged) { @@ -66,13 +68,10 @@ public abstract class InfrastructureUpgrader extends Maintainer { boolean converged = true; for (SystemApplication application : applications) { if (convergedOn(target, application.dependencies(), zone)) { - boolean currentAppConverged = convergedOn(target, application, zone); - // In dynamically provisioned zones there may be no tenant hosts at the time of upgrade, so we - // should always set the target version. - if (application == SystemApplication.tenantHost || !currentAppConverged) { + if (shouldUpgrade(target, application, zone)) { upgrade(target, application, zone); } - converged &= currentAppConverged; + converged &= convergedOn(target, application, zone); } } return converged; @@ -82,6 +81,9 @@ public abstract class InfrastructureUpgrader extends Maintainer { return applications.stream().allMatch(application -> convergedOn(target, application, zone)); } + /** Returns whether application in zone should be told to upgrade to given target */ + protected abstract boolean shouldUpgrade(Version target, SystemApplication application, ZoneApi zone); + /** Upgrade component to target version. Implementation should be idempotent */ protected abstract void upgrade(Version target, SystemApplication application, ZoneApi zone); @@ -94,7 +96,7 @@ public abstract class InfrastructureUpgrader extends Maintainer { /** Returns whether the upgrader should require given node to upgrade */ protected abstract boolean requireUpgradeOf(Node node, SystemApplication application, ZoneApi zone); - /** Find the minimum value of a version field in a zone */ + /** Find the minimum value of a version field in a zone by comparing all nodes */ protected final Optional<Version> minVersion(ZoneApi zone, SystemApplication application, Function<Node, Version> versionField) { try { return controller().serviceRegistry().configServer() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java index 21fd0ec87d4..9c3c1dc1f5e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.google.common.collect.ImmutableSet; import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; @@ -34,13 +33,15 @@ public abstract class Maintainer extends AbstractComponent implements Runnable { private final JobControl jobControl; private final ScheduledExecutorService service; private final String name; - private final Set<SystemName> permittedSystems; + + /** The systems in which this maintainer should run */ + private final Set<SystemName> activeSystems; public Maintainer(Controller controller, Duration interval, JobControl jobControl) { this(controller, interval, jobControl, null, EnumSet.allOf(SystemName.class)); } - public Maintainer(Controller controller, Duration interval, JobControl jobControl, String name, Set<SystemName> permittedSystems) { + public Maintainer(Controller controller, Duration interval, JobControl jobControl, String name, Set<SystemName> activeSystems) { if (interval.isNegative() || interval.isZero()) throw new IllegalArgumentException("Interval must be positive, but was " + interval); @@ -48,7 +49,7 @@ public abstract class Maintainer extends AbstractComponent implements Runnable { this.maintenanceInterval = interval; this.jobControl = jobControl; this.name = name; - this.permittedSystems = ImmutableSet.copyOf(permittedSystems); + this.activeSystems = Set.copyOf(activeSystems); service = new ScheduledThreadPoolExecutor(1); long delay = staggeredDelay(controller.curator().cluster(), controller.hostname(), controller.clock().instant(), interval); @@ -61,7 +62,7 @@ public abstract class Maintainer extends AbstractComponent implements Runnable { @Override public void run() { try { - if ( ! permittedSystems.contains(controller.system())) { + if ( ! activeSystems.contains(controller.system())) { return; } if (jobControl.isActive(name())) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java index 93d1dac7382..dcb93937eb3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java @@ -38,10 +38,6 @@ public class OsUpgrader extends InfrastructureUpgrader { @Override protected void upgrade(Version target, SystemApplication application, ZoneApi zone) { - if (!application.isEligibleForOsUpgrades()) return; - var existingTarget = targetVersion(zone, application); - if (existingTarget.isPresent() && existingTarget.get().equals(target)) return; - log.info(String.format("Upgrading OS of %s to version %s in %s in cloud %s", application.id(), target, zone.getId(), zone.getCloudName())); controller().serviceRegistry().configServer().nodeRepository().upgradeOs(zone.getId(), application.nodeType(), target); } @@ -65,20 +61,24 @@ public class OsUpgrader extends InfrastructureUpgrader { .map(OsVersion::version); } - private Version currentVersion(ZoneApi zone, SystemApplication application, Version defaultVersion) { - return minVersion(zone, application, Node::currentOsVersion).orElse(defaultVersion); - } - - private Optional<Version> targetVersion(ZoneApi zone, SystemApplication application) { + @Override + protected boolean shouldUpgrade(Version target, SystemApplication application, ZoneApi zone) { + if (!application.shouldUpgradeOs()) return false; // Never upgrade return controller().serviceRegistry().configServer().nodeRepository() .targetVersionsOf(zone.getId()) - .osVersion(application.nodeType()); + .osVersion(application.nodeType()) + .map(target::isAfter) // Upgrade if target is after current + .orElse(true); // Upgrade if target is unset + } + + private Version currentVersion(ZoneApi zone, SystemApplication application, Version defaultVersion) { + return minVersion(zone, application, Node::currentOsVersion).orElse(defaultVersion); } /** Returns whether node in application should be upgraded by this */ public static boolean eligibleForUpgrade(Node node, SystemApplication application) { return upgradableNodeStates.contains(node.state()) && - application.isEligibleForOsUpgrades(); + application.shouldUpgradeOs(); } private static String name(CloudName cloud) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index c43278f95b4..9b1bb300354 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -30,12 +30,8 @@ public class SystemUpgrader extends InfrastructureUpgrader { @Override protected void upgrade(Version target, SystemApplication application, ZoneApi zone) { - // TODO(mpolden): Simplify this by comparing with version from NodeRepository#targetVersionsOf instead - if (minVersion(zone, application, Node::wantedVersion).map(target::isAfter) - .orElse(true)) { - log.info(String.format("Deploying %s version %s in %s", application.id(), target, zone.getId())); - controller().applications().deploy(application, zone.getId(), target); - } + log.info(String.format("Deploying %s version %s in %s", application.id(), target, zone.getId())); + controller().applications().deploy(application, zone.getId(), target); } @Override @@ -61,6 +57,22 @@ public class SystemUpgrader extends InfrastructureUpgrader { .map(VespaVersion::versionNumber); } + @Override + protected boolean shouldUpgrade(Version target, SystemApplication application, ZoneApi zone) { + if (application.hasApplicationPackage()) { + // For applications with package we do not have a zone-wide version target. This means that we must check + // the wanted version of each node. + return minVersion(zone, application, Node::wantedVersion) + .map(target::isAfter) // Upgrade if target is after any wanted version + .orElse(true); // Upgrade if there are no nodes allocated + } + return controller().serviceRegistry().configServer().nodeRepository() + .targetVersionsOf(zone.getId()) + .vespaVersion(application.nodeType()) + .map(target::isAfter) // Upgrade if target is after current + .orElse(true); // Upgrade if target is unset + } + /** Returns whether node in application should be upgraded by this */ public static boolean eligibleForUpgrade(Node node) { return upgradableNodeStates.contains(node.state()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java index d5e83d99cdd..1773a9c122e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java @@ -65,7 +65,7 @@ public class OsVersionStatus { controller.osVersions().forEach(osVersion -> osVersions.put(osVersion, new ArrayList<>())); for (var application : SystemApplication.all()) { - if (!application.isEligibleForOsUpgrades()) continue; + if (!application.shouldUpgradeOs()) continue; for (var zone : zonesToUpgrade(controller)) { var targetOsVersion = controller.serviceRegistry().configServer().nodeRepository() .targetVersionsOf(zone.getId()) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 82d35701e7e..84bdedba33c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -234,6 +234,9 @@ public final class ControllerTester { public void upgradeSystemApplications(Version version, List<SystemApplication> systemApplications) { for (ZoneApi zone : zoneRegistry().zones().all().zones()) { for (SystemApplication application : systemApplications) { + if (!application.hasApplicationPackage()) { + configServer().nodeRepository().upgrade(zone.getId(), application.nodeType(), version); + } configServer().setVersion(application.id(), zone.getId(), version); configServer().convergeServices(application.id(), zone.getId()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index 36f962d653f..f99396b3b02 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -145,7 +145,14 @@ public class NodeRepositoryMock implements NodeRepository { @Override public void upgrade(ZoneId zone, NodeType type, Version version) { - nodeRepository.getOrDefault(zone, Collections.emptyMap()).values() + this.targetVersions.compute(zone, (ignored, targetVersions) -> { + if (targetVersions == null) { + targetVersions = TargetVersions.EMPTY; + } + return targetVersions.withVespaVersion(type, version); + }); + // Bump wanted version of each node. This is done by InfrastructureProvisioner in a real node repository. + nodeRepository.getOrDefault(zone, Map.of()).values() .stream() .filter(node -> node.type() == type) .map(node -> new Node.Builder(node).wantedVersion(version).build()) @@ -160,6 +167,12 @@ public class NodeRepositoryMock implements NodeRepository { } return targetVersions.withOsVersion(type, version); }); + // Bump wanted version of each node. This is done by OsUpgradeActivator in a real node repository. + nodeRepository.getOrDefault(zone, Map.of()).values() + .stream() + .filter(node -> node.type() == type) + .map(node -> new Node.Builder(node).wantedOsVersion(version).build()) + .forEach(node -> putByHostname(zone, node)); } @Override @@ -175,6 +188,11 @@ public class NodeRepositoryMock implements NodeRepository { public void cancelFirmwareCheck(ZoneId zone) { } + @Override + public void retireAndDeprovision(ZoneId zoneId, String hostName) { + nodeRepository.get(zoneId).remove(HostName.from(hostName)); + } + public void doUpgrade(DeploymentId deployment, Optional<HostName> hostName, Version version) { modifyNodes(deployment, hostName, node -> { assert node.wantedVersion().equals(version); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java new file mode 100644 index 00000000000..cd2a4fd8453 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java @@ -0,0 +1,156 @@ +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.aws.CloudEvent; +import com.yahoo.vespa.hosted.controller.api.integration.aws.MockAwsEventFetcher; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; +import com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueHandler; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; +import org.junit.Test; + +import java.time.Duration; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.Assert.*; + +/** + * @author olaa + */ +public class CloudEventReporterTest { + + private ControllerTester tester = new ControllerTester(); + private ZoneApiMock nonAwsZone = createZone("prod.zone3", "region-1", "other"); + private ZoneApiMock awsZone1 = createZone("prod.zone1", "region-1", "aws"); + private ZoneApiMock awsZone2 = createZone("prod.zone2", "region-2", "aws"); + + + /** + * Test scenario: + * Consider three zones, two of which are based in AWS + * We want to test the following: + * 1. Non-AWS zone is completely ignored + * 2. Tenant hosts affected by cloud event are deprovisioned + * 3. Infrastructure hosts affected by cloud event are reported by IssueHandler + */ + @Test + public void maintain() { + setUpZones(); + CloudEventReporter cloudEventReporter = new CloudEventReporter(tester.controller(), Duration.ofMinutes(15), new JobControl(tester.curator())); + + assertEquals(Set.of("host1.com", "host2.com", "host3.com"), getHostnames(nonAwsZone.getId())); + assertEquals(Set.of("host1.com", "host2.com", "host3.com"), getHostnames(awsZone1.getId())); + assertEquals(Set.of("host4.com", "host5.com", "confighost.com"), getHostnames(awsZone2.getId())); + + mockEvents(); + cloudEventReporter.maintain(); + + assertEquals(Set.of("host1.com", "host2.com", "host3.com"), getHostnames(nonAwsZone.getId())); + assertEquals(Set.of("host3.com"), getHostnames(awsZone1.getId())); + assertEquals(Set.of("host4.com", "confighost.com"), getHostnames(awsZone2.getId())); + + Map<IssueId, MockIssueHandler.MockIssue> createdIssues = tester.serviceRegistry().issueHandler().issues(); + assertEquals(1, createdIssues.size()); + String description = createdIssues.get(IssueId.from("1")).issue().description(); + assertTrue(description.contains("confighost")); + + } + + private void mockEvents() { + MockAwsEventFetcher mockAwsEventFetcher = (MockAwsEventFetcher)tester.controller().serviceRegistry().eventFetcherService(); + + Date date = new Date(); + CloudEvent event1 = new CloudEvent("event 1", + "instance code", + "description", + date, + date, + date, + "region-1", + Set.of("host1", "host2")); + + CloudEvent event2 = new CloudEvent("event 2", + "instance code", + "description", + date, + date, + date, + "region-2", + Set.of("host5", "confighost")); + + mockAwsEventFetcher.addEvent("region-1", event1); + mockAwsEventFetcher.addEvent("region-2", event2); + } + + private void setUpZones() { + + tester.zoneRegistry().setZones( + nonAwsZone, + awsZone1, + awsZone2); + + tester.configServer().nodeRepository().putByHostname( + nonAwsZone.getId(), + createNodesWithHostnames( + "host1.com", + "host2.com", + "host3.com" + ) + ); + tester.configServer().nodeRepository().putByHostname( + awsZone1.getId(), + createNodesWithHostnames( + "host1.com", + "host2.com", + "host3.com" + ) + ); + tester.configServer().nodeRepository().putByHostname( + awsZone2.getId(), + createNodesWithHostnames( + "host4.com", + "host5.com" + ) + ); + tester.configServer().nodeRepository().putByHostname( + awsZone2.getId(), + List.of(createNode("confighost.com", NodeType.confighost)) + ); + } + + private List<Node> createNodesWithHostnames(String... hostnames) { + return Arrays.stream(hostnames) + .map(hostname -> createNode(hostname, NodeType.host)) + .collect(Collectors.toUnmodifiableList()); + } + + private Node createNode(String hostname, NodeType nodeType) { + return new Node.Builder() + .hostname(HostName.from(hostname)) + .type(nodeType) + .build(); + } + + private Set<String> getHostnames(ZoneId zoneId) { + return tester.configServer().nodeRepository().list(zoneId) + .stream() + .map(node -> node.hostname().value()) + .collect(Collectors.toSet()); + } + + private ZoneApiMock createZone(String zoneId, String cloudNativeRegionName, String cloud) { + return ZoneApiMock.newBuilder().withId(zoneId) + .withCloudNativeRegionName(cloudNativeRegionName) + .withCloud(cloud) + .build(); + } + +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java index 06a815819f4..ed8918786c5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java @@ -116,7 +116,7 @@ public class DeploymentMetricsMaintainerTest { } private void setMetrics(ApplicationId application, Map<String, Double> metrics) { - var clusterMetrics = new ClusterMetrics("default", ClusterMetrics.ClusterType.container); + var clusterMetrics = new ClusterMetrics("default", "container"); for (var kv : metrics.entrySet()) { clusterMetrics = clusterMetrics.addMetric(kv.getKey(), kv.getValue()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 99eb4f049b6..e0fb2aa0ee1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; @@ -19,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import org.junit.Test; import java.time.Duration; +import java.util.Comparator; import java.util.List; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; @@ -250,6 +252,9 @@ public class MetricsReporterTest { assertEquals(0, getNodesFailingUpgrade()); // 1/3 nodes upgrade within timeout + assertEquals("Wanted version is raised for all nodes", version, + tester.configServer().nodeRepository().list(zone1.getId(), SystemApplication.configServer.id()).stream() + .map(Node::wantedVersion).min(Comparator.naturalOrder()).get()); tester.configServer().setVersion(SystemApplication.configServer.id(), zone1.getId(), version, 1); tester.clock().advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1))); tester.computeVersionStatus(); @@ -303,6 +308,9 @@ public class MetricsReporterTest { assertEquals(0, getNodesFailingOsUpgrade()); // 2/6 nodes upgrade within timeout + assertEquals("Wanted OS version is raised for all nodes", version, + tester.configServer().nodeRepository().list(zone.getId(), SystemApplication.tenantHost.id()).stream() + .map(Node::wantedOsVersion).min(Comparator.naturalOrder()).get()); tester.configServer().setOsVersion(SystemApplication.tenantHost.id(), zone.getId(), version, 2); tester.clock().advance(Duration.ofMinutes(30 * 3 /* time allowance * node count */).plus(Duration.ofSeconds(1))); statusUpdater.maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index cf3d978ef62..9d910afd476 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -13,7 +13,6 @@ import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.versions.NodeVersion; -import org.junit.Before; import org.junit.Test; import java.time.Duration; @@ -35,16 +34,9 @@ public class OsUpgraderTest { private static final ZoneApi zone4 = ZoneApiMock.newBuilder().withId("prod.us-east-3").build(); private static final ZoneApi zone5 = ZoneApiMock.newBuilder().withId("prod.us-north-1").withCloud("other").build(); - private ControllerTester tester; - private OsVersionStatusUpdater statusUpdater; - - @Before - public void before() { - tester = new ControllerTester(); - statusUpdater = new OsVersionStatusUpdater(tester.controller(), Duration.ofDays(1), - new JobControl(tester.controller().curator())); - } - + private final ControllerTester tester = new ControllerTester(); + private final OsVersionStatusUpdater statusUpdater = new OsVersionStatusUpdater(tester.controller(), Duration.ofDays(1), + new JobControl(tester.controller().curator())); @Test public void upgrade_os() { OsUpgrader osUpgrader = osUpgrader( diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index 08dbd7e48db..dbcd9b7f6c3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -10,13 +10,13 @@ import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import org.junit.Before; import org.junit.Test; import java.time.Duration; import java.util.List; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -31,12 +31,7 @@ public class SystemUpgraderTest { private static final ZoneApi zone3 = ZoneApiMock.fromId("prod.us-central-1"); private static final ZoneApi zone4 = ZoneApiMock.fromId("prod.us-east-3"); - private ControllerTester tester; - - @Before - public void before() { - tester = new ControllerTester(); - } + private final ControllerTester tester = new ControllerTester(); @Test public void upgrade_system() { @@ -295,27 +290,26 @@ public class SystemUpgraderTest { } /** Simulate upgrade of nodes allocated to given application. In a real system this is done by the node itself */ - private void completeUpgrade(SystemApplication application, Version version, ZoneApi... zones) { - assertWantedVersion(application, version, zones); - for (ZoneApi zone : zones) { + private void completeUpgrade(SystemApplication application, Version version, ZoneApi first, ZoneApi... rest) { + assertWantedVersion(application, version, first, rest); + Stream.concat(Stream.of(first), Stream.of(rest)).forEach(zone -> { for (Node node : listNodes(zone, application)) { nodeRepository().putByHostname( zone.getId(), new Node.Builder(node).currentVersion(node.wantedVersion()).build()); } - assertCurrentVersion(application, version, zone); - } + }); } - private void convergeServices(SystemApplication application, ZoneApi... zones) { - for (ZoneApi zone : zones) { + private void convergeServices(SystemApplication application, ZoneApi first, ZoneApi... rest) { + Stream.concat(Stream.of(first), Stream.of(rest)).forEach(zone -> { tester.configServer().convergeServices(application.id(), zone.getId()); - } + }); } - private void completeUpgrade(List<SystemApplication> applications, Version version, ZoneApi... zones) { - applications.forEach(application -> completeUpgrade(application, version, zones)); + private void completeUpgrade(List<SystemApplication> applications, Version version, ZoneApi zone, ZoneApi... rest) { + applications.forEach(application -> completeUpgrade(application, version, zone, rest)); } private void failNodeIn(ZoneApi zone, SystemApplication application) { @@ -337,29 +331,35 @@ public class SystemUpgraderTest { assertEquals(version, tester.controller().versionStatus().controllerVersion().get().versionNumber()); } - private void assertWantedVersion(SystemApplication application, Version version, ZoneApi... zones) { - assertVersion(application, version, Node::wantedVersion, zones); + private void assertWantedVersion(SystemApplication application, Version version, ZoneApi first, ZoneApi... rest) { + Stream.concat(Stream.of(first), Stream.of(rest)).forEach(zone -> { + if (!application.hasApplicationPackage()) { + assertEquals("Target version set for " + application + " in " + zone.getId(), version, + nodeRepository().targetVersionsOf(zone.getId()).vespaVersion(application.nodeType()).orElse(Version.emptyVersion)); + } + assertVersion(application, version, Node::wantedVersion, zone); + }); } - private void assertCurrentVersion(SystemApplication application, Version version, ZoneApi... zones) { - assertVersion(application, version, Node::currentVersion, zones); + private void assertCurrentVersion(SystemApplication application, Version version, ZoneApi first, ZoneApi... rest) { + assertVersion(application, version, Node::currentVersion, first, rest); } - private void assertWantedVersion(List<SystemApplication> applications, Version version, ZoneApi... zones) { - applications.forEach(application -> assertVersion(application, version, Node::wantedVersion, zones)); + private void assertWantedVersion(List<SystemApplication> applications, Version version, ZoneApi first, ZoneApi... rest) { + applications.forEach(application -> assertWantedVersion(application, version, first, rest)); } - private void assertCurrentVersion(List<SystemApplication> applications, Version version, ZoneApi... zones) { - applications.forEach(application -> assertVersion(application, version, Node::currentVersion, zones)); + private void assertCurrentVersion(List<SystemApplication> applications, Version version, ZoneApi first, ZoneApi... rest) { + applications.forEach(application -> assertVersion(application, version, Node::currentVersion, first, rest)); } private void assertVersion(SystemApplication application, Version version, Function<Node, Version> versionField, - ZoneApi... zones) { - for (ZoneApi zone : requireNonEmpty(zones)) { + ZoneApi first, ZoneApi... rest) { + Stream.concat(Stream.of(first), Stream.of(rest)).forEach(zone -> { for (Node node : listNodes(zone, application)) { assertEquals(application + " version", version, versionField.apply(node)); } - } + }); } private List<Node> listNodes(ZoneApi zone, SystemApplication application) { @@ -378,9 +378,4 @@ public class SystemUpgraderTest { new JobControl(tester.curator())); } - private static <T> T[] requireNonEmpty(T[] args) { - if (args.length == 0) throw new IllegalArgumentException("Need at least one argument"); - return args; - } - } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetricsTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetricsTest.java index dad7ea4ec31..3d1cb3eba86 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetricsTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetricsTest.java @@ -44,12 +44,12 @@ public class ConfigServerMetricsTest { // var deploymentId = new DeploymentId(applicationId, zoneId); - var clusterMetrics1 = new ClusterMetrics("niceCluster", ClusterMetrics.ClusterType.container) {{ + var clusterMetrics1 = new ClusterMetrics("niceCluster", "container") {{ addMetric("queriesPerSecond", 23.0); addMetric("queryLatency", 1337.0); }}; - var clusterMetrics2 = new ClusterMetrics("alsoNiceCluster", ClusterMetrics.ClusterType.container) {{ + var clusterMetrics2 = new ClusterMetrics("alsoNiceCluster", "container") {{ addMetric("queriesPerSecond", 11.0); addMetric("queryLatency", 12.0); }}; diff --git a/default_build_settings.cmake b/default_build_settings.cmake index 7d43c99b45d..244883f01f4 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -87,6 +87,7 @@ endfunction() function(vespa_use_default_build_settings) if (DEFINED CMAKE_INSTALL_PREFIX AND DEFINED CMAKE_PREFIX_PATH AND + NOT CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT AND DEFINED VESPA_LLVM_VERSION AND DEFINED EXTRA_INCLUDE_DIRECTORY AND DEFINED EXTRA_LINK_DIRECTORY AND DEFINED CMAKE_INSTALL_RPATH AND DEFINED CMAKE_BUILD_RPATH) @@ -187,10 +188,9 @@ function(vespa_use_default_build_settings) if(DEFINED DEFAULT_VESPA_USER) message("-- DEFAULT_VESPA_USER is ${DEFAULT_VESPA_USER}") endif() - if(NOT DEFINED CMAKE_INSTALL_PREFIX AND DEFINED DEFAULT_CMAKE_INSTALL_PREFIX) + if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT AND DEFINED DEFAULT_CMAKE_INSTALL_PREFIX) message("-- Setting CMAKE_INSTALL_PREFIX to ${DEFAULT_CMAKE_INSTALL_PREFIX}") - set(CMAKE_INSTALL_PREFIX "${DEFAULT_CMAKE_INSTALL_PREFIX}" PARENT_SCOPE) - set(CMAKE_INSTALL_PREFIX "${DEFAULT_CMAKE_INSTALL_PREFIX}") + set(CMAKE_INSTALL_PREFIX "${DEFAULT_CMAKE_INSTALL_PREFIX}" CACHE PATH "Install prefix for vespa project" FORCE) endif() if(NOT DEFINED CMAKE_PREFIX_PATH AND DEFINED DEFAULT_CMAKE_PREFIX_PATH) message("-- Setting CMAKE_PREFIX_PATH to ${DEFAULT_CMAKE_PREFIX_PATH}") diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java index bd8ffb0163c..f1f40a6726e 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java @@ -112,7 +112,7 @@ public class ContainerResources { /** Returns only the CPU component(s) of {@link #toString()} */ public String toStringCpu() { - return (cpus > 0 ? cpus : "unlimited") +" CPUs, " + + return (cpus > 0 ? String.format("%.2f", cpus) : "unlimited") +" CPUs, " + (cpuShares > 0 ? cpuShares : "unlimited") + " CPU Shares"; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java index 3b2c635992e..2874546da52 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java @@ -13,12 +13,15 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; import java.util.Comparator; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; import java.util.function.Supplier; import java.util.logging.Logger; import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameEndsWith; import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameMatches; @@ -33,6 +36,7 @@ import static com.yahoo.yolean.Exceptions.uncheck; public class CoredumpHandler { private static final Pattern JAVA_CORE_PATTERN = Pattern.compile("java_pid.*\\.hprof"); + private static final Pattern HS_ERR_PATTERN = Pattern.compile("hs_err_pid[0-9]+\\.log"); private static final String LZ4_PATH = "/usr/bin/lz4"; private static final String PROCESSING_DIRECTORY_NAME = "processing"; private static final String METADATA_FILE_NAME = "metadata.json"; @@ -96,26 +100,39 @@ public class CoredumpHandler { } /** - * Moves a coredump to a new directory under the processing/ directory. Limit to only processing - * one coredump at the time, starting with the oldest. + * Moves a coredump and related hs_err file(s) to a new directory under the processing/ directory. + * Limit to only processing one coredump at the time, starting with the oldest. + * + * Assumption: hs_err files are much smaller than core files and are written (last modified time) + * before the core file. * * @return path to directory inside processing directory which contains the enqueued core dump file */ Optional<Path> enqueueCoredump(Path containerCrashPathOnHost, Path containerProcessingPathOnHost) { - return FileFinder.files(containerCrashPathOnHost) - .match(nameStartsWith(".").negate()) + List<Path> toProcess = FileFinder.files(containerCrashPathOnHost) + .match(nameStartsWith(".").negate()) // Skip core dump files currently being written .maxDepth(1) .stream() - .min(Comparator.comparing(FileFinder.FileAttributes::lastModifiedTime)) + .sorted(Comparator.comparing(FileFinder.FileAttributes::lastModifiedTime)) .map(FileFinder.FileAttributes::path) - .map(coredumpPath -> { - UnixPath coredumpInProcessingDirectory = new UnixPath( - containerProcessingPathOnHost - .resolve(coredumpIdSupplier.get()) - .resolve(COREDUMP_FILENAME_PREFIX + coredumpPath.getFileName())); - coredumpInProcessingDirectory.createParents(); - return uncheck(() -> Files.move(coredumpPath, coredumpInProcessingDirectory.toPath())).getParent(); + .collect(Collectors.toList()); + + int coredumpIndex = IntStream.range(0, toProcess.size()) + .filter(i -> !HS_ERR_PATTERN.matcher(toProcess.get(i).getFileName().toString()).matches()) + .findFirst() + .orElse(-1); + + // Either there are no files in crash directory, or all the files are hs_err files. + if (coredumpIndex == -1) return Optional.empty(); + + Path enqueuedDir = uncheck(() -> Files.createDirectories(containerProcessingPathOnHost.resolve(coredumpIdSupplier.get()))); + IntStream.range(0, coredumpIndex + 1) + .forEach(i -> { + Path path = toProcess.get(i); + String prefix = i == coredumpIndex ? COREDUMP_FILENAME_PREFIX : ""; + uncheck(() -> Files.move(path, enqueuedDir.resolve(prefix + path.getFileName()))); }); + return Optional.of(enqueuedDir); } void processAndReportSingleCoredump(NodeAgentContext context, Path coredumpDirectory, Supplier<Map<String, Object>> nodeAttributesSupplier) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java index 1d6ccff4212..3d9e3c08276 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java @@ -64,7 +64,7 @@ public class CoredumpHandlerTest { final Path processingDir = fileSystem.getPath("/home/docker/container-1/some/other/processing"); Files.createDirectories(crashPathOnHost); - Files.setLastModifiedTime(Files.createFile(crashPathOnHost.resolve(".bash.core.431")), FileTime.from(Instant.now())); + createFileAged(crashPathOnHost.resolve(".bash.core.431"), Duration.ZERO); assertFolderContents(crashPathOnHost, ".bash.core.431"); Optional<Path> enqueuedPath = coredumpHandler.enqueueCoredump(crashPathOnHost, processingDir); @@ -72,8 +72,8 @@ public class CoredumpHandlerTest { // bash.core.431 finished writing... and 2 more have since been written Files.move(crashPathOnHost.resolve(".bash.core.431"), crashPathOnHost.resolve("bash.core.431")); - Files.setLastModifiedTime(Files.createFile(crashPathOnHost.resolve("vespa-proton.core.119")), FileTime.from(Instant.now().minus(Duration.ofMinutes(10)))); - Files.setLastModifiedTime(Files.createFile(crashPathOnHost.resolve("vespa-slobrok.core.673")), FileTime.from(Instant.now().minus(Duration.ofMinutes(5)))); + createFileAged(crashPathOnHost.resolve("vespa-proton.core.119"), Duration.ofMinutes(10)); + createFileAged(crashPathOnHost.resolve("vespa-slobrok.core.673"), Duration.ofMinutes(5)); when(coredumpIdSupplier.get()).thenReturn("id-123").thenReturn("id-321"); enqueuedPath = coredumpHandler.enqueueCoredump(crashPathOnHost, processingDir); @@ -93,6 +93,27 @@ public class CoredumpHandlerTest { } @Test + public void enqueue_with_hs_err_files() throws IOException { + final Path crashPathOnHost = fileSystem.getPath("/home/docker/container-1/some/crash/path"); + final Path processingDir = fileSystem.getPath("/home/docker/container-1/some/other/processing"); + Files.createDirectories(crashPathOnHost); + + createFileAged(crashPathOnHost.resolve("java.core.69"), Duration.ofSeconds(15)); + createFileAged(crashPathOnHost.resolve("hs_err_pid69.log"), Duration.ofSeconds(20)); + + createFileAged(crashPathOnHost.resolve("java.core.2420"), Duration.ofSeconds(40)); + createFileAged(crashPathOnHost.resolve("hs_err_pid2420.log"), Duration.ofSeconds(49)); + createFileAged(crashPathOnHost.resolve("hs_err_pid2421.log"), Duration.ofSeconds(50)); + + when(coredumpIdSupplier.get()).thenReturn("id-123").thenReturn("id-321"); + Optional<Path> enqueuedPath = coredumpHandler.enqueueCoredump(crashPathOnHost, processingDir); + assertEquals(Optional.of(processingDir.resolve("id-123")), enqueuedPath); + assertFolderContents(crashPathOnHost, "hs_err_pid69.log", "java.core.69"); + assertFolderContents(processingDir, "id-123"); + assertFolderContents(processingDir.resolve("id-123"), "hs_err_pid2420.log", "hs_err_pid2421.log", "dump_java.core.2420"); + } + + @Test public void coredump_to_process_test() throws IOException { final Path crashPathOnHost = fileSystem.getPath("/home/docker/container-1/some/crash/path"); final Path processingDir = fileSystem.getPath("/home/docker/container-1/some/other/processing"); @@ -103,9 +124,9 @@ public class CoredumpHandlerTest { // 3 core dumps occur Files.createDirectories(crashPathOnHost); - Files.setLastModifiedTime(Files.createFile(crashPathOnHost.resolve("bash.core.431")), FileTime.from(Instant.now())); - Files.setLastModifiedTime(Files.createFile(crashPathOnHost.resolve("vespa-proton.core.119")), FileTime.from(Instant.now().minus(Duration.ofMinutes(10)))); - Files.setLastModifiedTime(Files.createFile(crashPathOnHost.resolve("vespa-slobrok.core.673")), FileTime.from(Instant.now().minus(Duration.ofMinutes(5)))); + createFileAged(crashPathOnHost.resolve("bash.core.431"), Duration.ZERO); + createFileAged(crashPathOnHost.resolve("vespa-proton.core.119"), Duration.ofMinutes(10)); + createFileAged(crashPathOnHost.resolve("vespa-slobrok.core.673"), Duration.ofMinutes(5)); when(coredumpIdSupplier.get()).thenReturn("id-123"); enqueuedPath = coredumpHandler.getCoredumpToProcess(crashPathOnHost, processingDir); @@ -207,4 +228,10 @@ public class CoredumpHandlerTest { .collect(Collectors.toSet()); assertEquals(expectedContentsOfFolder, actualContentsOfFolder); } + + private static Path createFileAged(Path path, Duration age) { + return uncheck(() -> Files.setLastModifiedTime( + Files.createFile(path), + FileTime.from(Instant.now().minus(age)))); + } } diff --git a/searchlib/src/tests/attribute/posting_list_merger/posting_list_merger_test.cpp b/searchlib/src/tests/attribute/posting_list_merger/posting_list_merger_test.cpp index a9fde8c8dd3..6ee3eeaccba 100644 --- a/searchlib/src/tests/attribute/posting_list_merger/posting_list_merger_test.cpp +++ b/searchlib/src/tests/attribute/posting_list_merger/posting_list_merger_test.cpp @@ -3,6 +3,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/attribute/posting_list_merger.h> #include <vespa/vespalib/test/insertion_operators.h> +#include <algorithm> using search::btree::BTreeNoLeafData; using search::attribute::PostingListMerger; diff --git a/searchlib/src/tests/common/bitvector/bitvector_test.cpp b/searchlib/src/tests/common/bitvector/bitvector_test.cpp index e61c21bee1c..4cbe96c74b5 100644 --- a/searchlib/src/tests/common/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/common/bitvector/bitvector_test.cpp @@ -9,6 +9,7 @@ #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/searchlib/util/rand48.h> +#include <algorithm> using namespace search; diff --git a/searchlib/src/tests/docstore/document_store/document_store_test.cpp b/searchlib/src/tests/docstore/document_store/document_store_test.cpp index 86c6d3ad883..f950377be4b 100644 --- a/searchlib/src/tests/docstore/document_store/document_store_test.cpp +++ b/searchlib/src/tests/docstore/document_store/document_store_test.cpp @@ -105,9 +105,9 @@ void verifyValue(vespalib::stringref s, const Value & v) { TEST("require that Value and cache entries have expected size") { using pair = std::pair<DocumentIdT, Value>; using Node = vespalib::hash_node<pair>; - EXPECT_EQUAL(64ul, sizeof(Value)); - EXPECT_EQUAL(72ul, sizeof(pair)); - EXPECT_EQUAL(80ul, sizeof(Node)); + EXPECT_EQUAL(48ul, sizeof(Value)); + EXPECT_EQUAL(56ul, sizeof(pair)); + EXPECT_EQUAL(64ul, sizeof(Node)); } TEST("require that Value can store uncompressed data") { @@ -144,7 +144,7 @@ TEST("require that Value can store zstd compressed data") { TEST("require that Value can detect if output not equal to input") { Value v = createValue(S1, CompressionConfig::NONE); - static_cast<uint8_t *>(v.get())[8] ^= 0xff; + const_cast<uint8_t *>(static_cast<const uint8_t *>(v.get()))[8] ^= 0xff; EXPECT_FALSE(v.decompressed().second); } diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp index f92f5fd9581..c383358db9e 100644 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp @@ -626,7 +626,7 @@ TEST("test that the integrated visit cache works.") { for (size_t i(1); i <= 100; i++) { vcs.verifyRead(i); } - constexpr size_t BASE_SZ = 22174; + constexpr size_t BASE_SZ = 20594; TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 100, 100, BASE_SZ)); for (size_t i(1); i <= 100; i++) { vcs.verifyRead(i); @@ -648,16 +648,16 @@ TEST("test that the integrated visit cache works.") { vcs.verifyVisit({7,9,17,19,67,88,89}, true); TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 103, 99, BASE_SZ+180)); vcs.rewrite(17); - TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 103, 97, BASE_SZ-680)); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 103, 97, BASE_SZ-671)); vcs.verifyVisit({7,9,17,19,67,88,89}, true); TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 104, 98, BASE_SZ-20)); vcs.remove(17); - TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 104, 97, BASE_SZ-680)); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 104, 97, BASE_SZ-671)); vcs.verifyVisit({7,9,17,19,67,88,89}, {7,9,19,67,88,89}, true); - TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 105, 98, BASE_SZ-90)); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 105, 98, BASE_SZ-89)); vcs.verifyVisit({41, 42}, true); - TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 106, 99, BASE_SZ+210)); + TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 106, 99, BASE_SZ+215)); vcs.verifyVisit({43, 44}, true); TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 107, 100, BASE_SZ+520)); vcs.verifyVisit({41, 42, 43, 44}, true); diff --git a/searchlib/src/vespa/searchlib/attribute/posting_list_merger.cpp b/searchlib/src/vespa/searchlib/attribute/posting_list_merger.cpp index 39ca733eae7..b1ad8665441 100644 --- a/searchlib/src/vespa/searchlib/attribute/posting_list_merger.cpp +++ b/searchlib/src/vespa/searchlib/attribute/posting_list_merger.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "posting_list_merger.h" +#include <algorithm> namespace search::attribute { diff --git a/searchlib/src/vespa/searchlib/docstore/value.cpp b/searchlib/src/vespa/searchlib/docstore/value.cpp index 68bb060683d..ea29c894cba 100644 --- a/searchlib/src/vespa/searchlib/docstore/value.cpp +++ b/searchlib/src/vespa/searchlib/docstore/value.cpp @@ -12,29 +12,29 @@ namespace search::docstore { Value::Value() : _syncToken(0), + _uncompressedCrc(0), _compressedSize(0), _uncompressedSize(0), - _uncompressedCrc(0), + _buf(), _compression(CompressionConfig::NONE) {} Value::Value(uint64_t syncToken) : _syncToken(syncToken), + _uncompressedCrc(0), _compressedSize(0), _uncompressedSize(0), - _uncompressedCrc(0), + _buf(), _compression(CompressionConfig::NONE) {} -Value::Value(const Value &rhs) - : _syncToken(rhs._syncToken), - _compressedSize(rhs._compressedSize), - _uncompressedSize(rhs._uncompressedSize), - _uncompressedCrc(rhs._uncompressedCrc), - _compression(rhs._compression), - _buf(Alloc::alloc(rhs.size())) -{ - memcpy(get(), rhs.get(), size()); +Value::Value(const Value &rhs) = default; + +Value::~Value() = default; + +const void * +Value::get() const { + return _buf ? _buf->get() : nullptr; } void @@ -44,16 +44,18 @@ Value::set(vespalib::DataBuffer &&buf, ssize_t len) { void Value::set(vespalib::DataBuffer &&buf, ssize_t len, const CompressionConfig &compression) { + assert(len < std::numeric_limits<uint32_t>::max()); //Underlying buffer must be identical to allow swap. vespalib::DataBuffer compressed(buf.getData(), 0u); vespalib::ConstBufferRef input(buf.getData(), len); CompressionConfig::Type type = compress(compression, input, compressed, true); _compressedSize = compressed.getDataLen(); + if (buf.getData() == compressed.getData()) { // Uncompressed so we can just steal the underlying buffer. - buf.stealBuffer().swap(_buf); + _buf = std::make_shared<Alloc>(buf.stealBuffer()); } else { - compressed.stealBuffer().swap(_buf); + _buf = std::make_shared<Alloc>(compressed.stealBuffer()); } assert(((type == CompressionConfig::NONE) && (len == ssize_t(_compressedSize))) || diff --git a/searchlib/src/vespa/searchlib/docstore/value.h b/searchlib/src/vespa/searchlib/docstore/value.h index 426bcaf0e31..f58d96e3e77 100644 --- a/searchlib/src/vespa/searchlib/docstore/value.h +++ b/searchlib/src/vespa/searchlib/docstore/value.h @@ -25,6 +25,7 @@ public: Value &operator=(Value &&rhs) = default; Value(const Value &rhs); + ~Value(); uint64_t getSyncToken() const { return _syncToken; } CompressionConfig::Type getCompression() const { return _compression; } @@ -42,16 +43,15 @@ public: size_t size() const { return _compressedSize; } bool empty() const { return size() == 0; } - operator const void *() const { return _buf.get(); } - const void *get() const { return _buf.get(); } - void *get() { return _buf.get(); } + operator const void *() const { return get(); } + const void *get() const; private: - uint64_t _syncToken; - size_t _compressedSize; - size_t _uncompressedSize; - uint64_t _uncompressedCrc; + uint64_t _syncToken; + uint64_t _uncompressedCrc; + uint32_t _compressedSize; + uint32_t _uncompressedSize; + std::shared_ptr<Alloc> _buf; CompressionConfig::Type _compression; - Alloc _buf; }; } diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java index e2de3929d9b..6b9a1defed9 100644 --- a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java @@ -288,7 +288,7 @@ public class StandaloneContainerApplication implements Application { } private static void initializeContainer(DeployLogger deployLogger, Container container, Element spec) { - HostResource host = container.getRoot().getHostSystem().getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC); + HostResource host = container.getRoot().hostSystem().getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC); container.setBasePort(VespaDomBuilder.getXmlWantedPort(spec)); container.setHostResource(host); diff --git a/storage/src/tests/distributor/distributortest.cpp b/storage/src/tests/distributor/distributortest.cpp index d456401876e..83ef7891630 100644 --- a/storage/src/tests/distributor/distributortest.cpp +++ b/storage/src/tests/distributor/distributortest.cpp @@ -181,6 +181,12 @@ struct DistributorTest : Test, DistributorTestUtil { configureDistributor(builder); } + void configure_merge_operations_disabled(bool disabled) { + ConfigBuilder builder; + builder.mergeOperationsDisabled = disabled; + configureDistributor(builder); + } + void configureMaxClusterClockSkew(int seconds); void sendDownClusterStateCommand(); void replyToSingleRequestBucketInfoCommandWith1Bucket(); @@ -1021,6 +1027,17 @@ TEST_F(DistributorTest, fast_path_on_consistent_gets_config_is_propagated_to_int EXPECT_FALSE(getConfig().update_fast_path_restart_enabled()); } +TEST_F(DistributorTest, merge_disabling_config_is_propagated_to_internal_config) { + createLinks(true); + setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1"); + + configure_merge_operations_disabled(true); + EXPECT_TRUE(getConfig().merge_operations_disabled()); + + configure_merge_operations_disabled(false); + EXPECT_FALSE(getConfig().merge_operations_disabled()); +} + TEST_F(DistributorTest, concurrent_reads_not_enabled_if_btree_db_is_not_enabled) { createLinks(false); setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1"); diff --git a/storage/src/tests/distributor/statecheckerstest.cpp b/storage/src/tests/distributor/statecheckerstest.cpp index 01c4ad1cf6a..fa7afd39cf3 100644 --- a/storage/src/tests/distributor/statecheckerstest.cpp +++ b/storage/src/tests/distributor/statecheckerstest.cpp @@ -168,6 +168,7 @@ struct StateCheckersTest : Test, DistributorTestUtil { uint32_t _minSplitBits {0}; bool _includeMessagePriority {false}; bool _includeSchedulingPriority {false}; + bool _merge_operations_disabled {false}; CheckerParams(); ~CheckerParams(); @@ -203,6 +204,10 @@ struct StateCheckersTest : Test, DistributorTestUtil { _includeSchedulingPriority = includePri; return *this; } + CheckerParams& merge_operations_disabled(bool disabled) noexcept { + _merge_operations_disabled = disabled; + return *this; + } }; template <typename CheckerImpl> @@ -213,6 +218,7 @@ struct StateCheckersTest : Test, DistributorTestUtil { addNodesToBucketDB(bid, params._bucketInfo); setRedundancy(params._redundancy); enableDistributorClusterState(params._clusterState); + getConfig().set_merge_operations_disabled(params._merge_operations_disabled); if (!params._pending_cluster_state.empty()) { auto cmd = std::make_shared<api::SetSystemStateCommand>(lib::ClusterState(params._pending_cluster_state)); _distributor->onDown(cmd); @@ -817,6 +823,15 @@ TEST_F(StateCheckersTest, retired_nodes_out_of_sync_are_merged) { ".0.s:r .1.s:r .2.s:r .3.s:r")); } +TEST_F(StateCheckersTest, no_merge_operation_generated_if_merges_explicitly_config_disabled) { + runAndVerify<SynchronizeAndMoveStateChecker>( + CheckerParams() + .expect("NO OPERATIONS GENERATED") // Would normally generate a merge op + .bucketInfo("0=1,2=2") + .clusterState("distributor:1 storage:3") + .merge_operations_disabled(true)); +} + std::string StateCheckersTest::testDeleteExtraCopies( const std::string& bucketInfo, uint32_t redundancy, diff --git a/storage/src/vespa/storage/config/distributorconfiguration.cpp b/storage/src/vespa/storage/config/distributorconfiguration.cpp index 0b8564e561a..ed14e227fc1 100644 --- a/storage/src/vespa/storage/config/distributorconfiguration.cpp +++ b/storage/src/vespa/storage/config/distributorconfiguration.cpp @@ -40,6 +40,7 @@ DistributorConfiguration::DistributorConfiguration(StorageComponent& component) _sequenceMutatingOperations(true), _allowStaleReadsDuringClusterStateTransitions(false), _update_fast_path_restart_enabled(false), + _merge_operations_disabled(false), _minimumReplicaCountingMode(ReplicaCountingMode::TRUSTED) { } @@ -151,6 +152,7 @@ DistributorConfiguration::configure(const vespa::config::content::core::StorDist _sequenceMutatingOperations = config.sequenceMutatingOperations; _allowStaleReadsDuringClusterStateTransitions = config.allowStaleReadsDuringClusterStateTransitions; _update_fast_path_restart_enabled = config.restartWithFastUpdatePathIfAllGetTimestampsAreConsistent; + _merge_operations_disabled = config.mergeOperationsDisabled; _minimumReplicaCountingMode = config.minimumReplicaCountingMode; diff --git a/storage/src/vespa/storage/config/distributorconfiguration.h b/storage/src/vespa/storage/config/distributorconfiguration.h index 51ac7f8dae0..5b01e37992b 100644 --- a/storage/src/vespa/storage/config/distributorconfiguration.h +++ b/storage/src/vespa/storage/config/distributorconfiguration.h @@ -221,6 +221,13 @@ public: _update_fast_path_restart_enabled = enabled; } + bool merge_operations_disabled() const noexcept { + return _merge_operations_disabled; + } + void set_merge_operations_disabled(bool disabled) noexcept { + _merge_operations_disabled = disabled; + } + bool containsTimeStatement(const std::string& documentSelection) const; private: @@ -265,6 +272,7 @@ private: bool _sequenceMutatingOperations; bool _allowStaleReadsDuringClusterStateTransitions; bool _update_fast_path_restart_enabled; + bool _merge_operations_disabled; DistrConfig::MinimumReplicaCountingMode _minimumReplicaCountingMode; diff --git a/storage/src/vespa/storage/config/stor-distributormanager.def b/storage/src/vespa/storage/config/stor-distributormanager.def index e4a002ae81f..4587e7b3ebe 100644 --- a/storage/src/vespa/storage/config/stor-distributormanager.def +++ b/storage/src/vespa/storage/config/stor-distributormanager.def @@ -214,3 +214,8 @@ use_btree_database bool default=false restart ## Since all replicas of the document were in sync, applying the update in-place ## shall be considered safe. restart_with_fast_update_path_if_all_get_timestamps_are_consistent bool default=false + +## If set, no merge operations may be generated for any reason by a distributor. +## This is ONLY intended for system testing of certain transient edge cases and +## MUST NOT be set to true in a production environment. +merge_operations_disabled bool default=false diff --git a/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp index 6f82ceaab92..771bd90b247 100644 --- a/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp @@ -149,13 +149,13 @@ UpdateOperation::onReceive(DistributorMessageSender& sender, for (uint32_t i = 0; i < _results.size(); i++) { if (_results[i].oldTs < oldTs) { - LOG(warning, "Update operation for '%s' in bucket %s updated documents with different timestamps. " - "This should not happen and may indicate undetected replica divergence. " - "Found ts=%" PRIu64 " on node %u, ts=%" PRIu64 " on node %u", - reply.getDocumentId().toString().c_str(), - reply.getBucket().toString().c_str(), - _results[i].oldTs, _results[i].nodeId, - _results[goodNode].oldTs, _results[goodNode].nodeId); + LOG(error, "Update operation for '%s' in bucket %s updated documents with different timestamps. " + "This should not happen and may indicate undetected replica divergence. " + "Found ts=%" PRIu64 " on node %u, ts=%" PRIu64 " on node %u", + reply.getDocumentId().toString().c_str(), + reply.getBucket().toString().c_str(), + _results[i].oldTs, _results[i].nodeId, + _results[goodNode].oldTs, _results[goodNode].nodeId); _metrics.diverging_timestamp_updates.inc(); replyToSend.setNodeWithNewestTimestamp(_results[goodNode].nodeId); diff --git a/storage/src/vespa/storage/distributor/statecheckers.cpp b/storage/src/vespa/storage/distributor/statecheckers.cpp index b1fa3056cb1..2da4dd529c5 100644 --- a/storage/src/vespa/storage/distributor/statecheckers.cpp +++ b/storage/src/vespa/storage/distributor/statecheckers.cpp @@ -835,6 +835,9 @@ allCopiesAreInvalid(const StateChecker::Context& c) StateChecker::Result SynchronizeAndMoveStateChecker::check(StateChecker::Context& c) { + if (c.distributorConfig.merge_operations_disabled()) { + return Result::noMaintenanceNeeded(); + } if (isInconsistentlySplit(c)) { return Result::noMaintenanceNeeded(); } diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessorTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessorTest.java index 0c636ba804e..9753a180618 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessorTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessorTest.java @@ -11,14 +11,11 @@ import com.yahoo.vespa.http.client.core.EndpointResult; import org.junit.Test; import java.util.ArrayDeque; -import java.util.Optional; import java.util.Queue; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; @@ -79,46 +76,45 @@ public class OperationProcessorTest { //check a, b, c, d Result aggregated = queue.poll(); - assertThat(aggregated.getDocumentId(), equalTo("id:a:type::b")); - assertThat(aggregated.getDetails().size(), is(4)); - assertThat(aggregated.getDetails().get(0).getEndpoint().getHostname(), equalTo("a")); - assertThat(aggregated.getDetails().get(1).getEndpoint().getHostname(), equalTo("b")); - assertThat(aggregated.getDetails().get(2).getEndpoint().getHostname(), equalTo("c")); - assertThat(aggregated.getDetails().get(3).getEndpoint().getHostname(), equalTo("d")); - assertThat(aggregated.getDocumentDataAsCharSequence().toString(), is("data doc 1")); - - assertThat(queue.size(), is(0)); + assertEquals("id:a:type::b", aggregated.getDocumentId()); + assertEquals(4, aggregated.getDetails().size()); + assertEquals("a", aggregated.getDetails().get(0).getEndpoint().getHostname()); + assertEquals("b", aggregated.getDetails().get(1).getEndpoint().getHostname()); + assertEquals("c", aggregated.getDetails().get(2).getEndpoint().getHostname()); + assertEquals("d", aggregated.getDetails().get(3).getEndpoint().getHostname()); + assertEquals("data doc 1", aggregated.getDocumentDataAsCharSequence().toString()); + assertEquals(0, queue.size()); q.sendDocument(doc2); - assertThat(queue.size(), is(0)); + assertEquals(0, queue.size()); q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertThat(queue.size(), is(0)); + assertEquals(0, queue.size()); q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1); - assertThat(queue.size(), is(0)); + assertEquals(0, queue.size()); q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2); - assertThat(queue.size(), is(0)); + assertEquals(0, queue.size()); q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("d"))), 3); - assertThat(queue.size(), is(1)); + assertEquals(1, queue.size()); q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("e"))), 0); - assertThat(queue.size(), is(1)); + assertEquals(1, queue.size()); - //check a, b, c, d + // check a, b, c, d aggregated = queue.poll(); - assertThat(aggregated.getDocumentId(), equalTo("id:a:type::b2")); - assertThat(aggregated.getDetails().size(), is(4)); - assertThat(aggregated.getDetails().get(0).getEndpoint().getHostname(), equalTo("a")); - assertThat(aggregated.getDetails().get(1).getEndpoint().getHostname(), equalTo("b")); - assertThat(aggregated.getDetails().get(2).getEndpoint().getHostname(), equalTo("c")); - assertThat(aggregated.getDetails().get(3).getEndpoint().getHostname(), equalTo("d")); - assertThat(aggregated.getDocumentDataAsCharSequence().toString(), is("data doc 2")); - - assertThat(queue.size(), is(0)); + assertEquals("id:a:type::b2", aggregated.getDocumentId()); + assertEquals(4, aggregated.getDetails().size()); + assertEquals("a", aggregated.getDetails().get(0).getEndpoint().getHostname()); + assertEquals("b", aggregated.getDetails().get(1).getEndpoint().getHostname()); + assertEquals("c", aggregated.getDetails().get(2).getEndpoint().getHostname()); + assertEquals("d", aggregated.getDetails().get(3).getEndpoint().getHostname()); + assertEquals("data doc 2", aggregated.getDocumentDataAsCharSequence().toString()); + + assertEquals(0, queue.size()); } @Test @@ -136,27 +132,27 @@ public class OperationProcessorTest { operationProcessor.sendDocument(doc1); operationProcessor.sendDocument(doc1b); - assertThat(queue.size(), is(0)); + assertEquals(0, queue.size()); // Only one operations should be in flight. - assertThat(operationProcessor.getIncompleteResultQueueSize(), is(1)); + assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertThat(queue.size(), is(0)); - assertThat(operationProcessor.getIncompleteResultQueueSize(), is(1)); + assertEquals(0, queue.size()); + assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 1); - assertThat(queue.size(), is(1)); - assertThat(operationProcessor.getIncompleteResultQueueSize(), is(1)); + assertEquals(1, queue.size()); + assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertThat(queue.size(), is(1)); - assertThat(operationProcessor.getIncompleteResultQueueSize(), is(1)); + assertEquals(1, queue.size()); + assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 1); - assertThat(queue.size(), is(2)); - assertThat(operationProcessor.getIncompleteResultQueueSize(), is(0)); + assertEquals(2, queue.size()); + assertEquals(0, operationProcessor.getIncompleteResultQueueSize()); // This should have no effect. operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 1); operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 1); - assertThat(queue.size(), is(2)); + assertEquals(2, queue.size()); } @Test @@ -174,22 +170,22 @@ public class OperationProcessorTest { operationProcessor.sendDocument(doc1); operationProcessor.sendDocument(doc1b); - assertThat(queue.size(), is(0)); + assertEquals(0, queue.size()); // Only one operations should be in flight. - assertThat(operationProcessor.getIncompleteResultQueueSize(), is(1)); - assertThat(operationProcessor.oldestIncompleteResultId(), is(Optional.of(doc1.getOperationId()))); + assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); + assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertThat(queue.size(), is(1)); - assertThat(operationProcessor.getIncompleteResultQueueSize(), is(1)); - assertThat(operationProcessor.oldestIncompleteResultId(), is(Optional.of(doc1b.getOperationId()))); + assertEquals(1, queue.size()); + assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); + assertEquals(doc1b.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertThat(queue.size(), is(2)); - assertThat(operationProcessor.getIncompleteResultQueueSize(), is(0)); - assertThat(operationProcessor.oldestIncompleteResultId(), is(Optional.empty())); + assertEquals(2, queue.size()); + assertEquals(0, operationProcessor.getIncompleteResultQueueSize()); + assertFalse(operationProcessor.oldestIncompleteResultId().isPresent()); // This should have no effect. operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertThat(queue.size(), is(2)); + assertEquals(2, queue.size()); } @Test @@ -212,16 +208,16 @@ public class OperationProcessorTest { } for (int x = 0; x < 100; x++) { - assertThat(queue.size(), is(x)); + assertEquals(x, queue.size()); // Only one operations should be in flight. - assertThat(operationProcessor.getIncompleteResultQueueSize(), is(1)); + assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); Document document = documentQueue.poll(); operationProcessor.resultReceived(new EndpointResult(document.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertThat(queue.size(), is(x + 1)); + assertEquals(x+1, queue.size()); if (x < 99) { - assertThat(operationProcessor.getIncompleteResultQueueSize(), is(1)); + assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); } else { - assertThat(operationProcessor.getIncompleteResultQueueSize(), is(0)); + assertEquals(0, operationProcessor.getIncompleteResultQueueSize()); } } } @@ -244,26 +240,26 @@ public class OperationProcessorTest { operationProcessor.sendDocument(doc2); operationProcessor.sendDocument(doc3); - assertThat(queue.size(), is(0)); - assertThat(operationProcessor.getIncompleteResultQueueSize(), is(3)); - assertThat(operationProcessor.oldestIncompleteResultId(), is(Optional.of(doc1.getOperationId()))); + assertEquals(0, queue.size()); + assertEquals(3, operationProcessor.getIncompleteResultQueueSize()); + assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); // This should have no effect since it should not be sent. operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(endpoint)), 0); - assertThat(operationProcessor.getIncompleteResultQueueSize(), is(3)); - assertThat(operationProcessor.oldestIncompleteResultId(), is(Optional.of(doc1.getOperationId()))); + assertEquals(3, operationProcessor.getIncompleteResultQueueSize()); + assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); operationProcessor.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(endpoint)), 0); - assertThat(operationProcessor.getIncompleteResultQueueSize(), is(2)); - assertThat(operationProcessor.oldestIncompleteResultId(), is(Optional.of(doc1.getOperationId()))); + assertEquals(2, operationProcessor.getIncompleteResultQueueSize()); + assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); operationProcessor.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(endpoint)), 0); - assertThat(operationProcessor.getIncompleteResultQueueSize(), is(1)); - assertThat(operationProcessor.oldestIncompleteResultId(), is(Optional.of(doc1.getOperationId()))); + assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); + assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(endpoint)), 0); - assertThat(operationProcessor.getIncompleteResultQueueSize(), is(1)); - assertThat(operationProcessor.oldestIncompleteResultId(), is(Optional.of(doc1b.getOperationId()))); + assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); + assertEquals(doc1b.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(endpoint)), 0); - assertThat(operationProcessor.getIncompleteResultQueueSize(), is(0)); - assertThat(operationProcessor.oldestIncompleteResultId(), is(Optional.empty())); + assertEquals(0, operationProcessor.getIncompleteResultQueueSize()); + assertFalse(operationProcessor.oldestIncompleteResultId().isPresent()); } @Test @@ -280,16 +276,16 @@ public class OperationProcessorTest { sessionParams, null); q.sendDocument(doc1); - assertThat(queue.size(), is(0)); + assertEquals(0, queue.size()); q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertThat(queue.size(), is(0)); + assertEquals(0, queue.size()); q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("b"))), 0); - assertThat(queue.size(), is(0)); + assertEquals(0, queue.size()); q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("c"))), 0); - assertThat(queue.size(), is(0)); + assertEquals(0, queue.size()); } @Test @@ -306,52 +302,51 @@ public class OperationProcessorTest { sessionParams, null); q.sendDocument(doc1); - assertThat(queue.size(), is(0)); + assertEquals(0, queue.size()); q.sendDocument(doc2); - assertThat(queue.size(), is(0)); + assertEquals(0, queue.size()); q.sendDocument(doc3); - assertThat(queue.size(), is(0)); + assertEquals(0, queue.size()); q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertThat(queue.size(), is(0)); + assertEquals(0, queue.size()); q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertThat(queue.size(), is(0)); + assertEquals(0, queue.size()); q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1); - assertThat(queue.size(), is(0)); + assertEquals(0, queue.size()); q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2); - assertThat(queue.size(), is(1)); + assertEquals(1, queue.size()); q.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertThat(queue.size(), is(1)); + assertEquals(1, queue.size()); q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertThat(queue.size(), is(1)); + assertEquals(1, queue.size()); q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1); - assertThat(queue.size(), is(1)); + assertEquals(1, queue.size()); q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2); - assertThat(queue.size(), is(2)); + assertEquals(2, queue.size()); q.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2); - assertThat(queue.size(), is(2)); + assertEquals(2, queue.size()); q.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2); - assertThat(queue.size(), is(2)); + assertEquals(2, queue.size()); q.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1); - assertThat(queue.size(), is(3)); + assertEquals(3, queue.size()); q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1); - assertThat(queue.size(), is(3)); - assertThat(queue.remove().getDocumentDataAsCharSequence().toString(), is("data doc 1")); - assertThat(queue.remove().getDocumentDataAsCharSequence().toString(), is("data doc 2")); - assertThat(queue.remove().getDocumentDataAsCharSequence().toString(), is("data doc 3")); - + assertEquals(3, queue.size()); + assertEquals("data doc 1", queue.remove().getDocumentDataAsCharSequence().toString()); + assertEquals("data doc 2", queue.remove().getDocumentDataAsCharSequence().toString()); + assertEquals("data doc 3", queue.remove().getDocumentDataAsCharSequence().toString()); } @Test @@ -425,7 +420,7 @@ public class OperationProcessorTest { CountDownLatch countDownLatch = new CountDownLatch(3); - OperationProcessor operationProcessor = new OperationProcessor( + new OperationProcessor( new IncompleteResultsThrottler(19, 19, null, null), (docId, documentResult) -> { countDownLatch.countDown(); |