diff options
28 files changed, 350 insertions, 291 deletions
diff --git a/application/src/main/java/com/yahoo/application/content/ContentCluster.java b/application/src/main/java/com/yahoo/application/content/ContentCluster.java index 9fb025848e0..8679fa9273e 100644 --- a/application/src/main/java/com/yahoo/application/content/ContentCluster.java +++ b/application/src/main/java/com/yahoo/application/content/ContentCluster.java @@ -1,9 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.application.content; -import com.yahoo.config.model.ConfigModel; -import com.yahoo.vespa.model.builder.xml.dom.DomContentBuilder; - import java.nio.file.Path; import java.util.Collections; import java.util.List; diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/HostInfo.java b/config-model-api/src/main/java/com/yahoo/config/model/api/HostInfo.java index dbb2a1a07a9..657e2c3570a 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/HostInfo.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/HostInfo.java @@ -10,6 +10,7 @@ import java.util.Collection; * @since 5.37 */ public class HostInfo { + private final String hostname; private final Collection<ServiceInfo> services; @@ -45,4 +46,5 @@ public class HostInfo { result = 31 * result + (services != null ? services.hashCode() : 0); return result; } + } diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java index a36175accd0..d4b6751c356 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java @@ -118,7 +118,7 @@ public class ConfigModelRepo implements ConfigModelRepoAdder, Serializable, Iter String tagName = servicesElement.getTagName(); if (tagName.equals("config")) continue; // TODO: Remove on Vespa 6 if (tagName.equals("cluster")) continue; // TODO: Remove on Vespa 6 - if ((tagName.equals("clients")) && deployState.isHostedVespa()) + if ((tagName.equals("clients")) && deployState.isHosted()) throw new IllegalArgumentException("<" + tagName + "> is not allowed when running Vespa in a hosted environment"); String tagVersion = servicesElement.getAttribute("version"); @@ -236,7 +236,7 @@ public class ConfigModelRepo implements ConfigModelRepoAdder, Serializable, Iter // TODO: Doctoring on the XML is the wrong level for this. We should be able to mark a model as default instead -Jon private static Element getImplicitAdmin(DeployState deployState) throws IOException, SAXException { - String defaultAdminElement = deployState.isHostedVespa() ? getImplicitAdminV4() : getImplicitAdminV2(); + String defaultAdminElement = deployState.isHosted() ? getImplicitAdminV4() : getImplicitAdminV2(); log.log(LogLevel.DEBUG, "No <admin> defined, using " + defaultAdminElement); return XmlHelper.getDocumentBuilder().parse(new InputSource(new StringReader(defaultAdminElement))).getDocumentElement(); } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 5896dc59df2..8ca82692e98 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -241,7 +241,7 @@ public class DeployState implements ConfigDefinitionStore { public Optional<Model> getPreviousModel() { return previousModel; } - public boolean isHostedVespa() { + public boolean isHosted() { return properties.hostedVespa(); } diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java index 41927bc09a9..8e1097907f1 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java @@ -49,7 +49,7 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce return (parent != null) && (parent.getRoot() != null) && (parent.getRoot().getDeployState() != null) - && parent.getRoot().getDeployState().isHostedVespa(); + && parent.getRoot().getDeployState().isHosted(); } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Host.java b/config-model/src/main/java/com/yahoo/vespa/model/Host.java index 99109a881a1..78e0a472e13 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/Host.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/Host.java @@ -78,10 +78,7 @@ public final class Host extends AbstractConfigProducer<AbstractConfigProducer<?> return multitenant; } - /** - * Returns the string representation of this Host object. - * @return The string representation of this Host object. - */ + /** Returns the string representation of this Host object. */ public String toString() { return "host '" + getHostName() + "'"; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java index 782487ee12c..002a649c9a9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java @@ -24,7 +24,7 @@ public class HostResource implements Comparable<HostResource> { private final Host host; // Map from "sentinel name" to service - private final Map<String,Service> services = new LinkedHashMap<>(); + private final Map<String, Service> services = new LinkedHashMap<>(); private final Map<Integer, Service> portDB = new LinkedHashMap<>(); private int allocatedPorts = 0; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaConfigModelRegistry.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaConfigModelRegistry.java index 6b9aa3fe8c5..ff13a1321b4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaConfigModelRegistry.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaConfigModelRegistry.java @@ -8,6 +8,7 @@ import com.yahoo.config.model.builder.xml.ConfigModelId; import com.yahoo.vespa.model.builder.xml.dom.*; import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking; +import com.yahoo.vespa.model.content.Content; import com.yahoo.vespa.model.generic.GenericServicesBuilder; import java.util.ArrayList; @@ -32,7 +33,7 @@ public class VespaConfigModelRegistry extends ConfigModelRegistry { builderList.add(new AdminModel.BuilderV4()); builderList.add(new DomRoutingBuilder()); builderList.add(new DomClientsBuilder()); - builderList.add(new DomContentBuilder()); + builderList.add(new Content.Builder()); builderList.add(new ContainerModelBuilder(false, Networking.enable)); builderList.add(new GenericServicesBuilder()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index 1290b0b22d6..e39bddd5594 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -31,7 +31,7 @@ public class ClusterControllerContainer extends Container implements BundlesConf private final Set<String> bundles = new TreeSet<>(); public ClusterControllerContainer(AbstractConfigProducer parent, int index, boolean runStandaloneZooKeeper) { - super(parent, "" + index); + super(parent, "" + index, index); this.index = index; addHandler( new Handler(new ComponentModel(new BundleInstantiationSpecification( diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomContainerClusterBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomContainerClusterBuilder.java index fe339835c78..bfbff219653 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomContainerClusterBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomContainerClusterBuilder.java @@ -106,4 +106,5 @@ public abstract class DomContainerClusterBuilder<CLUSTER extends ContainerCluste protected void buildAndAddProcessingRenderers(ContainerCluster cluster, Element spec) { ContainerModelBuilder.addConfiguredComponents(cluster, spec, "renderer"); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomContentBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomContentBuilder.java deleted file mode 100644 index aecd0f9efb5..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomContentBuilder.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.builder.xml.dom; - -import com.yahoo.config.model.ConfigModelContext; -import com.yahoo.config.model.builder.xml.ConfigModelBuilder; -import com.yahoo.config.model.builder.xml.ConfigModelId; -import com.yahoo.vespa.model.admin.Admin; -import com.yahoo.vespa.model.content.Content; -import com.yahoo.vespa.model.content.cluster.ContentCluster; - -import org.w3c.dom.Element; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -/** - * @author baldersheim - */ -public class DomContentBuilder extends ConfigModelBuilder<Content> { - - public static final List<ConfigModelId> configModelIds = Collections.singletonList(ConfigModelId.fromName("content")); - - public DomContentBuilder() { - super(Content.class); - } - - @Override - public List<ConfigModelId> handlesElements() { - return configModelIds; - } - - @Override - public void doBuild(Content content, Element xml, ConfigModelContext modelContext) { - Admin admin = content.adminModel() != null ? content.adminModel().getAdmin() : null; // This is null in tests only - ContentCluster cluster = new ContentCluster.Builder(admin, modelContext.getDeployLogger()).build(modelContext.getParentProducer(), xml); - content.setCluster(cluster, modelContext); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java index 65519637bd5..673d72c9c2f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java @@ -444,14 +444,13 @@ public class DomV20ClientsBuilder { } @Override - protected ContainerHttpGateway doBuild(AbstractConfigProducer parent, - Element spec) { + protected ContainerHttpGateway doBuild(AbstractConfigProducer parent, Element spec) { // TODO: remove port handling int port = 19020; if (spec != null && spec.hasAttribute("baseport")) { port = Integer.parseInt(spec.getAttribute("baseport")); } - ContainerHttpGateway httpGateway = new ContainerHttpGateway(cluster, "" + index, port); + ContainerHttpGateway httpGateway = new ContainerHttpGateway(cluster, "" + index, port, index); List<Container> containers = new ArrayList<>(); containers.add(httpGateway); @@ -463,7 +462,7 @@ public class DomV20ClientsBuilder { /** * This class parses the feederoptions xml tag and produces Vespa config output. * - * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a> + * @author Gunnar Gauslaa Bergem */ private class FeederOptionsParser implements Serializable { private static final long serialVersionUID = 1L; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index a27739b42ef..84a9b5f7e88 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -17,7 +17,6 @@ import java.util.Optional; * * @author bratseth */ - // TODO: Use this for all nodes tags and unify with NodesUtil public class NodesSpecification { private final boolean dedicated; @@ -101,5 +100,11 @@ public class NodesSpecification { return hostSystem.allocateHosts(cluster, Capacity.fromNodeCount(count, flavor), groups, logger); } + @Override + public String toString() { + return "specification of " + count + (dedicated ? " dedicated " : " ") + "nodes" + + (flavor.isPresent() ? " of flavor " + flavor.get() : "") + + (groups > 1 ? " in " + groups + " groups" : ""); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index a13c7c9cec4..f1d274bc80c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -72,7 +72,10 @@ public class Container extends AbstractService implements private boolean httpServerEnabled = true; private boolean messageBusEnabled = true; + /** Whether this node has been marked as retired (e.g, will be removed) */ private final boolean retired; + /** The index of this node. Non-critical: This is persisted on hosted, just a counter otherwise. */ + private final int index; private final ComponentGroup<Handler<?>> handlers = new ComponentGroup<>(this, "handler"); private final ComponentGroup<Component<?, ?>> components = new ComponentGroup(this, "components"); @@ -85,21 +88,22 @@ public class Container extends AbstractService implements private final int numRpcServerPorts = 2; private static String defaultHostedJVMArgs = "-XX:+UseOSErrorReporting -XX:+SuppressFatalErrorMessage"; - public Container(AbstractConfigProducer parent, String name) { - this(parent, name, Collections.<PortOverride>emptyList()); + public Container(AbstractConfigProducer parent, String name, int index) { + this(parent, name, Collections.<PortOverride>emptyList(), index); } - public Container(AbstractConfigProducer parent, String name, boolean retired) { - this(parent, name, retired, Collections.<PortOverride>emptyList()); + public Container(AbstractConfigProducer parent, String name, boolean retired, int index) { + this(parent, name, retired, Collections.<PortOverride>emptyList(), index); } - public Container(AbstractConfigProducer parent, String name, List<PortOverride> portOverrides) { - this(parent, name, false, portOverrides); + public Container(AbstractConfigProducer parent, String name, List<PortOverride> portOverrides, int index) { + this(parent, name, false, portOverrides, index); } - public Container(AbstractConfigProducer parent, String name, boolean retired, List<PortOverride> portOverrides) { + public Container(AbstractConfigProducer parent, String name, boolean retired, List<PortOverride> portOverrides, int index) { super(parent, name); this.name = name; this.parent = parent; this.portOverrides = Collections.unmodifiableList(new ArrayList<>(portOverrides)); this.retired = retired; + this.index = index; if (getHttp() == null) { numHttpServerPorts = 2; @@ -143,6 +147,9 @@ public class Container extends AbstractService implements public JettyHttpServer getDefaultHttpServer() { return defaultHttpServer; } + + /** Returns the index of this node. The index of a given node is stable through changes with best effort. */ + public int index() { return index; } // We cannot set bindings yet, as baseport is not initialized public void addBuiltinHandlers() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java index d3ccefc3e26..bce1d28a863 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java @@ -72,7 +72,7 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> throw new IllegalArgumentException(String.format("Invalid port %d.", port)); int legalPortInHostedVespa = Container.BASEPORT; - if (deployState.isHostedVespa() && port != legalPortInHostedVespa) { + if (deployState.isHosted() && port != legalPortInHostedVespa) { deployState.getDeployLogger().log(LogLevel.WARNING, String.format("Trying to set port to %d for http server with id %s. You cannot set port to anything else than %s", port, spec.getAttribute("id"), legalPortInHostedVespa)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerHttpGateway.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerHttpGateway.java index 88d4a0c8599..06aed4b3caa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerHttpGateway.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerHttpGateway.java @@ -9,8 +9,8 @@ import com.yahoo.vespa.model.container.ContainerCluster; */ public class ContainerHttpGateway extends Container { - public ContainerHttpGateway(ContainerCluster parent, String name, int wantedPort) { - super(parent, name); + public ContainerHttpGateway(ContainerCluster parent, String name, int wantedPort, int index) { + super(parent, name, index); // TODO: when this class is removed, all ports for the gateway will map to standard container ports // this is just a tjuvtriks to keep the old gateway port allocation for now. @@ -19,4 +19,5 @@ public class ContainerHttpGateway extends Container { @Override public String getServiceType() { return "container-httpgateway"; } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index a8055afeea8..4240ea40ebb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -188,7 +188,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } protected void addStatusHandlers(ContainerCluster cluster, ConfigModelContext configModelContext) { - if (configModelContext.getDeployState().isHostedVespa()) { + if (configModelContext.getDeployState().isHosted()) { String name = "status.html"; Optional<String> statusFile = Optional.ofNullable(System.getenv(HOSTED_VESPA_STATUS_FILE_YINST_SETTING)); cluster.addComponent( @@ -368,14 +368,14 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } private void addStandaloneNode(ContainerCluster cluster) { - Container container = new Container(cluster, "standalone"); + Container container = new Container(cluster, "standalone", cluster.getContainers().size()); cluster.addContainers(Collections.singleton(container)); } private void addNodesFromXml(ContainerCluster cluster, Element spec) { Element nodesElement = XML.getChild(spec, "nodes"); if (nodesElement == null) { // default single node on localhost - Container container = new Container(cluster, "container.0"); + Container container = new Container(cluster, "container.0", 0); HostResource host = allocateSingleNodeHost(cluster, log); container.setHostResource(host); if ( ! container.isInitialized() ) // TODO: Fold this into initService @@ -434,7 +434,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Element spec, Element nodesElement, List<Container> result) { int nodeCount = 0; for (Element nodeElem: XML.getChildren(nodesElement, "node")) { - Container container = new ContainerServiceBuilder("container." + nodeCount).build(cluster, nodeElem); + Container container = new ContainerServiceBuilder("container." + nodeCount, nodeCount).build(cluster, nodeElem); result.add(container); ++nodeCount; } @@ -447,7 +447,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Map<HostResource, ClusterMembership> hosts = nodesSpecification.provision(cluster.getRoot().getHostSystem(), ClusterSpec.Type.container, ClusterSpec.Id.from(cluster.getName()), Optional.empty(), log); for (Map.Entry<HostResource, ClusterMembership> entry : hosts.entrySet()) { String id = "container." + entry.getValue().index(); - Container container = new Container(cluster, id, entry.getValue().retired()); + Container container = new Container(cluster, id, entry.getValue().retired(), entry.getValue().index()); container.setHostResource(entry.getKey()); container.initService(); result.add(container); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java index 785ab1f7504..20a5c09b258 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java @@ -20,14 +20,16 @@ import java.util.logging.Logger; public class ContainerServiceBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Container> { private final String id; + private final int index; - public ContainerServiceBuilder(String id) { + public ContainerServiceBuilder(String id, int index) { this.id = id; + this.index = index; } @Override protected Container doBuild(AbstractConfigProducer parent, Element nodeElem) { - return new Container(parent, id, readServerPortOverrides(nodeElem)); + return new Container(parent, id, readServerPortOverrides(nodeElem), index); } private List<Container.PortOverride> readServerPortOverrides(Element spec) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index 4bb0144ed0c..bb734efcad1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -11,12 +11,12 @@ import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.ConfigModelRepo; import com.yahoo.config.model.ConfigModelRepoAdder; import com.yahoo.config.model.admin.AdminModel; +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.builder.xml.ConfigModelId; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.log.LogLevel; import com.yahoo.vespa.model.*; -import com.yahoo.vespa.model.builder.VespaModelBuilder; -import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; @@ -29,6 +29,7 @@ import com.yahoo.vespa.model.search.AbstractSearchCluster; import com.yahoo.vespa.model.search.IndexedSearchCluster; import com.yahoo.vespa.model.search.IndexingDocprocChain; import com.yahoo.vespa.model.search.SearchNode; +import org.w3c.dom.Element; import java.util.*; import java.util.logging.Logger; @@ -45,32 +46,23 @@ public class Content extends ConfigModel { private ContentCluster cluster; private Optional<ContainerCluster> ownedIndexingCluster = Optional.empty(); - private final boolean hostedVespa; + private final boolean isHosted; // Dependencies to other models private final AdminModel adminModel; - private final Collection<ContainerModel> containers; // to find or add the docproc container + + // to find or add the docproc container and supplement cluster controllers with clusters having less then 3 nodes + private final Collection<ContainerModel> containers; @SuppressWarnings({ "UnusedDeclaration"}) // Created by reflection in ConfigModelRepo public Content(ConfigModelContext modelContext, AdminModel adminModel, Collection<ContainerModel> containers) { super(modelContext); modelContext.getParentProducer().getRoot(); - hostedVespa = modelContext.getDeployState().isHostedVespa(); + isHosted = modelContext.getDeployState().isHosted(); this.adminModel = adminModel; this.containers = containers; } - /** Returns the admin model of this system */ - public AdminModel adminModel() { return adminModel; } - - /** Called by DomContentBuilder during build */ - public void setCluster(ContentCluster cluster, ConfigModelContext configModelContext) { - this.cluster = cluster; - initializeIndexingClusters(containers, - configModelContext.getConfigModelRepoAdder(), - (ApplicationConfigProducerRoot)configModelContext.getParentProducer()); - } - public ContentCluster getCluster() { return cluster; } /** @@ -81,20 +73,14 @@ public class Content extends ConfigModel { public void createTlds(ConfigModelRepo modelRepo) { IndexedSearchCluster indexedCluster = cluster.getSearch().getIndexed(); - if (indexedCluster == null) { - return; - } + if (indexedCluster == null) return; SimpleConfigProducer tldParent = new SimpleConfigProducer(indexedCluster, "tlds"); for (ConfigModel model : modelRepo.asMap().values()) { - if (!(model instanceof ContainerModel)) { - continue; - } + if ( ! (model instanceof ContainerModel)) continue; ContainerCluster containerCluster = ((ContainerModel) model).getCluster(); - if (containerCluster.getSearch() == null) { - continue; // this is not a qrs cluster - } + if (containerCluster.getSearch() == null) continue; // this is not a qrs cluster log.log(LogLevel.DEBUG, "Adding tlds for indexed cluster " + indexedCluster.getClusterName() + ", container cluster " + containerCluster.getName()); indexedCluster.addTldsWithSameIdsAsContainers(tldParent, containerCluster); @@ -102,92 +88,13 @@ public class Content extends ConfigModel { indexedCluster.setupDispatchGroups(); } - /** Select/creates and initializes the indexing cluster coupled to this */ - private void initializeIndexingClusters(Collection<ContainerModel> containers, - ConfigModelRepoAdder configModelRepoAdder, - ApplicationConfigProducerRoot root) { - if (getCluster().getSearch().hasIndexedCluster()) - initializeOrSetExistingIndexingCluster(getCluster().getSearch().getIndexed(), hostedVespa, - containers, configModelRepoAdder, root); - } - - private void initializeOrSetExistingIndexingCluster(IndexedSearchCluster indexedSearchCluster, - boolean isHostedVespa, - Collection<ContainerModel> containers, - ConfigModelRepoAdder configModelRepoAdder, - ApplicationConfigProducerRoot root) { - if (indexedSearchCluster.hasExplicitIndexingCluster()) { - setExistingIndexingCluster(indexedSearchCluster, containers); - } else if (isHostedVespa) { - setContainerAsIndexingCluster(indexedSearchCluster, containers, configModelRepoAdder, root); - } else { - createImplicitIndexingCluster(indexedSearchCluster, configModelRepoAdder, root); - } - } - - private void setContainerAsIndexingCluster(IndexedSearchCluster indexedSearchCluster, - Collection<ContainerModel> containers, - ConfigModelRepoAdder configModelRepoAdder, - ApplicationConfigProducerRoot root) { - if (containers.isEmpty()) { - createImplicitIndexingCluster(indexedSearchCluster, configModelRepoAdder, root); - } else { - ContainerCluster targetCluster = getContainerWithDocproc(containers); - if (targetCluster == null) - targetCluster = getContainerWithSearch(containers); - if (targetCluster == null) - targetCluster = containers.iterator().next().getCluster(); - - addDocproc(targetCluster); - indexedSearchCluster.setIndexingClusterName(targetCluster.getName()); - addIndexingChainsTo(targetCluster, indexedSearchCluster); - } - } - - private void setExistingIndexingCluster(IndexedSearchCluster cluster, Collection<ContainerModel> containers) { - String indexingClusterName = cluster.getIndexingClusterName(); - ContainerModel containerModel = findByName(indexingClusterName, containers); - if (containerModel == null) - throw new RuntimeException("Content cluster '" + cluster.getClusterName() + "' refers to docproc " + - "cluster '" + indexingClusterName + "', but this cluster does not exist."); - addIndexingChainsTo(containerModel.getCluster(), cluster); - } - - private ContainerModel findByName(String name, Collection<ContainerModel> containers) { - for (ContainerModel container : containers) - if (container.getId().equals(name)) - return container; - return null; - } - - private void addIndexingChainsTo(ContainerCluster indexer, IndexedSearchCluster cluster) { - addIndexingChain(indexer); - DocprocChain indexingChain; - ComponentRegistry<DocprocChain> allChains = indexer.getDocprocChains().allChains(); - if (cluster.hasExplicitIndexingChain()) { - indexingChain = allChains.getComponent(cluster.getIndexingChainName()); - if (indexingChain == null) { - throw new RuntimeException("Indexing cluster " + cluster.getClusterName() + " refers to docproc " + - "chain " + cluster.getIndexingChainName() + " for indexing, which does not exist."); - } else { - checkThatExplicitIndexingChainInheritsCorrectly(allChains, indexingChain.getChainSpecification()); - } - } else { - indexingChain = allChains.getComponent(IndexingDocprocChain.NAME); - } - - cluster.setIndexingChain(indexingChain); - } - - private static boolean checkParentChain(ComponentRegistry<DocprocChain> allChains, ChainSpecification chainSpec) { - if (IndexingDocprocChain.NAME.equals(chainSpec.componentId.stringValue())) { - return true; - } + private static boolean containsIndexingChain(ComponentRegistry<DocprocChain> allChains, ChainSpecification chainSpec) { + if (IndexingDocprocChain.NAME.equals(chainSpec.componentId.stringValue())) return true; ChainSpecification.Inheritance inheritance = chainSpec.inheritance; for (ComponentSpecification parentComponentSpec : inheritance.chainSpecifications) { ChainSpecification parentSpec = getChainSpec(allChains, parentComponentSpec); - checkParentChain(allChains, parentSpec); + if (containsIndexingChain(allChains, parentSpec)) return true; } return false; @@ -195,9 +102,8 @@ public class Content extends ConfigModel { private static ChainSpecification getChainSpec(ComponentRegistry<DocprocChain> allChains, ComponentSpecification componentSpec) { DocprocChain docprocChain = allChains.getComponent(componentSpec); - if (docprocChain == null) { - throw new IllegalArgumentException("Chain '" + componentSpec + "' not found."); - } + if (docprocChain == null) throw new IllegalArgumentException("Chain '" + componentSpec + "' not found."); + return docprocChain.getChainSpecification(); } @@ -205,73 +111,14 @@ public class Content extends ConfigModel { DocprocChain chainAlreadyPresent = containerCluster.getDocprocChains().allChains(). getComponent(new ComponentId(IndexingDocprocChain.NAME)); if (chainAlreadyPresent != null) { - if (chainAlreadyPresent instanceof IndexingDocprocChain) { - return; - } else { - throw new IllegalArgumentException("A docproc chain may not have the ID '" + - IndexingDocprocChain.NAME + ", since this is reserved by Vespa. Please use a different ID."); - } + if (chainAlreadyPresent instanceof IndexingDocprocChain) return; + throw new IllegalArgumentException("A docproc chain may not have the ID '" + + IndexingDocprocChain.NAME + ", since this is reserved by Vespa. Please use a different ID."); } containerCluster.getDocprocChains().add(new IndexingDocprocChain()); } - /** Create a new container cluster for indexing and add it to the Vespa model */ - private void createImplicitIndexingCluster(IndexedSearchCluster cluster, - ConfigModelRepoAdder configModelRepoAdder, - ApplicationConfigProducerRoot root) { - String indexerName = cluster.getIndexingClusterName(); - AbstractConfigProducer p = root.getChildren().get(ContainerModel.DOCPROC_RESERVED_NAME); - if (p == null) - p = new SimpleConfigProducer(root, ContainerModel.DOCPROC_RESERVED_NAME); - ConfigModelContext context = ConfigModelContext.createFromParentAndId(configModelRepoAdder, p, ContainerModel.DOCPROC_RESERVED_NAME); - ContainerCluster indexingCluster = new ContainerCluster(context.getParentProducer(), "cluster." + indexerName, indexerName); - ContainerModel indexingClusterModel = new ContainerModel(ConfigModelContext.createFromParentAndId(configModelRepoAdder, p, indexingCluster.getSubId())); - indexingClusterModel.setCluster(indexingCluster); - configModelRepoAdder.add(indexingClusterModel); - ownedIndexingCluster = Optional.of(indexingCluster); - - ContainerModelBuilder.addDefaultHandler_legacyBuilder(indexingCluster); - - addDocproc(indexingCluster); - - List<Container> nodes = new ArrayList<>(); - int index = 0; - Set<HostResource> processedHosts = new LinkedHashSet<>(); - boolean isElastic = cluster.isElastic(); - for (SearchNode searchNode : cluster.getSearchNodes()) { - HostResource host = searchNode.getHostResource(); - if (!processedHosts.contains(host)) { - String containerName = String.valueOf(isElastic ? searchNode.getDistributionKey() : index++); - Container docprocService = new Container(indexingCluster, containerName); - docprocService.setBasePort(host.nextAvailableBaseport(docprocService.getPortCount())); - docprocService.setHostResource(host); - docprocService.initService(); - nodes.add(docprocService); - processedHosts.add(host); - } - } - indexingCluster.addContainers(nodes); - - addIndexingChain(indexingCluster); - cluster.setIndexingChain(indexingCluster.getDocprocChains().allChains().getComponent(IndexingDocprocChain.NAME)); - } - - private void addDocproc(ContainerCluster cluster) { - if (cluster.getDocproc() == null) { - DocprocChains chains = new DocprocChains(cluster, "docprocchains"); - ContainerDocproc containerDocproc = new ContainerDocproc(cluster, chains); - cluster.setDocproc(containerDocproc); - } - } - - private ContainerCluster getContainerWithDocproc(Collection<ContainerModel> containers) { - for (ContainerModel container : containers) - if (container.getCluster().getDocproc() != null) - return container.getCluster(); - return null; - } - private static ContainerCluster getContainerWithSearch(Collection<ContainerModel> containers) { for (ContainerModel container : containers) if (container.getCluster().getSearch() != null) @@ -281,48 +128,32 @@ public class Content extends ConfigModel { private static void checkThatExplicitIndexingChainInheritsCorrectly(ComponentRegistry<DocprocChain> allChains, ChainSpecification chainSpec) { ChainSpecification.Inheritance inheritance = chainSpec.inheritance; - boolean found = false; for (ComponentSpecification componentSpec : inheritance.chainSpecifications) { ChainSpecification parentSpec = getChainSpec(allChains, componentSpec); - found = checkParentChain(allChains, parentSpec); - if (found) { - break; - } - } - if (!found) { - throw new IllegalArgumentException("Docproc chain '" + chainSpec.componentId + "' does not inherit from 'indexing' chain."); + if (containsIndexingChain(allChains, parentSpec)) return; } + throw new IllegalArgumentException("Docproc chain '" + chainSpec.componentId + "' does not inherit from 'indexing' chain."); } public static List<Content> getContent(ConfigModelRepo pc) { List<Content> contents = new ArrayList<>(); - - for (ConfigModel model : pc.asMap().values()) { - if (model instanceof Content) { + for (ConfigModel model : pc.asMap().values()) + if (model instanceof Content) contents.add((Content)model); - } - } - return contents; } public static List<AbstractSearchCluster> getSearchClusters(ConfigModelRepo pc) { List<AbstractSearchCluster> clusters = new ArrayList<>(); - - for (ContentCluster c : getContentClusters(pc)) { + for (ContentCluster c : getContentClusters(pc)) clusters.addAll(c.getSearch().getClusters().values()); - } - return clusters; } public static List<ContentCluster> getContentClusters(ConfigModelRepo pc) { List<ContentCluster> clusters = new ArrayList<>(); - - for (Content c : getContent(pc)) { + for (Content c : getContent(pc)) clusters.add(c.getCluster()); - } - return clusters; } @@ -343,5 +174,158 @@ public class Content extends ConfigModel { // Currently only distribute affinity for search nodes AbstractService.distributeCpuSocketAffinity(cluster.getSearch().getSearchNodes()); } + + public static class Builder extends ConfigModelBuilder<Content> { + + public static final List<ConfigModelId> configModelIds = Collections.singletonList(ConfigModelId.fromName("content")); + + public Builder() { + super(Content.class); + } + + @Override + public List<ConfigModelId> handlesElements() { + return configModelIds; + } + + @Override + public void doBuild(Content content, Element xml, ConfigModelContext modelContext) { + Admin admin = content.adminModel != null ? content.adminModel.getAdmin() : null; // This is null in tests only + content.cluster = new ContentCluster.Builder(admin, modelContext.getDeployLogger()).build(content.containers, modelContext.getParentProducer(), xml); + buildIndexingClusters(content, + modelContext.getConfigModelRepoAdder(), + (ApplicationConfigProducerRoot)modelContext.getParentProducer()); + } + + /** Select/creates and initializes the indexing cluster coupled to this */ + private void buildIndexingClusters(Content content, + ConfigModelRepoAdder configModelRepoAdder, + ApplicationConfigProducerRoot root) { + if ( ! content.getCluster().getSearch().hasIndexedCluster()) return; + + IndexedSearchCluster indexedSearchCluster = content.getCluster().getSearch().getIndexed(); + if (indexedSearchCluster.hasExplicitIndexingCluster()) { + setExistingIndexingCluster(indexedSearchCluster, content.containers); + } else if (content.isHosted) { + setContainerAsIndexingCluster(indexedSearchCluster, content, configModelRepoAdder, root); + } else { + createImplicitIndexingCluster(indexedSearchCluster, content, configModelRepoAdder, root); + } + } + + private void setContainerAsIndexingCluster(IndexedSearchCluster indexedSearchCluster, + Content content, + ConfigModelRepoAdder configModelRepoAdder, + ApplicationConfigProducerRoot root) { + if (content.containers.isEmpty()) { + createImplicitIndexingCluster(indexedSearchCluster, content, configModelRepoAdder, root); + } else { + ContainerCluster targetCluster = getContainerWithDocproc(content.containers); + if (targetCluster == null) + targetCluster = getContainerWithSearch(content.containers); + if (targetCluster == null) + targetCluster = content.containers.iterator().next().getCluster(); + + addDocproc(targetCluster); + indexedSearchCluster.setIndexingClusterName(targetCluster.getName()); + addIndexingChainsTo(targetCluster, indexedSearchCluster); + } + } + + private void setExistingIndexingCluster(IndexedSearchCluster cluster, Collection<ContainerModel> containers) { + String indexingClusterName = cluster.getIndexingClusterName(); + ContainerModel containerModel = findByName(indexingClusterName, containers); + if (containerModel == null) + throw new RuntimeException("Content cluster '" + cluster.getClusterName() + "' refers to docproc " + + "cluster '" + indexingClusterName + "', but this cluster does not exist."); + addIndexingChainsTo(containerModel.getCluster(), cluster); + } + + private ContainerModel findByName(String name, Collection<ContainerModel> containers) { + for (ContainerModel container : containers) + if (container.getId().equals(name)) + return container; + return null; + } + + private void addIndexingChainsTo(ContainerCluster indexer, IndexedSearchCluster cluster) { + addIndexingChain(indexer); + DocprocChain indexingChain; + ComponentRegistry<DocprocChain> allChains = indexer.getDocprocChains().allChains(); + if (cluster.hasExplicitIndexingChain()) { + indexingChain = allChains.getComponent(cluster.getIndexingChainName()); + if (indexingChain == null) { + throw new RuntimeException("Indexing cluster " + cluster.getClusterName() + " refers to docproc " + + "chain " + cluster.getIndexingChainName() + " for indexing, which does not exist."); + } else { + checkThatExplicitIndexingChainInheritsCorrectly(allChains, indexingChain.getChainSpecification()); + } + } else { + indexingChain = allChains.getComponent(IndexingDocprocChain.NAME); + } + + cluster.setIndexingChain(indexingChain); + } + + /** Create a new container cluster for indexing and add it to the Vespa model */ + private void createImplicitIndexingCluster(IndexedSearchCluster cluster, + Content content, + ConfigModelRepoAdder configModelRepoAdder, + ApplicationConfigProducerRoot root) { + String indexerName = cluster.getIndexingClusterName(); + AbstractConfigProducer p = root.getChildren().get(ContainerModel.DOCPROC_RESERVED_NAME); + if (p == null) + p = new SimpleConfigProducer(root, ContainerModel.DOCPROC_RESERVED_NAME); + ConfigModelContext context = ConfigModelContext.createFromParentAndId(configModelRepoAdder, p, ContainerModel.DOCPROC_RESERVED_NAME); + ContainerCluster indexingCluster = new ContainerCluster(context.getParentProducer(), "cluster." + indexerName, indexerName); + ContainerModel indexingClusterModel = new ContainerModel(ConfigModelContext.createFromParentAndId(configModelRepoAdder, p, indexingCluster.getSubId())); + indexingClusterModel.setCluster(indexingCluster); + configModelRepoAdder.add(indexingClusterModel); + content.ownedIndexingCluster = Optional.of(indexingCluster); + + ContainerModelBuilder.addDefaultHandler_legacyBuilder(indexingCluster); + + addDocproc(indexingCluster); + + List<Container> nodes = new ArrayList<>(); + int index = 0; + Set<HostResource> processedHosts = new LinkedHashSet<>(); + boolean isElastic = cluster.isElastic(); + for (SearchNode searchNode : cluster.getSearchNodes()) { + HostResource host = searchNode.getHostResource(); + if (!processedHosts.contains(host)) { + String containerName = String.valueOf(isElastic ? searchNode.getDistributionKey() : index); + Container docprocService = new Container(indexingCluster, containerName, index); + index++; + docprocService.setBasePort(host.nextAvailableBaseport(docprocService.getPortCount())); + docprocService.setHostResource(host); + docprocService.initService(); + nodes.add(docprocService); + processedHosts.add(host); + } + } + indexingCluster.addContainers(nodes); + + addIndexingChain(indexingCluster); + cluster.setIndexingChain(indexingCluster.getDocprocChains().allChains().getComponent(IndexingDocprocChain.NAME)); + } + + private ContainerCluster getContainerWithDocproc(Collection<ContainerModel> containers) { + for (ContainerModel container : containers) + if (container.getCluster().getDocproc() != null) + return container.getCluster(); + return null; + } + + private void addDocproc(ContainerCluster cluster) { + if (cluster.getDocproc() == null) { + DocprocChains chains = new DocprocChains(cluster, "docprocchains"); + ContainerDocproc containerDocproc = new ContainerDocproc(cluster, chains); + cluster.setDocproc(containerDocproc); + } + } + + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 08685659d35..f29cc07ee93 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -28,6 +28,7 @@ import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.builder.xml.dom.NodesSpecification; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.ContainerModel; import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; import com.yahoo.vespa.model.content.*; import com.yahoo.vespa.model.content.engines.PersistenceEngine; @@ -89,8 +90,10 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri this.admin = admin; this.deployLogger = deployLogger; } + + public ContentCluster build(Collection<ContainerModel> containers, + AbstractConfigProducer ancestor, Element w3cContentElement) { - public ContentCluster build(AbstractConfigProducer ancestor, Element w3cContentElement) { ModelElement contentElement = new ModelElement(w3cContentElement); ModelElement documentsElement = contentElement.getChild("documents"); @@ -137,7 +140,7 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri AbstractConfigProducerRoot root = ancestor.getRoot(); if (root == null) return c; - addClusterControllers(root, c.rootGroup, contentElement, c.clusterName, c); + addClusterControllers(containers, root, c.rootGroup, contentElement, c.clusterName, c); return c; } @@ -253,7 +256,9 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri } } - private void addClusterControllers(AbstractConfigProducerRoot root, StorageGroup rootGroup, ModelElement contentElement, String contentClusterName, ContentCluster contentCluster) { + private void addClusterControllers(Collection<ContainerModel> containers, AbstractConfigProducerRoot root, + StorageGroup rootGroup, ModelElement contentElement, + String contentClusterName, ContentCluster contentCluster) { if (admin == null) return; // only in tests if (contentCluster.getPersistence() == null) return; @@ -272,7 +277,7 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri NodesSpecification.optionalDedicatedFromParent(contentElement.getChild("controllers")).orElse(NodesSpecification.nonDedicated(3)); Collection<HostResource> hosts = nodesSpecification.isDedicated() ? getControllerHosts(nodesSpecification, admin, clusterName) : - drawContentHosts(nodesSpecification.count(), rootGroup); + drawControllerHosts(nodesSpecification.count(), rootGroup, containers); clusterControllers = createClusterControllers(new ClusterControllerCluster(contentCluster, "standalone"), hosts, clusterName, true); contentCluster.clusterControllers = clusterControllers; @@ -312,14 +317,46 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri return nodesSpecification.provision(admin.getHostSystem(), ClusterSpec.Type.admin, ClusterSpec.Id.from(clusterName), Optional.empty(), deployLogger).keySet(); } - private List<HostResource> drawContentHosts(int count, StorageGroup rootGroup) { + private List<HostResource> drawControllerHosts(int count, StorageGroup rootGroup, Collection<ContainerModel> containers) { List<HostResource> hosts = drawContentHostsRecursively(count, rootGroup); + if (hosts.size() < count) // supply with containers + hosts.addAll(drawContainerHosts(count - hosts.size(), containers)); if (hosts.size() % 2 == 0) // ZK clusters of even sizes are less available (even in the size=2 case) hosts = hosts.subList(0, hosts.size()-1); return hosts; } /** + * Draws <code>count</code> container nodes to use as cluster controllers, or as many as possible + * if less than <code>count</code> are available. + * + * This will draw the same nodes each time it is + * invoked if cluster names and node indexes are unchanged. + */ + private List<HostResource> drawContainerHosts(int count, Collection<ContainerModel> containerClusters) { + if (containerClusters.isEmpty()) return Collections.emptyList(); + + List<HostResource> hosts = new ArrayList<>(); + for (ContainerCluster cluster : clustersSortedByName(containerClusters)) + hosts.addAll(hostResourcesSortedByIndex(cluster)); + return hosts.subList(0, Math.min(hosts.size(), count)); + } + + private List<ContainerCluster> clustersSortedByName(Collection<ContainerModel> containerModels) { + return containerModels.stream() + .map(ContainerModel::getCluster) + .sorted(Comparator.comparing(ContainerCluster::getName)) + .collect(Collectors.toList()); + } + + private List<HostResource> hostResourcesSortedByIndex(ContainerCluster cluster) { + return cluster.getContainers().stream() + .sorted(Comparator.comparing(Container::index)) + .map(Container::getHostResource) + .collect(Collectors.toList()); + } + + /** * Draw <code>count</code> nodes from as many different content groups below this as possible. * This will only achieve maximum spread in the case where the groups are balanced and never on the same * physical node. It will not achieve maximum spread over all levels in a multilevel group hierarchy. diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index b74398fc4ae..e25aa667f10 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -465,6 +465,38 @@ public class ModelProvisioningTest { } @Test + public void testClusterControllersCanSupplementWithAllContainerClusters() throws ParseException { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <admin version='4.0'/>" + + " <container version='1.0' id='foo1'>" + + " <nodes count='2'/>" + + " </container>" + + " <container version='1.0' id='foo2'>" + + " <nodes count='1'/>" + + " </container>" + + " <content version='1.0' id='bar'>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <controllers><nodes dedicated='false' count='5'/></controllers>" + + " <nodes count='2'/>" + + " </content>" + + "</services>"; + + int numberOfHosts = 5; + Hosts hosts = createHosts(numberOfHosts); + VespaModel model = createModel(services, hosts, true); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + + ContentCluster cluster = model.getContentClusters().get("bar"); + ContainerCluster clusterControllers = cluster.getClusterControllers(); + assertEquals(5, clusterControllers.getContainers().size()); + } + + @Test public void testClusterControllersAreNotPlacedOnRetiredNodes() throws ParseException { String services = "<?xml version='1.0' encoding='utf-8' ?>\n" + @@ -602,6 +634,33 @@ public class ModelProvisioningTest { } @Test + public void test2ContentNodesWithContainerClusterProducesMixedClusterControllerCluster() throws ParseException { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <container version='1.0' id='foo'>" + + " <nodes count='3'/>" + + " </container>" + + " <content version='1.0' id='bar'>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='2'/>" + + " </content>" + + "</services>"; + + int numberOfHosts = 5; + Hosts hosts = createHosts(numberOfHosts); + VespaModel model = createModel(services, hosts, true); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + + ContentCluster cluster = model.getContentClusters().get("bar"); + ContainerCluster clusterControllers = cluster.getClusterControllers(); + assertEquals(3, clusterControllers.getContainers().size()); + } + + @Test public void testExplicitDedicatedClusterControllers() throws ParseException { String services = "<?xml version='1.0' encoding='utf-8' ?>\n" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index e1fbcecdf49..f39c214cd02 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -37,7 +37,7 @@ import static org.junit.Assert.*; /** * @author baldersheim */ -public class DomContentBuilderTest extends DomBuilderTest { +public class ContentBuilderTest extends DomBuilderTest { private ContentCluster createContent(String xml) throws Exception { String combined = "" + "<services>"+ diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 3365177409a..c6de6835d49 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -19,6 +19,7 @@ import com.yahoo.vespa.model.container.search.ContainerSearch; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; import org.junit.Test; +import java.util.Collections; import java.util.Iterator; import static org.junit.Assert.assertEquals; @@ -164,7 +165,7 @@ public class ContainerClusterTest { } private static void addContainer(ContainerCluster cluster, String name, String hostName) { - Container container = new Container(cluster, name); + Container container = new Container(cluster, name, 0); container.setHostResource(new HostResource(new Host(null, hostName))); container.initService(); cluster.addContainer(container); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 95761de7331..990c0e10927 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -17,6 +17,8 @@ import org.junit.Test; import org.w3c.dom.Document; import org.w3c.dom.Element; +import java.util.Collections; + import static org.junit.Assert.*; public class StorageClusterTest { @@ -31,7 +33,7 @@ public class StorageClusterTest { ); Document doc = XML.getDocument(xml); Element clusterElem = doc.getDocumentElement(); - ContentCluster cluster = new ContentCluster.Builder(null, null).build(root, clusterElem); + ContentCluster cluster = new ContentCluster.Builder(null, null).build(Collections.emptyList(), root, clusterElem); root.freezeModelTopology(); return cluster.getStorageNodes(); @@ -182,7 +184,7 @@ public class StorageClusterTest { "</cluster>" ); - ContentCluster cluster = new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement()); + ContentCluster cluster = new ContentCluster.Builder(null, null).build(Collections.emptyList(), new MockRoot(), doc.getDocumentElement()); for (int i = 0; i < 3; ++i) { StorageNode node = cluster.getStorageNodes().getChildren().get("" + i); @@ -205,7 +207,7 @@ public class StorageClusterTest { "</cluster>" ); - ContentCluster cluster = new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement()); + ContentCluster cluster = new ContentCluster.Builder(null, null).build(Collections.emptyList(), new MockRoot(), doc.getDocumentElement()); StorageNode node = cluster.getStorageNodes().getChildren().get("0"); @@ -249,7 +251,7 @@ public class StorageClusterTest { "</cluster>" ); - ContentCluster cluster = new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement()); + ContentCluster cluster = new ContentCluster.Builder(null, null).build(Collections.emptyList(), new MockRoot(), doc.getDocumentElement()); PersistenceConfig.Builder builder = new PersistenceConfig.Builder(); cluster.getStorageNodes().getConfig(builder); @@ -279,7 +281,7 @@ public class StorageClusterTest { ); try { - new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement()); + new ContentCluster.Builder(null, null).build(Collections.emptyList(), new MockRoot(), doc.getDocumentElement()); assertTrue(false); } catch (Exception e) { @@ -303,7 +305,7 @@ public class StorageClusterTest { "</cluster>" ); try { - new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement()); + new ContentCluster.Builder(null, null).build(Collections.emptyList(), new MockRoot(), doc.getDocumentElement()); fail("Did not get exception with duplicate group names"); } catch (RuntimeException e) { assertEquals("Cluster 'storage' has multiple groups with name 'bar' in the same subgroup. " + @@ -332,7 +334,7 @@ public class StorageClusterTest { "</cluster>" ); // Should not throw. - new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement()); + new ContentCluster.Builder(null, null).build(Collections.emptyList(), new MockRoot(), doc.getDocumentElement()); } @Test @@ -351,7 +353,7 @@ public class StorageClusterTest { "</cluster>" ); try { - new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement()); + new ContentCluster.Builder(null, null).build(Collections.emptyList(), new MockRoot(), doc.getDocumentElement()); fail("Did not get exception with missing distribution element"); } catch (RuntimeException e) { assertEquals("'distribution' attribute is required with multiple subgroups", e.getMessage()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageGroupTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageGroupTest.java index da6636255ba..f1768ab6927 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageGroupTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageGroupTest.java @@ -8,6 +8,8 @@ import com.yahoo.vespa.model.content.cluster.ContentCluster; import org.junit.Test; import org.w3c.dom.Document; +import java.util.Collections; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -15,9 +17,10 @@ import static org.junit.Assert.assertTrue; * Test for storage groups. */ public class StorageGroupTest { + ContentCluster parse(String xml) { Document doc = XML.getDocument(xml); - return new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement()); + return new ContentCluster.Builder(null, null).build(Collections.emptyList(), new MockRoot(), doc.getDocumentElement()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java index 7da263696c1..808c9a97287 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.model.content.cluster.ContentCluster; import org.w3c.dom.Document; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -72,7 +73,7 @@ public class ContentClusterBuilder { public ContentCluster build(MockRoot root) { Document doc = XML.getDocument(getXml()); - return new ContentCluster.Builder(null, null).build(root, doc.getDocumentElement()); + return new ContentCluster.Builder(null, null).build(Collections.emptyList(), root, doc.getDocumentElement()); } public String getXml() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java index c8b35bd0a37..f2f9ff482c1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import org.w3c.dom.Document; +import java.util.Collections; import java.util.List; import java.util.Optional; @@ -45,7 +46,7 @@ public class ContentClusterUtils { public static ContentCluster createCluster(String clusterXml, MockRoot root) throws Exception { Document doc = XML.getDocument(clusterXml); - return new ContentCluster.Builder(null, null).build(root, doc.getDocumentElement()); + return new ContentCluster.Builder(null, null).build(Collections.emptyList(), root, doc.getDocumentElement()); } public static ContentCluster createCluster(String clusterXml, List<String> searchDefinitions) throws Exception { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java index 1032f5099c6..6cb340fe4a0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java @@ -10,7 +10,6 @@ import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.builder.xml.dom.DomContentBuilder; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; @@ -139,7 +138,7 @@ public class ModelAmendingTestCase { @Override public List<ConfigModelId> handlesElements() { - return DomContentBuilder.configModelIds; + return Content.Builder.configModelIds; } @Override |