diff options
author | Jon Bratseth <bratseth@verizonmedia.com> | 2019-10-15 15:54:38 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@verizonmedia.com> | 2019-10-15 15:54:38 +0200 |
commit | cbefb58c71bc888061d1586e70bf01f1a432a756 (patch) | |
tree | 8e5f0d5d55400161a73f8f546a0c83fa204bc98e | |
parent | c51b015a8dd8f9fc8f6c6f65c4ceab4090831df3 (diff) |
Allow deploying self-hosted applications to dev
6 files changed, 71 insertions, 30 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java index dea2095d8e4..ec1038c67db 100644 --- a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java +++ b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java @@ -77,6 +77,10 @@ public class AdminModel extends ConfigModel { @Override public void doBuild(AdminModel model, Element adminElement, ConfigModelContext modelContext) { + if (modelContext.getDeployState().isHosted()) { // admin v2 is used on hosted: Build a default V4 instead + new BuilderV4().doBuild(model, adminElement, modelContext); + return; + } AbstractConfigProducer parent = modelContext.getParentProducer(); ModelContext.Properties properties = modelContext.getDeployState().getProperties(); DomAdminV2Builder domBuilder = new DomAdminV2Builder(modelContext.getApplicationType(), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index b0a74c08b4e..86c72221cab 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -63,13 +63,13 @@ public class NodesSpecification { private NodesSpecification(boolean dedicated, boolean canFail, Version version, ModelElement nodesElement) { this(dedicated, - nodesElement.requiredIntegerAttribute("count"), + nodesElement.integerAttribute("count", 1), nodesElement.integerAttribute("groups", 1), version, nodesElement.booleanAttribute("required", false), canFail, nodesElement.booleanAttribute("exclusive", false), - getFlavor(nodesElement), + getResources(nodesElement), Optional.ofNullable(nodesElement.stringAttribute("docker-image"))); } @@ -162,7 +162,7 @@ public class NodesSpecification { return hostSystem.allocateHosts(cluster, Capacity.fromCount(count, resources, required, canFail), groups, logger); } - private static Optional<NodeResources> getFlavor(ModelElement nodesElement) { + private static Optional<NodeResources> getResources(ModelElement nodesElement) { ModelElement resources = nodesElement.child("resources"); if (resources != null) { return Optional.of(new NodeResources(resources.requiredDoubleAttribute("vcpu"), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index efc566f2dec..cc78b92c260 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -499,8 +499,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } void extractJvmFromLegacyNodesTag(List<ApplicationContainer> nodes, ApplicationContainerCluster cluster, - Element nodesElement, ConfigModelContext context) - { + Element nodesElement, ConfigModelContext context) { applyNodesTagJvmArgs(nodes, getJvmOptions(cluster, nodesElement, context.getDeployLogger())); if (!cluster.getJvmGCOptions().isPresent()) { @@ -512,9 +511,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { applyMemoryPercentage(cluster, nodesElement.getAttribute(VespaDomBuilder.Allocated_MEMORY_ATTRIB_NAME)); } + void extractJvmTag(List<ApplicationContainer> nodes, ApplicationContainerCluster cluster, - Element jvmElement, ConfigModelContext context) - { + Element jvmElement, ConfigModelContext context) { applyNodesTagJvmArgs(nodes, jvmElement.getAttribute(VespaDomBuilder.OPTIONS)); applyMemoryPercentage(cluster, jvmElement.getAttribute(VespaDomBuilder.Allocated_MEMORY_ATTRIB_NAME)); String jvmGCOptions = jvmElement.hasAttribute(VespaDomBuilder.GC_OPTIONS) @@ -522,9 +521,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { : null; cluster.setJvmGCOptions(buildJvmGCOptions(context.getDeployState().zone(), jvmGCOptions, context.getDeployState().isHosted())); } + private void addNodesFromXml(ApplicationContainerCluster cluster, Element containerElement, ConfigModelContext context) { Element nodesElement = XML.getChild(containerElement, "nodes"); - Element rotationsElement = XML.getChild(containerElement, "rotations"); if (nodesElement == null) { // default single node on localhost ApplicationContainer node = new ApplicationContainer(cluster, "container.0", 0, cluster.isHostedVespa()); HostResource host = allocateSingleNodeHost(cluster, log, containerElement, context); @@ -564,12 +563,14 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } private List<ApplicationContainer> createNodes(ApplicationContainerCluster cluster, Element nodesElement, ConfigModelContext context) { - if (nodesElement.hasAttribute("count")) // regular, hosted node spec - return createNodesFromNodeCount(cluster, nodesElement, context); - else if (nodesElement.hasAttribute("type")) // internal use for hosted system infrastructure nodes + if (nodesElement.hasAttribute("type")) // internal use for hosted system infrastructure nodes return createNodesFromNodeType(cluster, nodesElement, context); else if (nodesElement.hasAttribute("of")) // hosted node spec referencing a content cluster return createNodesFromContentServiceReference(cluster, nodesElement, context); + else if (nodesElement.hasAttribute("count")) // regular, hosted node spec + return createNodesFromNodeCount(cluster, nodesElement, context); + else if (cluster.isHostedVespa() && cluster.getZone().environment().isManuallyDeployed()) // default to 1 in manual zones + return createNodesFromNodeCount(cluster, nodesElement, context); else // the non-hosted option return createNodesFromNodeList(context.getDeployState(), cluster, nodesElement); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index 4d1252a2618..228d972b839 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -401,9 +401,11 @@ public class StorageGroup { Optional<NodesSpecification> nodeRequirement; if (nodesElement.isPresent() && nodesElement.get().stringAttribute("count") != null ) // request these nodes nodeRequirement = Optional.of(NodesSpecification.from(nodesElement.get(), context)); + else if (nodesElement.isPresent() && context.getDeployState().isHosted() && context.getDeployState().zone().environment().isManuallyDeployed() ) // default to 1 node + nodeRequirement = Optional.of(NodesSpecification.from(nodesElement.get(), context)); else if (! nodesElement.isPresent() && subGroups.isEmpty() && context.getDeployState().isHosted()) // request one node nodeRequirement = Optional.of(NodesSpecification.nonDedicated(1, context)); - else // Nodes or groups explicitly listed, and/opr not hosted - resolve in GroupBuilder + else // Nodes or groups explicitly listed - resolve in GroupBuilder nodeRequirement = Optional.empty(); return new GroupBuilder(group, subGroups, explicitNodes, nodeRequirement, context.getDeployLogger()); diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index dfd2fe00d46..7c71c399f35 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -7,7 +7,9 @@ import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.search.config.QrStartConfig; @@ -166,10 +168,10 @@ public class ModelProvisioningTest { tester.addHosts(numberOfHosts); int numberOfContentNodes = 2; VespaModel model = tester.createModel(xmlWithNodes, true); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); - final Map<String, ContentCluster> contentClusters = model.getContentClusters(); + assertEquals(numberOfHosts, model.getRoot().getHostSystem().getHosts().size()); + Map<String, ContentCluster> contentClusters = model.getContentClusters(); ContentCluster cluster = contentClusters.get("bar"); - assertThat(cluster.getRootGroup().getNodes().size(), is(numberOfContentNodes)); + assertEquals(numberOfContentNodes, cluster.getRootGroup().getNodes().size()); int i = 0; for (StorageNode node : cluster.getRootGroup().getNodes()) assertEquals(i++, node.getDistributionKey()); @@ -385,7 +387,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, true); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + assertEquals(numberOfHosts, model.getRoot().getHostSystem().getHosts().size()); // Check container cluster assertEquals(1, model.getContainerClusters().size()); @@ -411,9 +413,9 @@ public class ModelProvisioningTest { assertEquals("node-1-3-9-48", clusterControllers.getContainers().get(2).getHostName()); assertEquals(0, cluster.getRootGroup().getNodes().size()); assertEquals(9, cluster.getRootGroup().getSubgroups().size()); - assertThat(cluster.getRootGroup().getSubgroups().get(0).getIndex(), is("0")); - assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().size(), is(3)); - assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey(), is(0)); + assertEquals("0", cluster.getRootGroup().getSubgroups().get(0).getIndex()); + assertEquals(3, cluster.getRootGroup().getSubgroups().get(0).getNodes().size()); + assertEquals(0, cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey()); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getConfigId(), is("bar/storage/0")); assertEquals("node-1-3-9-54", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getDistributionKey(), is(1)); @@ -1270,7 +1272,6 @@ public class ModelProvisioningTest { assertEquals(1, model.getRoot().getHostSystem().getHosts().size()); assertEquals(1, model.getAdmin().getSlobroks().size()); } - @Test public void testThatStandaloneSyntaxWorksOnHostedVespa() { String services = @@ -1283,8 +1284,45 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(1); VespaModel model = tester.createModel(services, true); - assertThat(model.getHosts().size(), is(1)); - assertThat(model.getContainerClusters().size(), is(1)); + assertEquals(1, model.getHosts().size()); + assertEquals(1, model.getContainerClusters().size()); + } + + @Test + public void testThatStandaloneSyntaxWorksOnHostedManuallyDeployed() { + String services = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + " <admin version='2.0'>" + + " <adminserver hostalias='node1'/>" + + " </admin>" + + " <container id='foo' version='1.0'>" + + " <nodes>" + + " <node hostalias='node1'/>" + + " <node hostalias='node2'/>" + + " </nodes>" + + " </container>" + + " <content id='bar' version='1.0'>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <redundancy>2</redundancy>" + + " <nodes>" + + " <group>" + + " <node distribution-key='0' hostalias='node3'/>" + + " <node distribution-key='1' hostalias='node4'/>" + + " </group>" + + " </nodes>" + + " </content>" + + "</services>"; + VespaModelTester tester = new VespaModelTester(); + tester.setHosted(true); + tester.addHosts(4); + VespaModel model = tester.createModel(new Zone(Environment.dev, RegionName.from("us-central-1")), services, true); + assertEquals("We get 1 node per cluster and no admin node", 2, model.getHosts().size()); + assertEquals(1, model.getContainerClusters().size()); + assertEquals(1, model.getContainerClusters().get("foo").getContainers().size()); + assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes()); } @Test @@ -1417,7 +1455,7 @@ public class ModelProvisioningTest { " </services>"; VespaModel model = createNonProvisionedMultitenantModel(services); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(1)); + assertEquals(1, model.getRoot().getHostSystem().getHosts().size()); ContentCluster content = model.getContentClusters().get("storage"); assertEquals(2, content.getRootGroup().getNodes().size()); ContainerCluster controller = content.getClusterControllers(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java index f9c7a96737f..e0d2e165772 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java @@ -328,10 +328,6 @@ public class ClusterControllerTestCase extends DomBuilderTest { @Test public void testUnconfiguredNoTuning() throws Exception { - verifyUnconfiguredNoTuning(false); - verifyUnconfiguredNoTuning(true); - } - private void verifyUnconfiguredNoTuning(boolean isHosted) throws Exception { String xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + "<services>\n" + "\n" + @@ -355,18 +351,18 @@ public class ClusterControllerTestCase extends DomBuilderTest { "\n" + "</services>"; - VespaModel model = createVespaModel(xml, isHosted); + VespaModel model = createVespaModel(xml, false); assertTrue(model.getService("admin/cluster-controllers/0").isPresent()); assertTrue(existsHostsWithClusterControllerConfigId(model)); assertGroupSize(model, "admin/cluster-controllers/0/components/clustercontroller-bar-configurer", 1); - assertThat(model.getAdmin().getClusterControllers().getContainers().size(), is(1)); + assertEquals(1, model.getAdmin().getClusterControllers().getContainers().size()); FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); model.getConfig(builder, "admin/cluster-controllers/0/components/clustercontroller-bar-configurer"); FleetcontrollerConfig cfg = new FleetcontrollerConfig(builder); - assertThat(cfg.index(), is(0)); + assertEquals(0, cfg.index()); QrStartConfig.Builder qrBuilder = new QrStartConfig.Builder(); model.getConfig(qrBuilder, "admin/cluster-controllers/0/components/clustercontroller-bar-configurer"); |