diff options
author | Øyvind Grønnesby <oyving@verizonmedia.com> | 2021-04-28 12:21:57 +0200 |
---|---|---|
committer | Øyvind Grønnesby <oyving@verizonmedia.com> | 2021-04-28 12:21:57 +0200 |
commit | dd2ef6cfc4d3d6e3735d1cb553f7ae2560a7f1ff (patch) | |
tree | e1ba3a56439bb3c16022b60d2a7ab3534037827e /config-model/src/test/java | |
parent | a0db2b1020ea53aa356a7547a23d4e1dfaa851c0 (diff) | |
parent | e79af49a3159e5505cd3e5f2605c299d38fe40cd (diff) |
Merge remote-tracking branch 'origin/master' into ogronnesby/billing-api-v2
Diffstat (limited to 'config-model/src/test/java')
24 files changed, 593 insertions, 262 deletions
diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java index 9780e9b503a..75cb41be13f 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model; import com.google.common.io.Files; @@ -197,13 +197,21 @@ public class ApplicationDeployTest { @Test public void testThatAppWithInvalidParallelDeploymentFails() throws IOException { + String expectedMessage = "4: <staging/>\n" + + "5: <prod global-service-id=\"query\">\n" + + "6: <parallel>\n" + + "7: <instance id=\"hello\" />\n" + + "8: </parallel>\n" + + "9: </prod>\n" + + "10:</deployment>\n"; File tmpDir = tmpFolder.getRoot(); IOUtils.copyDirectory(new File(TESTDIR, "invalid_parallel_deployment_xml"), tmpDir); try { ApplicationPackageTester.create(tmpDir.getAbsolutePath()); fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("Invalid XML according to XML schema, error in deployment.xml: element \"instance\" not allowed here; expected the element end-tag or element \"delay\", \"region\", \"steps\" or \"test\" [7:30], input:\n", e.getMessage()); + assertEquals("Invalid XML according to XML schema, error in deployment.xml: element \"instance\" not allowed here; expected the element end-tag or element \"delay\", \"region\", \"steps\" or \"test\" [7:30], input:\n" + expectedMessage, + e.getMessage()); } } diff --git a/config-model/src/test/java/com/yahoo/config/model/application/provider/SchemaValidatorTest.java b/config-model/src/test/java/com/yahoo/config/model/application/provider/SchemaValidatorTest.java index c2938746443..3849e9e03fd 100644 --- a/config-model/src/test/java/com/yahoo/config/model/application/provider/SchemaValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/application/provider/SchemaValidatorTest.java @@ -2,12 +2,14 @@ package com.yahoo.config.model.application.provider; import com.yahoo.component.Version; +import com.yahoo.io.IOUtils; import com.yahoo.vespa.config.VespaVersion; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.xml.sax.InputSource; +import java.io.File; import java.io.IOException; import java.io.StringReader; @@ -47,15 +49,15 @@ public class SchemaValidatorTest { @Test public void testXMLParse() throws IOException { SchemaValidator validator = createValidator(); - validator.validate(new InputSource(new StringReader(okServices)), "services.xml"); + validator.validate(new StringReader(okServices)); } @Test public void testXMLParseError() throws IOException { SchemaValidator validator = createValidator(); expectedException.expect(RuntimeException.class); - expectedException.expectMessage(expectedErrorMessage("services.xml")); - validator.validate(new InputSource(new StringReader(invalidServices)), "services.xml"); + expectedException.expectMessage(expectedErrorMessage("input")); + validator.validate(new StringReader(invalidServices)); } @Test @@ -72,6 +74,14 @@ public class SchemaValidatorTest { validator.validate(new StringReader(invalidServices)); } + @Test + public void testXMLParseErrorFromFile() throws IOException { + SchemaValidator validator = createValidator(); + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage(expectedErrorMessage("services.xml")); + validator.validate(new File("src/test/cfg/application/invalid-services-syntax/services.xml")); + } + private SchemaValidator createValidator() { return new SchemaValidators(new Version(VespaVersion.major)).servicesXmlValidator(); } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 5a06379a1c2..86668fe3098 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -264,8 +264,7 @@ public class ModelProvisioningTest { " </content>" + "</services>"; VespaModelTester tester = new VespaModelTester(); - tester.dedicatedClusterControllerCluster(false); - tester.addHosts(2); + tester.addHosts(5); VespaModel model = tester.createModel(xmlWithNodes, true); assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size()); assertEquals("Nodes in container1", 2, model.getContainerClusters().get("container1").getContainers().size()); @@ -330,8 +329,7 @@ public class ModelProvisioningTest { " </content>" + "</services>"; VespaModelTester tester = new VespaModelTester(); - tester.dedicatedClusterControllerCluster(false); - tester.addHosts(2); + tester.addHosts(5); VespaModel model = tester.createModel(xmlWithNodes, true); assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size()); @@ -583,9 +581,8 @@ public class ModelProvisioningTest { " </content>" + "</services>"; - int numberOfHosts = 64; + int numberOfHosts = 67; VespaModelTester tester = new VespaModelTester(); - tester.dedicatedClusterControllerCluster(false); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, true); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); @@ -604,23 +601,17 @@ public class ModelProvisioningTest { assertTrue("Slobroks are assigned from container nodes", containerHosts.containsAll(slobrokHosts)); assertTrue("Logserver is assigned from container nodes", containerHosts.contains(admin.getLogserver().getHost())); assertEquals("No in-cluster config servers in a hosted environment", 0, admin.getConfigservers().size()); - assertEquals("No admin cluster controller when multitenant", null, admin.getClusterControllers()); + assertEquals(3, admin.getClusterControllers().getContainers().size()); // Check content clusters ContentCluster cluster = model.getContentClusters().get("bar"); - ClusterControllerContainerCluster clusterControllers = cluster.getClusterControllers(); - assertEquals(3, clusterControllers.getContainers().size()); - assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("node-1-3-10-54", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("node-1-3-10-51", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("node-1-3-10-48", clusterControllers.getContainers().get(2).getHostName()); assertEquals(0, cluster.getRootGroup().getNodes().size()); assertEquals(9, cluster.getRootGroup().getSubgroups().size()); assertEquals("0", cluster.getRootGroup().getSubgroups().get(0).getIndex()); assertEquals(3, cluster.getRootGroup().getSubgroups().get(0).getNodes().size()); assertEquals(0, cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey()); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getConfigId(), is("bar/storage/0")); - assertEquals("node-1-3-10-54", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); + assertEquals("node-1-3-10-57", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getDistributionKey(), is(1)); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getConfigId(), is("bar/storage/1")); assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(2).getDistributionKey(), is(2)); @@ -629,13 +620,13 @@ public class ModelProvisioningTest { assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().size(), is(3)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getDistributionKey(), is(3)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getConfigId(), is("bar/storage/3")); - assertEquals("node-1-3-10-51", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); + assertEquals("node-1-3-10-54", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getDistributionKey(), is(4)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getConfigId(), is("bar/storage/4")); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(2).getDistributionKey(), is(5)); assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(2).getConfigId(), is("bar/storage/5")); // ... - assertEquals("node-1-3-10-48", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName()); + assertEquals("node-1-3-10-51", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName()); // ... assertThat(cluster.getRootGroup().getSubgroups().get(8).getIndex(), is("8")); assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().size(), is(3)); @@ -647,12 +638,6 @@ public class ModelProvisioningTest { assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().get(2).getConfigId(), is("bar/storage/26")); cluster = model.getContentClusters().get("baz"); - clusterControllers = cluster.getClusterControllers(); - assertEquals(3, clusterControllers.getContainers().size()); - assertEquals("baz-controllers", clusterControllers.getName()); - assertEquals("node-1-3-10-27", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("node-1-3-10-26", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("node-1-3-10-25", clusterControllers.getContainers().get(2).getHostName()); assertEquals(0, cluster.getRootGroup().getNodes().size()); assertEquals(27, cluster.getRootGroup().getSubgroups().size()); assertThat(cluster.getRootGroup().getSubgroups().get(0).getIndex(), is("0")); @@ -732,41 +717,6 @@ public class ModelProvisioningTest { } @Test - public void testClusterControllersWithGroupSize2() { - String services = - "<?xml version='1.0' encoding='utf-8' ?>\n" + - "<services>" + - " <admin version='4.0'/>" + - " <container version='1.0' id='foo'>" + - " <nodes count='10'/>" + - " </container>" + - " <content version='1.0' id='bar'>" + - " <redundancy>2</redundancy>" + - " <documents>" + - " <document type='type1' mode='index'/>" + - " </documents>" + - " <nodes count='8' groups='4'/>" + - " </content>" + - "</services>"; - - int numberOfHosts = 18; - VespaModelTester tester = new VespaModelTester(); - tester.dedicatedClusterControllerCluster(false); - tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true); - assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); - - // Check content clusters - ContentCluster cluster = model.getContentClusters().get("bar"); - ClusterControllerContainerCluster clusterControllers = cluster.getClusterControllers(); - assertEquals("We get the closest odd number", 3, clusterControllers.getContainers().size()); - assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("node-1-3-10-08", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("node-1-3-10-06", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("node-1-3-10-04", clusterControllers.getContainers().get(2).getHostName()); - } - - @Test public void testSlobroksClustersAreExpandedToIncludeRetiredNodes() { String services = "<?xml version='1.0' encoding='utf-8' ?>\n" + @@ -777,7 +727,7 @@ public class ModelProvisioningTest { " </container>" + "</services>"; - int numberOfHosts = 10; + int numberOfHosts = 11; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, true, "node-1-3-10-09"); @@ -785,9 +735,9 @@ public class ModelProvisioningTest { // Check slobroks clusters assertEquals("Includes retired node", 1+3, model.getAdmin().getSlobroks().size()); - assertEquals("node-1-3-10-10", model.getAdmin().getSlobroks().get(0).getHostName()); - assertEquals("node-1-3-10-08", model.getAdmin().getSlobroks().get(1).getHostName()); - assertEquals("node-1-3-10-07", model.getAdmin().getSlobroks().get(2).getHostName()); + assertEquals("node-1-3-10-11", model.getAdmin().getSlobroks().get(0).getHostName()); + assertEquals("node-1-3-10-10", model.getAdmin().getSlobroks().get(1).getHostName()); + assertEquals("node-1-3-10-08", model.getAdmin().getSlobroks().get(2).getHostName()); assertEquals("Included in addition because it is retired", "node-1-3-10-09", model.getAdmin().getSlobroks().get(3).getHostName()); } @@ -802,19 +752,19 @@ public class ModelProvisioningTest { " </container>" + "</services>"; - int numberOfHosts = 10; + int numberOfHosts = 12; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true, "node-1-3-10-01", "node-1-3-10-02"); - assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); + VespaModel model = tester.createModel(services, true, "node-1-3-10-03", "node-1-3-10-04"); + assertEquals(10+2, model.getRoot().hostSystem().getHosts().size()); // Check slobroks clusters assertEquals("Includes retired node", 3+2, model.getAdmin().getSlobroks().size()); - assertEquals("node-1-3-10-10", model.getAdmin().getSlobroks().get(0).getHostName()); - assertEquals("node-1-3-10-09", model.getAdmin().getSlobroks().get(1).getHostName()); - assertEquals("node-1-3-10-08", model.getAdmin().getSlobroks().get(2).getHostName()); - assertEquals("Included in addition because it is retired", "node-1-3-10-02", model.getAdmin().getSlobroks().get(3).getHostName()); - assertEquals("Included in addition because it is retired", "node-1-3-10-01", model.getAdmin().getSlobroks().get(4).getHostName()); + assertEquals("node-1-3-10-12", model.getAdmin().getSlobroks().get(0).getHostName()); + assertEquals("node-1-3-10-11", model.getAdmin().getSlobroks().get(1).getHostName()); + assertEquals("node-1-3-10-10", model.getAdmin().getSlobroks().get(2).getHostName()); + assertEquals("Included in addition because it is retired", "node-1-3-10-04", model.getAdmin().getSlobroks().get(3).getHostName()); + assertEquals("Included in addition because it is retired", "node-1-3-10-03", model.getAdmin().getSlobroks().get(4).getHostName()); } @Test @@ -831,48 +781,22 @@ public class ModelProvisioningTest { " </container>" + "</services>"; - int numberOfHosts = 13; + int numberOfHosts = 16; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true, "node-1-3-10-12", "node-1-3-10-03", "node-1-3-10-02"); + VespaModel model = tester.createModel(services, true, "node-1-3-10-15", "node-1-3-10-05", "node-1-3-10-04"); assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); // Check slobroks clusters // ... from cluster default - assertEquals("Includes retired node", 3+3, model.getAdmin().getSlobroks().size()); - assertEquals("node-1-3-10-13", model.getAdmin().getSlobroks().get(0).getHostName()); - assertEquals("node-1-3-10-11", model.getAdmin().getSlobroks().get(1).getHostName()); - assertEquals("Included in addition because it is retired", "node-1-3-10-12", model.getAdmin().getSlobroks().get(2).getHostName()); + assertEquals("Includes retired node", 7, model.getAdmin().getSlobroks().size()); + assertEquals("node-1-3-10-16", model.getAdmin().getSlobroks().get(0).getHostName()); + assertEquals("node-1-3-10-14", model.getAdmin().getSlobroks().get(1).getHostName()); + assertEquals("Included in addition because it is retired", "node-1-3-10-15", model.getAdmin().getSlobroks().get(2).getHostName()); // ... from cluster bar - assertEquals("node-1-3-10-01", model.getAdmin().getSlobroks().get(3).getHostName()); - assertEquals("Included in addition because it is retired", "node-1-3-10-03", model.getAdmin().getSlobroks().get(4).getHostName()); - assertEquals("Included in addition because it is retired", "node-1-3-10-02", model.getAdmin().getSlobroks().get(5).getHostName()); - } - - @Test - public void test2ContentNodesProduces1ClusterController() { - String services = - "<?xml version='1.0' encoding='utf-8' ?>\n" + - "<services>" + - " <content version='1.0' id='bar'>" + - " <redundancy>2</redundancy>" + - " <documents>" + - " <document type='type1' mode='index'/>" + - " </documents>" + - " <nodes count='2'/>" + - " </content>" + - "</services>"; - - int numberOfHosts = 2; - VespaModelTester tester = new VespaModelTester(); - tester.dedicatedClusterControllerCluster(false); - tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true); - assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); - - ContentCluster cluster = model.getContentClusters().get("bar"); - ContainerCluster clusterControllers = cluster.getClusterControllers(); - assertEquals(1, clusterControllers.getContainers().size()); + assertEquals("node-1-3-10-03", model.getAdmin().getSlobroks().get(3).getHostName()); + assertEquals("Included in addition because it is retired", "node-1-3-10-05", model.getAdmin().getSlobroks().get(5).getHostName()); + assertEquals("Included in addition because it is retired", "node-1-3-10-04", model.getAdmin().getSlobroks().get(6).getHostName()); } @Test @@ -967,7 +891,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, false); - assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); + assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); ContentCluster cluster = model.getContentClusters().get("bar"); assertEquals(2*3, cluster.redundancy().effectiveInitialRedundancy()); // Reduced from 3*3 @@ -997,6 +921,72 @@ public class ModelProvisioningTest { } @Test + public void testRedundancyWithGroupsTooHighRedundancyAndOneRetiredNode() { + String services = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + " <content version='1.0' id='bar'>" + + " <redundancy>2</redundancy>" + // Should have been illegal since we only have 1 node per group + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='2' groups='2'/>" + + " </content>" + + "</services>"; + + int numberOfHosts = 3; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(numberOfHosts); + VespaModel model = tester.createModel(services, false, "node-1-3-10-03"); + assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); + + ContentCluster cluster = model.getContentClusters().get("bar"); + assertEquals(2, cluster.redundancy().effectiveInitialRedundancy()); + assertEquals(2, cluster.redundancy().effectiveFinalRedundancy()); + assertEquals(2, cluster.redundancy().effectiveReadyCopies()); + assertEquals("1|*", cluster.getRootGroup().getPartitions().get()); + assertEquals(0, cluster.getRootGroup().getNodes().size()); + assertEquals(2, cluster.getRootGroup().getSubgroups().size()); + System.out.println("Nodes in group 0: "); + cluster.getRootGroup().getSubgroups().get(0).getNodes().forEach(n -> System.out.println(" " + n)); + System.out.println("Nodes in group 1: "); + cluster.getRootGroup().getSubgroups().get(1).getNodes().forEach(n -> System.out.println(" " + n)); + } + + @Test + public void testRedundancyWithGroupsAndThreeRetiredNodes() { + String services = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + " <content version='1.0' id='bar'>" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='2' groups='2'/>" + + " </content>" + + "</services>"; + + int numberOfHosts = 5; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(numberOfHosts); + VespaModel model = tester.createModel(services, false, "node-1-3-10-05", "node-1-3-10-04", "node-1-3-10-03"); + assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); + + ContentCluster cluster = model.getContentClusters().get("bar"); + assertEquals(2, cluster.redundancy().effectiveInitialRedundancy()); + assertEquals(2, cluster.redundancy().effectiveFinalRedundancy()); + assertEquals(2, cluster.redundancy().effectiveReadyCopies()); + assertEquals("1|*", cluster.getRootGroup().getPartitions().get()); + assertEquals(0, cluster.getRootGroup().getNodes().size()); + assertEquals(2, cluster.getRootGroup().getSubgroups().size()); + System.out.println("Nodes in group 0: "); + cluster.getRootGroup().getSubgroups().get(0).getNodes().forEach(n -> System.out.println(" " + n)); + System.out.println("Nodes in group 1: "); + cluster.getRootGroup().getSubgroups().get(1).getNodes().forEach(n -> System.out.println(" " + n)); + } + + @Test public void testUsingNodesCountAttributesAndGettingTooFewNodes() { String services = "<?xml version='1.0' encoding='utf-8' ?>" + @@ -1065,16 +1055,11 @@ public class ModelProvisioningTest { int numberOfHosts = 1; // We only have 1 content node -> 1 groups with redundancy 1 VespaModelTester tester = new VespaModelTester(); - tester.dedicatedClusterControllerCluster(false); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, false); assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); ContentCluster cluster = model.getContentClusters().get("bar"); - ClusterControllerContainerCluster clusterControllers = cluster.getClusterControllers(); - assertEquals(1, clusterControllers.getContainers().size()); - assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("node-1-3-10-01", clusterControllers.getContainers().get(0).getHostName()); assertEquals(1, cluster.redundancy().effectiveInitialRedundancy()); // Reduced from 3*3 assertEquals(1, cluster.redundancy().effectiveFinalRedundancy()); // Reduced from 3*4 assertEquals(1, cluster.redundancy().effectiveReadyCopies()); // Reduced from 3*3 @@ -1229,14 +1214,14 @@ public class ModelProvisioningTest { " </content>" + "</services>"; - int totalHosts = 18; + int totalHosts = 21; VespaModelTester tester = new VespaModelTester(); - tester.dedicatedClusterControllerCluster(false); tester.addHosts(new NodeResources(0.1, 0.2, 300, 0.3, NodeResources.DiskSpeed.slow), 1);// Logserver tester.addHosts(new NodeResources(0.1, 0.3, 1, 0.5), 2); // Slobrok tester.addHosts(new NodeResources(12, 10, 30, 0.3), 4); // Container tester.addHosts(new NodeResources(8, 200, 1000000, 0.3), 5); // Content-foo tester.addHosts(new NodeResources(10, 64, 200, 0.3), 6); // Content-bar + tester.addHosts(new NodeResources(0.5, 2, 10, 0.3), 6); // Cluster-controller VespaModel model = tester.createModel(services, true, 0); assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size()); } @@ -1263,11 +1248,11 @@ public class ModelProvisioningTest { int totalHosts = 10; VespaModelTester tester = new VespaModelTester(); - tester.dedicatedClusterControllerCluster(false); - tester.addHosts(new NodeResources(11.5, 10, 30, 0.3), 6); - tester.addHosts(new NodeResources(85, 200, 1000_000_000, 0.3), 20); + tester.addHosts(new NodeResources(11.5, 10, 30, 0.3), 6); + tester.addHosts(new NodeResources(85, 200, 1000_000_000, 0.3), 20); + tester.addHosts(new NodeResources( 0.5, 2, 10, 0.3), 3); VespaModel model = tester.createModel(services, true); - assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size()); + assertEquals(totalHosts + 3, model.getRoot().hostSystem().getHosts().size()); } @Test @@ -1290,11 +1275,11 @@ public class ModelProvisioningTest { " </content>" + "</services>"; - int totalHosts = 26; + int totalHosts = 29; VespaModelTester tester = new VespaModelTester(); - tester.dedicatedClusterControllerCluster(false); - tester.addHosts(new NodeResources(13.5, 100, 1000, 0.3), 6); - tester.addHosts(new NodeResources(85, 200, 1000_000_000, 0.3), 20); + tester.addHosts(new NodeResources(13.5, 100, 1000, 0.3), 6); + tester.addHosts(new NodeResources(85, 200, 1000_000_000, 0.3), 20); + tester.addHosts(new NodeResources( 0.5, 2, 10, 0.3), 3); VespaModel model = tester.createModel(services, true, true); assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size()); } @@ -1515,10 +1500,9 @@ public class ModelProvisioningTest { " </content>" + "</services>"; VespaModelTester tester = new VespaModelTester(); - tester.dedicatedClusterControllerCluster(false); - tester.addHosts(3); + tester.addHosts(6); VespaModel model = tester.createModel(services, true); - assertEquals(3, model.getRoot().hostSystem().getHosts().size()); + assertEquals(6, model.getRoot().hostSystem().getHosts().size()); assertEquals(2, model.getAdmin().getSlobroks().size()); assertEquals(2, model.getContainerClusters().get("foo").getContainers().size()); assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes()); @@ -1633,7 +1617,7 @@ public class ModelProvisioningTest { assertEquals(1, model.getRoot().hostSystem().getHosts().size()); ContentCluster content = model.getContentClusters().get("storage"); assertEquals(2, content.getRootGroup().getNodes().size()); - ContainerCluster controller = content.getClusterControllers(); + ContainerCluster<?> controller = model.getAdmin().getClusterControllers(); assertEquals(1, controller.getContainers().size()); } @@ -1679,7 +1663,7 @@ public class ModelProvisioningTest { assertThat(model.getRoot().hostSystem().getHosts().size(), is(1)); ContentCluster content = model.getContentClusters().get("storage"); assertEquals(1, content.getRootGroup().getNodes().size()); - ContainerCluster controller = content.getClusterControllers(); + ContainerCluster<?> controller = model.getAdmin().getClusterControllers(); assertEquals(1, controller.getContainers().size()); } @@ -1788,10 +1772,10 @@ public class ModelProvisioningTest { " </services>"; VespaModel model = createNonProvisionedMultitenantModel(services); - assertThat(model.getRoot().hostSystem().getHosts().size(), is(1)); + assertEquals(1, model.getRoot().hostSystem().getHosts().size()); ContentCluster content = model.getContentClusters().get("storage"); assertEquals(2, content.getRootGroup().getNodes().size()); - ContainerCluster controller = content.getClusterControllers(); + ContainerCluster<?> controller = model.getAdmin().getClusterControllers(); assertEquals(1, controller.getContainers().size()); } @@ -1816,8 +1800,7 @@ public class ModelProvisioningTest { " </content>" + "</services>"; VespaModelTester tester = new VespaModelTester(); - tester.dedicatedClusterControllerCluster(false); - tester.addHosts(6); + tester.addHosts(9); VespaModel model = tester.createModel(servicesXml, true); Map<String, Boolean> tests = Map.of("qrs", false, @@ -1841,12 +1824,12 @@ public class ModelProvisioningTest { "<services>" + " <container version='1.0' id='zk'>" + " <zookeeper/>" + - " <nodes count='4'/>" + // (3 + 1 retired) + " <nodes count='3'/>" + " </container>" + "</services>"; VespaModelTester tester = new VespaModelTester(); tester.addHosts(4); - VespaModel model = tester.createModel(servicesXml, true, "node-1-3-10-01"); + VespaModel model = tester.createModel(servicesXml, true, "node-1-3-10-04"); ApplicationContainerCluster cluster = model.getContainerClusters().get("zk"); assertEquals(1, cluster.getContainers().stream().filter(Container::isRetired).count()); assertEquals(3, cluster.getContainers().stream().filter(c -> !c.isRetired()).count()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java index 8fe4a8fb022..d665b7f20f0 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java @@ -159,7 +159,7 @@ public class RankingExpressionShadowingTestCase extends SchemaTestCase { public void testNeuralNetworkSetup() throws ParseException { // Note: the type assigned to query profile and constant tensors here is not the correct type RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - QueryProfileRegistry queryProfiles = queryProfileWith("query(q)", "tensor(x[1])"); + QueryProfileRegistry queryProfiles = queryProfileWith("query(q)", "tensor(input[1])"); SearchBuilder builder = new SearchBuilder(rankProfileRegistry, queryProfiles); builder.importString( "search test {\n" + @@ -184,19 +184,19 @@ public class RankingExpressionShadowingTestCase extends SchemaTestCase { " }\n" + " }\n" + " constant W_hidden {\n" + - " type: tensor(x[1])\n" + + " type: tensor(hidden[1])\n" + " file: ignored.json\n" + " }\n" + " constant b_input {\n" + - " type: tensor(x[1])\n" + + " type: tensor(hidden[1])\n" + " file: ignored.json\n" + " }\n" + " constant W_final {\n" + - " type: tensor(x[1])\n" + + " type: tensor(final[1])\n" + " file: ignored.json\n" + " }\n" + " constant b_final {\n" + - " type: tensor(x[1])\n" + + " type: tensor(final[1])\n" + " file: ignored.json\n" + " }\n" + "}\n"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java index ba51caca0f7..df0d58d3c8a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java @@ -6,12 +6,16 @@ import com.yahoo.config.model.test.TestUtil; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.derived.AttributeFields; +import com.yahoo.searchdefinition.document.Case; import com.yahoo.searchdefinition.document.Dictionary; +import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.searchdefinition.document.Matching; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.config.search.AttributesConfig; import org.junit.Test; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.fail; /** @@ -44,14 +48,19 @@ public class DictionaryTestCase { " }", "}"); Search search = createSearch(def); - assertEquals(Dictionary.Type.BTREE, search.getAttribute("s1").getDictionary().getType()); - assertEquals(Dictionary.Type.BTREE, search.getAttribute("n1").getDictionary().getType()); + assertNull(search.getAttribute("s1").getDictionary()); + assertNull(search.getAttribute("n1").getDictionary()); + assertEquals(AttributesConfig.Attribute.Dictionary.Type.BTREE, + getConfig(search).attribute().get(0).dictionary().type()); + assertEquals(AttributesConfig.Attribute.Dictionary.Type.BTREE, + getConfig(search).attribute().get(1).dictionary().type()); + assertEquals(AttributesConfig.Attribute.Dictionary.Match.UNCASED, + getConfig(search).attribute().get(0).dictionary().match()); + assertEquals(AttributesConfig.Attribute.Dictionary.Match.UNCASED, + getConfig(search).attribute().get(1).dictionary().match()); } - void verifyNumericDictionaryControl(Dictionary.Type expected, - AttributesConfig.Attribute.Dictionary.Type.Enum expectedConfig, - String type, - String ... cfg) throws ParseException + Search verifyDictionaryControl(Dictionary.Type expected, String type, String ... cfg) throws ParseException { String def = TestUtil.joinLines( "search test {", @@ -64,75 +73,147 @@ public class DictionaryTestCase { " }", "}"); Search search = createSearch(def); + AttributesConfig.Attribute.Dictionary.Type.Enum expectedConfig = toCfg(expected); assertEquals(expected, search.getAttribute("n1").getDictionary().getType()); - assertEquals(expectedConfig, - getConfig(search).attribute().get(0).dictionary().type()); + assertEquals(expectedConfig, getConfig(search).attribute().get(0).dictionary().type()); + return search; + } + + AttributesConfig.Attribute.Dictionary.Type.Enum toCfg(Dictionary.Type v) { + return (v == Dictionary.Type.HASH) + ? AttributesConfig.Attribute.Dictionary.Type.Enum.HASH + : (v == Dictionary.Type.BTREE) + ? AttributesConfig.Attribute.Dictionary.Type.Enum.BTREE + : AttributesConfig.Attribute.Dictionary.Type.Enum.BTREE_AND_HASH; + } + AttributesConfig.Attribute.Dictionary.Match.Enum toCfg(Case v) { + return (v == Case.CASED) + ? AttributesConfig.Attribute.Dictionary.Match.Enum.CASED + : AttributesConfig.Attribute.Dictionary.Match.Enum.UNCASED; + } + + void verifyStringDictionaryControl(Dictionary.Type expectedType, Case expectedCase, Case matchCasing, + String ... cfg) throws ParseException + { + Search search = verifyDictionaryControl(expectedType, "string", cfg); + ImmutableSDField f = search.getField("n1"); + AttributesConfig.Attribute.Dictionary.Match.Enum expectedCaseCfg = toCfg(expectedCase); + assertEquals(matchCasing, f.getMatching().getCase()); + assertEquals(expectedCase, search.getAttribute("n1").getDictionary().getMatch()); + assertEquals(expectedCaseCfg, getConfig(search).attribute().get(0).dictionary().match()); + } + + @Test + public void testCasedBtreeSettings() throws ParseException { + verifyDictionaryControl(Dictionary.Type.BTREE, "int", "dictionary:cased"); } @Test public void testNumericBtreeSettings() throws ParseException { - verifyNumericDictionaryControl(Dictionary.Type.BTREE, - AttributesConfig.Attribute.Dictionary.Type.BTREE, - "int", - "dictionary:btree"); + verifyDictionaryControl(Dictionary.Type.BTREE, "int", "dictionary:btree"); } @Test public void testNumericHashSettings() throws ParseException { - verifyNumericDictionaryControl(Dictionary.Type.HASH, - AttributesConfig.Attribute.Dictionary.Type.HASH, - "int", - "dictionary:hash"); + verifyDictionaryControl(Dictionary.Type.HASH, "int", "dictionary:hash"); } @Test public void testNumericBtreeAndHashSettings() throws ParseException { - verifyNumericDictionaryControl(Dictionary.Type.BTREE_AND_HASH, - AttributesConfig.Attribute.Dictionary.Type.BTREE_AND_HASH, - "int", - "dictionary:btree", "dictionary:hash"); + verifyDictionaryControl(Dictionary.Type.BTREE_AND_HASH, "int", "dictionary:btree", "dictionary:hash"); } @Test public void testNumericArrayBtreeAndHashSettings() throws ParseException { - verifyNumericDictionaryControl(Dictionary.Type.BTREE_AND_HASH, - AttributesConfig.Attribute.Dictionary.Type.BTREE_AND_HASH, - "array<int>", - "dictionary:btree", "dictionary:hash"); + verifyDictionaryControl(Dictionary.Type.BTREE_AND_HASH, "array<int>", "dictionary:btree", "dictionary:hash"); } @Test public void testNumericWSetBtreeAndHashSettings() throws ParseException { - verifyNumericDictionaryControl(Dictionary.Type.BTREE_AND_HASH, - AttributesConfig.Attribute.Dictionary.Type.BTREE_AND_HASH, - "weightedset<int>", - "dictionary:btree", "dictionary:hash"); + verifyDictionaryControl(Dictionary.Type.BTREE_AND_HASH, "weightedset<int>", "dictionary:btree", "dictionary:hash"); + } + @Test + public void testStringBtreeSettings() throws ParseException { + verifyStringDictionaryControl(Dictionary.Type.BTREE, Case.UNCASED, Case.UNCASED, "dictionary:btree"); + } + @Test + public void testStringBtreeUnCasedSettings() throws ParseException { + verifyStringDictionaryControl(Dictionary.Type.BTREE, Case.UNCASED, Case.UNCASED, "dictionary { btree\nuncased\n}"); + } + @Test + public void testStringBtreeCasedSettings() throws ParseException { + verifyStringDictionaryControl(Dictionary.Type.BTREE, Case.CASED, Case.CASED, "dictionary { btree\ncased\n}", "match:cased"); + } + @Test + public void testStringHashSettings() throws ParseException { + try { + verifyStringDictionaryControl(Dictionary.Type.HASH, Case.UNCASED, Case.UNCASED, "dictionary:hash"); + } catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 'n1': hash dictionary require cased match", e.getMessage()); + } + } + @Test + public void testStringHashUnCasedSettings() throws ParseException { + try { + verifyStringDictionaryControl(Dictionary.Type.HASH, Case.UNCASED, Case.UNCASED, "dictionary { hash\nuncased\n}"); + } catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 'n1': hash dictionary require cased match", e.getMessage()); + } + } + @Test + public void testStringHashBothCasedSettings() throws ParseException { + verifyStringDictionaryControl(Dictionary.Type.HASH, Case.CASED, Case.CASED, "dictionary { hash\ncased\n}", "match:cased"); + } + @Test + public void testStringHashCasedSettings() throws ParseException { + try { + verifyStringDictionaryControl(Dictionary.Type.HASH, Case.CASED, Case.CASED, "dictionary { hash\ncased\n}"); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 'n1': Dictionary casing 'CASED' does not match field match casing 'UNCASED'", e.getMessage()); + } + } + @Test + public void testStringBtreeHashSettings() throws ParseException { + verifyStringDictionaryControl(Dictionary.Type.BTREE_AND_HASH, Case.UNCASED, Case.UNCASED, "dictionary{hash\nbtree\n}"); + } + @Test + public void testStringBtreeHashUnCasedSettings() throws ParseException { + verifyStringDictionaryControl(Dictionary.Type.BTREE_AND_HASH, Case.UNCASED, Case.UNCASED, "dictionary { hash\nbtree\nuncased\n}"); + } + @Test + public void testStringBtreeHashCasedSettings() throws ParseException { + try { + verifyStringDictionaryControl(Dictionary.Type.BTREE_AND_HASH, Case.CASED, Case.CASED, "dictionary { btree\nhash\ncased\n}"); + } catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 'n1': Dictionary casing 'CASED' does not match field match casing 'UNCASED'", e.getMessage()); + } } @Test public void testNonNumericFieldsFailsDictionaryControl() throws ParseException { - String def = - "search test {\n" + - " document test {\n" + - " field n1 type string {\n" + - " indexing: summary | attribute\n" + - " dictionary:btree\n" + - " }\n" + - " }\n" + - "}\n"; + String def = TestUtil.joinLines( + "search test {", + " document test {", + " field n1 type bool {", + " indexing: summary | attribute", + " dictionary:btree", + " }", + " }", + "}"); try { SearchBuilder sb = SearchBuilder.createFromString(def); fail("Controlling dictionary for non-numeric fields are not yet supported."); } catch (IllegalArgumentException e) { - assertEquals("For search 'test', field 'n1': You can only specify 'dictionary:' for numeric fields", e.getMessage()); + assertEquals("For search 'test', field 'n1': You can only specify 'dictionary:' for numeric or string fields", e.getMessage()); } } @Test - public void testNonFastSearchFieldsFailsDictionaryControl() throws ParseException { - String def = - "search test {\n" + - " document test {\n" + - " field n1 type int {\n" + - " indexing: summary | attribute\n" + - " dictionary:btree\n" + - " }\n" + - " }\n" + - "}\n"; + public void testNonFastSearchNumericFieldsFailsDictionaryControl() throws ParseException { + String def = TestUtil.joinLines( + "search test {", + " document test {", + " field n1 type int {", + " indexing: summary | attribute", + " dictionary:btree", + " }", + " }", + "}"); try { SearchBuilder sb = SearchBuilder.createFromString(def); fail("Controlling dictionary for non-fast-search fields are not allowed."); @@ -140,4 +221,31 @@ public class DictionaryTestCase { assertEquals("For search 'test', field 'n1': You must specify 'attribute:fast-search' to allow dictionary control", e.getMessage()); } } + + @Test + public void testCasingForNonFastSearch() throws ParseException { + String def = TestUtil.joinLines( + "search test {", + " document test {", + " field s1 type string {", + " indexing: attribute | summary", + " }", + " field s2 type string {", + " indexing: attribute | summary", + " match:uncased", + " }", + " field s3 type string {", + " indexing: attribute | summary", + " match:cased", + " }", + " }", + "}"); + Search search = createSearch(def); + assertEquals(Case.UNCASED, search.getAttribute("s1").getCase()); + assertEquals(Case.UNCASED, search.getAttribute("s2").getCase()); + assertEquals(Case.CASED, search.getAttribute("s3").getCase()); + assertEquals(AttributesConfig.Attribute.Match.UNCASED, getConfig(search).attribute().get(0).match()); + assertEquals(AttributesConfig.Attribute.Match.UNCASED, getConfig(search).attribute().get(1).match()); + assertEquals(AttributesConfig.Attribute.Match.CASED, getConfig(search).attribute().get(2).match()); + } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java index 96f12a47a2f..b149dafab95 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java @@ -196,6 +196,112 @@ public class RankingExpressionTypeResolverTestCase { } @Test + public void testAttributeInvocationViaBoundIdentifier() throws Exception { + SearchBuilder builder = new SearchBuilder(); + builder.importString(joinLines( + "search newsarticle {", + " document newsarticle {", + " field title type string {", + " indexing {", + " input title | index", + " }", + " weight: 30", + " }", + " field usstaticrank type int {", + " indexing: summary | attribute", + " }", + " field eustaticrank type int {", + " indexing: summary | attribute", + " }", + " }", + " rank-profile default {", + " macro newsboost() { ", + " expression: 200 * matches(title)", + " }", + " macro commonboost(mystaticrank) { ", + " expression: attribute(mystaticrank) + newsboost", + " }", + " macro commonfirstphase(mystaticrank) { ", + " expression: nativeFieldMatch(title) + commonboost(mystaticrank) ", + " }", + " first-phase { expression: commonfirstphase(usstaticrank) }", + " }", + " rank-profile eurank inherits default {", + " first-phase { expression: commonfirstphase(eustaticrank) }", + " }", + "}")); + builder.build(); + RankProfile profile = builder.getRankProfileRegistry().get(builder.getSearch(), "eurank"); + } + + @Test + public void testTensorFunctionInvocationTypes_NestedSameName() throws Exception { + SearchBuilder builder = new SearchBuilder(); + builder.importString(joinLines( + "search test {", + " document test { ", + " field a type tensor(x[10],y[1]) {", + " indexing: attribute", + " }", + " field b type tensor(z[10]) {", + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " function return_a() {", + " expression: return_first(attribute(a), attribute(b))", + " }", + " function return_b() {", + " expression: return_second(attribute(a), attribute(b))", + " }", + " function return_first(e1, e2) {", + " expression: just_return(e1)", + " }", + " function just_return(e1) {", + " expression: e1", + " }", + " function return_second(e1, e2) {", + " expression: return_first(e2+0, e1)", + " }", + " summary-features {", + " return_a", + " return_b", + " }", + " }", + "}" + )); + builder.build(); + RankProfile profile = + builder.getRankProfileRegistry().get(builder.getSearch(), "my_rank_profile"); + assertEquals(TensorType.fromSpec("tensor(x[10],y[1])"), + summaryFeatures(profile).get("return_a").type(profile.typeContext(builder.getQueryProfileRegistry()))); + assertEquals(TensorType.fromSpec("tensor(z[10])"), + summaryFeatures(profile).get("return_b").type(profile.typeContext(builder.getQueryProfileRegistry()))); + } + + @Test + public void testTensorFunctionInvocationTypes_viaFuncWithExpr() throws Exception { + SearchBuilder builder = new SearchBuilder(); + builder.importString(joinLines( + "search test {", + " document test {", + " field t1 type tensor<float>(y{}) { indexing: attribute | summary }", + " field t2 type tensor<float>(x{}) { indexing: attribute | summary }", + " }", + " rank-profile test {", + " function my_func(t) { expression: sum(t, x) + 1 }", + " function test_func_via_func_with_expr() { expression: call_func_with_expr( attribute(t1), attribute(t2) ) }", + " function call_func_with_expr(a, b) { expression: my_func( a * b ) }", + " summary-features { test_func_via_func_with_expr }", + " }", + "}")); + builder.build(); + RankProfile profile = builder.getRankProfileRegistry().get(builder.getSearch(), "test"); + assertEquals(TensorType.fromSpec("tensor<float>(y{})"), + summaryFeatures(profile).get("test_func_via_func_with_expr").type(profile.typeContext(builder.getQueryProfileRegistry()))); + } + + @Test public void importedFieldsAreAvailable() throws Exception { SearchBuilder builder = new SearchBuilder(); builder.importString(joinLines( diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java index 7241b7ca5e7..84ddf4f2d51 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java @@ -103,6 +103,25 @@ public class TensorFieldTestCase { } @Test + public void tensor_with_hnsw_index_parameters_must_be_an_index() throws ParseException { + try { + createFromString(getSd(joinLines( + "field t1 type tensor(x[64]) {", + " indexing: attribute ", + " index {", + " hnsw { max-links-per-node: 32 }", + " }", + "}"))); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 't1': " + + "A tensor that specifies hnsw index parameters must also specify 'index' in 'indexing'", + e.getMessage()); + } + } + + @Test public void tensors_with_at_least_one_mapped_dimension_can_be_direct() throws ParseException { assertTrue(getAttributeFromSd( "field t1 type tensor(x{}) { indexing: attribute \n attribute: fast-search }", "t1").isFastSearch()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java index 4ce52aaf4d3..989ae87913d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java @@ -291,6 +291,10 @@ public class ClusterControllerTestCase extends DomBuilderTest { assertThat(cfg.index(), is(0)); assertThat(cfg.fleet_controller_count(), is(1)); assertThat(cfg.init_progress_time(), is(34567000)); + + Service cc = model.getService("admin/cluster-controllers/0").get(); + assertTrue(cc instanceof ClusterControllerContainer); + assertEquals("-Dio.netty.allocator.pageSize=4096 -Dio.netty.allocator.maxOrder=8", cc.getJvmOptions()); } private boolean existsHostsWithClusterControllerConfigId(VespaModel model) { @@ -392,7 +396,7 @@ public class ClusterControllerTestCase extends DomBuilderTest { assertEquals(256, qrStartConfig.jvm().heapsize()); assertEquals(0, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory()); assertEquals(2, qrStartConfig.jvm().availableProcessors()); - assertTrue(qrStartConfig.jvm().verbosegc()); + assertFalse(qrStartConfig.jvm().verbosegc()); assertEquals("-XX:+UseG1GC -XX:MaxTenuringThreshold=15", qrStartConfig.jvm().gcopts()); assertEquals(512, qrStartConfig.jvm().stacksize()); assertEquals(0, qrStartConfig.jvm().directMemorySizeCache()); @@ -437,7 +441,7 @@ public class ClusterControllerTestCase extends DomBuilderTest { assertEquals(32, qrStartConfig.jvm().minHeapsize()); // Overridden values from ClusterControllerContainerCluster assertEquals(256, qrStartConfig.jvm().heapsize()); - assertTrue(qrStartConfig.jvm().verbosegc()); + assertFalse(qrStartConfig.jvm().verbosegc()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java index a11d23dbfbb..3f211a595b9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java @@ -144,7 +144,7 @@ public class MetricsProxyContainerTest { @Test public void vespa_services_config_has_all_services() { VespaServicesConfig vespaServicesConfig = getVespaServicesConfig(servicesWithContent()); - assertEquals(6, vespaServicesConfig.service().size()); + assertEquals(7, vespaServicesConfig.service().size()); for (var service : vespaServicesConfig.service()) { if (service.configId().equals("admin/cluster-controllers/0")) { @@ -185,7 +185,7 @@ public class MetricsProxyContainerTest { private static String servicesWithContent() { return String.join("\n", "<services>", - " <admin version='4.0'>", + " <admin version='2.0'>", " <adminserver hostalias='node1'/>", " </admin>", " <content version='1.0' id='my-content'>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java index 6ad74231cae..d92ace2939a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java @@ -25,13 +25,13 @@ public class QuotaValidatorTest { @Test public void test_deploy_under_quota() { - var tester = new ValidationTester(5, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone)); + var tester = new ValidationTester(8, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone)); tester.deploy(null, getServices("testCluster", 5), Environment.prod, null); } @Test public void test_deploy_above_quota_clustersize() { - var tester = new ValidationTester(11, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone)); + var tester = new ValidationTester(14, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone)); try { tester.deploy(null, getServices("testCluster", 11), Environment.prod, null); fail(); @@ -42,32 +42,32 @@ public class QuotaValidatorTest { @Test public void test_deploy_above_quota_budget() { - var tester = new ValidationTester(10, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone)); + var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone)); try { tester.deploy(null, getServices("testCluster", 10), Environment.prod, null); fail(); } catch (RuntimeException e) { assertEquals("Please free up some capacity! This deployment's quota use ($-.--) exceeds reserved quota ($-.--)!", - ValidationTester.censorNumbers(e.getMessage())); + ValidationTester.censorNumbers(e.getMessage())); } } @Test public void test_deploy_above_quota_budget_in_publiccd() { - var tester = new ValidationTester(10, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicCdZone)); + var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicCdZone)); try { tester.deploy(null, getServices("testCluster", 10), Environment.prod, null); fail(); } catch (RuntimeException e) { assertEquals("publiccd: Please free up some capacity! This deployment's quota use ($-.--) exceeds reserved quota ($-.--)!", - ValidationTester.censorNumbers(e.getMessage())); + ValidationTester.censorNumbers(e.getMessage())); } } @Test public void test_deploy_with_negative_budget() { var quota = Quota.unlimited().withBudget(BigDecimal.valueOf(-1)); - var tester = new ValidationTester(10, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone)); + var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone)); try { tester.deploy(null, getServices("testCluster", 10), Environment.prod, null); fail(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java index b7fa72d8a64..02f1195d50e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java @@ -34,9 +34,9 @@ public class ValidationTester { private final TestProperties properties; private final InMemoryProvisioner hostProvisioner; - /** Creates a validation tester with 1 node available */ + /** Creates a validation tester with 1 node available (in addition to cluster controllers) */ public ValidationTester() { - this(1); + this(4); } /** Creates a validation tester with number of nodes available and the given test properties */ diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java index bade5a746f7..f90762e8fc0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java @@ -19,7 +19,7 @@ public class ClusterSizeReductionValidatorTest { @Test public void testSizeReductionValidation() { - ValidationTester tester = new ValidationTester(30); + ValidationTester tester = new ValidationTester(33); VespaModel previous = tester.deploy(null, getServices(30), Environment.prod, null).getFirst(); try { @@ -45,7 +45,7 @@ public class ClusterSizeReductionValidatorTest { @Test public void testOverridingSizereductionValidation() { - ValidationTester tester = new ValidationTester(30); + ValidationTester tester = new ValidationTester(33); VespaModel previous = tester.deploy(null, getServices(30), Environment.prod, null).getFirst(); tester.deploy(previous, getServices(14), Environment.prod, sizeReductionOverride); // Allowed due to override diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java index 2157839ef5c..7f92d2f409c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java @@ -6,7 +6,6 @@ import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ClusterSpec; -import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java index 392c37f50f3..30dab17635a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java @@ -17,7 +17,7 @@ import static org.junit.Assert.fail; */ public class ContentClusterRemovalValidatorTest { - private final ValidationTester tester = new ValidationTester(2); + private final ValidationTester tester = new ValidationTester(5); @Test public void testContentRemovalValidation() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java index e89f0c0a9cd..2cf8069c988 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java @@ -69,6 +69,22 @@ public class AttributeChangeValidatorTest { } @Test + public void changing_btree2hash_require_restart() throws Exception { + new Fixture("field f1 type long { indexing: attribute\n attribute: fast-search\n dictionary: btree}", + "field f1 type long { indexing: attribute\n attribute: fast-search\n dictionary: hash }"). + assertValidation(newRestartAction(ClusterSpec.Id.from("test"), + "Field 'f1' changed: change property 'dictionary: btree/hash' from 'BTREE' to 'HASH'")); + } + + @Test + public void changing_hash2btree_require_restart() throws Exception { + new Fixture("field f1 type long { indexing: attribute\n attribute: fast-search\n dictionary: hash}", + "field f1 type long { indexing: attribute\n attribute: fast-search\n dictionary: btree }"). + assertValidation(newRestartAction(ClusterSpec.Id.from("test"), + "Field 'f1' changed: change property 'dictionary: btree/hash' from 'HASH' to 'BTREE'")); + } + + @Test public void changing_fast_access_require_restart() throws Exception { new Fixture("field f1 type string { indexing: attribute \n attribute: fast-access }", "field f1 type string { indexing: attribute }"). @@ -77,6 +93,14 @@ public class AttributeChangeValidatorTest { } @Test + public void changing_uncased2cased_require_restart() throws Exception { + new Fixture("field f1 type string { indexing: attribute\n attribute: fast-search\n dictionary { btree\ncased}\nmatch:cased}", + "field f1 type string { indexing: attribute\n attribute: fast-search\n dictionary{ btree\nuncased}\nmatch:uncased }"). + assertValidation(newRestartAction(ClusterSpec.Id.from("test"), + "Field 'f1' changed: change property 'dictionary: cased/uncased' from 'CASED' to 'UNCASED'")); + } + + @Test public void changing_huge_require_restart() throws Exception { new Fixture("field f1 type string { indexing: attribute }", "field f1 type string { indexing: attribute \n attribute: huge }"). diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java index 18063bff16b..69a0c8d656c 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java @@ -122,12 +122,10 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { Admin admin = buildAdmin(servicesMultitenantAdminOnly(), true, configServerSpecs); assertThat(admin.getConfigservers().size(), is(3)); assertThat(admin.getSlobroks().size(), is(1)); - assertThat(admin.getClusterControllerHosts().size(), is(1)); assertNotNull(admin.hostSystem().getHostByHostname("test1")); for (Configserver configserver : admin.getConfigservers()) { - assertThat(configserver.getHostName(), is(not(admin.getClusterControllerHosts().get(0).getHost().getHostname()))); for (Slobrok slobrok : admin.getSlobroks()) { - assertThat(slobrok.getHostName(), is(not(configserver.getHostName()))); + assertThat(slobrok.getHostName(), is(not(configserver.getHostName()))); } } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java index 4993a51ab74..39b5cc139d9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java @@ -33,6 +33,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author gjoranv @@ -290,17 +291,47 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { @Test public void access_control_client_auth_can_be_overridden() { - Http http = createModelAndGetHttp( - " <http>", - " <filtering>", - " <access-control tls-handshake-client-auth=\"want\"/>", - " </filtering>", - " </http>"); + AthenzDomain tenantDomain = AthenzDomain.from("my-tenant-domain"); + DeployState state = new DeployState.Builder().properties( + new TestProperties() + .setAthenzDomain(tenantDomain) + .setHostedVespa(true) + .allowDisableMtls(true)) + .build(); + Http http = createModelAndGetHttp(state, + " <http>", + " <filtering>", + " <access-control tls-handshake-client-auth=\"want\"/>", + " </filtering>", + " </http>"); assertTrue(http.getAccessControl().isPresent()); assertEquals(AccessControl.ClientAuthentication.want, http.getAccessControl().get().clientAuthentication); } @Test + public void access_control_client_auth_cannot_be_overridden_when_disabled() { + AthenzDomain tenantDomain = AthenzDomain.from("my-tenant-domain"); + DeployState state = new DeployState.Builder().properties( + new TestProperties() + .setAthenzDomain(tenantDomain) + .setHostedVespa(true) + .allowDisableMtls(false)) + .build(); + + try { + Http http = createModelAndGetHttp(state, + " <http>", + " <filtering>", + " <access-control tls-handshake-client-auth=\"want\"/>", + " </filtering>", + " </http>"); + fail("Overriding tls-handshake-client-auth allowed, but should have failed"); + } catch (IllegalArgumentException e) { + assertEquals("Overriding 'tls-handshake-client-auth' for application is not allowed.", e.getMessage()); + } + } + + @Test public void local_connector_has_default_chain() { Http http = createModelAndGetHttp( " <http>", @@ -323,17 +354,20 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { } private Http createModelAndGetHttp(String... httpElement) { - List<String> servicesXml = new ArrayList<>(); - servicesXml.add("<container version='1.0'>"); - servicesXml.addAll(List.of(httpElement)); - servicesXml.add("</container>"); - AthenzDomain tenantDomain = AthenzDomain.from("my-tenant-domain"); DeployState state = new DeployState.Builder().properties( new TestProperties() .setAthenzDomain(tenantDomain) .setHostedVespa(true)) .build(); + return createModelAndGetHttp(state, httpElement); + } + private Http createModelAndGetHttp(DeployState state, String... httpElement) { + List<String> servicesXml = new ArrayList<>(); + servicesXml.add("<container version='1.0'>"); + servicesXml.addAll(List.of(httpElement)); + servicesXml.add("</container>"); + createModel(root, state, null, DomBuilderTest.parse(servicesXml.toArray(String[]::new))); return ((ApplicationContainer) root.getProducer("container/container.0")).getHttp(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java index f3f3b2b1076..f3199f6a46f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java @@ -86,6 +86,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { assertEquals("pattern", fileHandlerConfig.pattern()); assertEquals("interval", fileHandlerConfig.rotation()); assertEquals(10000, fileHandlerConfig.queueSize()); + assertEquals(4*1024*1024, fileHandlerConfig.bufferSize()); } { // json @@ -97,6 +98,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { assertEquals("pattern", fileHandlerConfig.pattern()); assertEquals("interval", fileHandlerConfig.rotation()); assertEquals(10000, fileHandlerConfig.queueSize()); + assertEquals(4*1024*1024, fileHandlerConfig.bufferSize()); } } @@ -116,6 +118,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { ConnectionLogConfig config = root.getConfig(ConnectionLogConfig.class, "default/component/com.yahoo.container.logging.FileConnectionLog"); assertEquals("default", config.cluster()); assertEquals(10000, config.queueSize()); + assertEquals(256*1024, config.bufferSize()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 811e789752e..4aadc0e3f05 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -790,8 +790,8 @@ public class ContentClusterTest extends ContentBaseTest { } @Test - public void flush_on_shutdown_is_default_off_for_hosted() throws Exception { - assertNoPreShutdownCommand(createOneNodeCluster(true)); + public void flush_on_shutdown_is_default_on_for_hosted() throws Exception { + assertPrepareRestartCommand(createOneNodeCluster(true)); } @Test @@ -1021,34 +1021,30 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(0.1, resolveMaxDeadBytesRatio(0.1), 1e-5); } - void assertZookeeperServerImplementation(String expectedClassName) { - VespaModel model = createEnd2EndOneNode(new TestProperties().setMultitenant(true)); - - ContentCluster cc = model.getContentClusters().get("storage"); - for (ClusterControllerContainer c : cc.getClusterControllers().getContainers()) { + void assertZookeeperServerImplementation(String expectedClassName, + ClusterControllerContainerCluster clusterControllerCluster) { + for (ClusterControllerContainer c : clusterControllerCluster.getContainers()) { var builder = new ComponentsConfig.Builder(); c.getConfig(builder); assertEquals(1, new ComponentsConfig(builder).components().stream() - .filter(component -> component.classId().equals(expectedClassName)) - .count()); + .filter(component -> component.classId().equals(expectedClassName)) + .count()); } } - @Test - public void reconfigurableZookeeperServerComponentsForClusterController() { - assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.ReconfigurableVespaZooKeeperServer"); - assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.Reconfigurer"); - assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.VespaZooKeeperAdminImpl"); - } - - private int resolveMaxInhibitedGroupsConfigWithFeatureFlag(int maxGroups) { - VespaModel model = createEnd2EndOneNode(new TestProperties().maxActivationInhibitedOutOfSyncGroups(maxGroups)); + private StorDistributormanagerConfig resolveStorDistributormanagerConfig(TestProperties props) { + VespaModel model = createEnd2EndOneNode(props); ContentCluster cc = model.getContentClusters().get("storage"); var builder = new StorDistributormanagerConfig.Builder(); cc.getDistributorNodes().getConfig(builder); - return (new StorDistributormanagerConfig(builder)).max_activation_inhibited_out_of_sync_groups(); + return (new StorDistributormanagerConfig(builder)); + } + + private int resolveMaxInhibitedGroupsConfigWithFeatureFlag(int maxGroups) { + var cfg = resolveStorDistributormanagerConfig(new TestProperties().maxActivationInhibitedOutOfSyncGroups(maxGroups)); + return cfg.max_activation_inhibited_out_of_sync_groups(); } @Test @@ -1057,6 +1053,22 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(2, resolveMaxInhibitedGroupsConfigWithFeatureFlag(2)); } + private int resolveNumDistributorStripesConfigWithFeatureFlag(TestProperties props) { + var cfg = resolveStorDistributormanagerConfig(props); + return cfg.num_distributor_stripes(); + } + + private int resolveNumDistributorStripesConfigWithFeatureFlag(int numStripes) { + return resolveNumDistributorStripesConfigWithFeatureFlag(new TestProperties().setNumDistributorStripes(numStripes)); + } + + @Test + public void num_distributor_stripes_config_controlled_by_properties() { + assertEquals(0, resolveNumDistributorStripesConfigWithFeatureFlag(new TestProperties())); + assertEquals(0, resolveNumDistributorStripesConfigWithFeatureFlag(0)); + assertEquals(1, resolveNumDistributorStripesConfigWithFeatureFlag(1)); + } + @Test public void testDedicatedClusterControllers() { VespaModel noContentModel = createEnd2EndOneNode(new TestProperties().setHostedVespa(true) @@ -1116,17 +1128,7 @@ public class ContentClusterTest extends ContentBaseTest { assertNull("No own cluster controller for content", twoContentModel.getContentClusters().get("dev-null").getClusterControllers()); assertNotNull("Shared cluster controller with content", twoContentModel.getAdmin().getClusterControllers()); - Map<String, ContentCluster> clustersWithOwnCCC = createEnd2EndOneNode(new TestProperties().setMultitenant(true), twoContentServices).getContentClusters(); ClusterControllerContainerCluster clusterControllers = twoContentModel.getAdmin().getClusterControllers(); - assertEquals("Union of components in own clusters is equal to those in shared cluster", - clusterControllers.getAllComponents().stream() - .map(Component::getComponentId) - .collect(toList()), - clustersWithOwnCCC.values().stream() - .flatMap(cluster -> Optional.ofNullable(cluster.getClusterControllers()).stream() - .flatMap(c -> c.getAllComponents().stream())) - .map(Component::getComponentId) - .collect(toList())); assertEquals(1, clusterControllers.reindexingContext().documentTypesForCluster("storage").size()); assertEquals(1, clusterControllers.reindexingContext().documentTypesForCluster("dev-null").size()); @@ -1136,6 +1138,13 @@ public class ContentClusterTest extends ContentBaseTest { twoContentModel.getConfig(devNullBuilder, "admin/standalone/cluster-controllers/0/components/clustercontroller-dev-null-configurer"); assertEquals(0.618, storageBuilder.build().min_distributor_up_ratio(), 1e-9); assertEquals(0.418, devNullBuilder.build().min_distributor_up_ratio(), 1e-9); + + assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.ReconfigurableVespaZooKeeperServer", + clusterControllers); + assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.Reconfigurer", + clusterControllers); + assertZookeeperServerImplementation("com.yahoo.vespa.zookeeper.VespaZooKeeperAdminImpl", + clusterControllers); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java index 5fd4885a1f2..46bd005deb6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.content; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.searchlib.TranslogserverConfig; import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; import com.yahoo.vespa.config.content.core.BucketspacesConfig; @@ -246,4 +247,16 @@ public class ContentSearchClusterTest { assertFalse(getFleetcontrollerConfig(cluster).cluster_has_global_document_types()); } + TranslogserverConfig getTlsConfig(ContentCluster cluster) { + TranslogserverConfig.Builder tlsBuilder = new TranslogserverConfig.Builder(); + cluster.getSearch().getSearchNodes().get(0).getConfig(tlsBuilder); + return tlsBuilder.build(); + } + + @Test + public void fsync_is_controllable() throws Exception { + assertTrue(getTlsConfig(createCluster(new ContentClusterBuilder().getXml())).usefsync()); + assertTrue(getTlsConfig(createCluster(new ContentClusterBuilder().syncTransactionLog(true).getXml())).usefsync()); + assertFalse(getTlsConfig(createCluster(new ContentClusterBuilder().syncTransactionLog(false).getXml())).usefsync()); + } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java index 491326fdc9c..d97aeffb107 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java @@ -28,6 +28,7 @@ public class ContentClusterBuilder { private Optional<Double> protonMemoryLimit = Optional.empty(); private Optional<Double> clusterControllerDiskLimit = Optional.empty(); private Optional<Double> clusterControllerMemoryLimit = Optional.empty(); + private Optional<Boolean> syncTransactionLog = Optional.empty(); public ContentClusterBuilder() { } @@ -42,6 +43,11 @@ public class ContentClusterBuilder { return this; } + public ContentClusterBuilder syncTransactionLog(boolean syncTransactionLog) { + this.syncTransactionLog = Optional.of(syncTransactionLog); + return this; + } + public ContentClusterBuilder searchableCopies(int searchableCopies) { this.searchableCopies = searchableCopies; return this; @@ -101,6 +107,7 @@ public class ContentClusterBuilder { " <proton>", " <searchable-copies>" + searchableCopies + "</searchable-copies>", getProtonResourceLimitsXml(" "), + getTransactionLogSyncXml(" "), " </proton>", " </engine>"); if (dispatchXml.isPresent()) { @@ -138,7 +145,11 @@ public class ContentClusterBuilder { return ""; } - private static String getXmlLine(String tag, Optional<Double> value, String indent) { + private String getTransactionLogSyncXml(String indent) { + return getXmlLine("sync-transactionlog", syncTransactionLog, indent); + } + + private static <T> String getXmlLine(String tag, Optional<T> value, String indent) { if (value.isPresent()) { return indent + "<" + tag + ">" + value.get() + "</" + tag + ">\n"; } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java index 4c1c24c9790..1aaa1669377 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java @@ -145,6 +145,7 @@ public class ModelEvaluationTest { private final String profile = "rankingExpression(imported_ml_function_small_constants_and_functions_exp_output).rankingScript: map(input, f(a)(exp(a)))\n" + + "rankingExpression(imported_ml_function_small_constants_and_functions_exp_output).type: tensor<float>(d0[3])\n" + "rankingExpression(default.output).rankingScript: join(rankingExpression(imported_ml_function_small_constants_and_functions_exp_output), reduce(join(join(reduce(rankingExpression(imported_ml_function_small_constants_and_functions_exp_output), sum, d0), tensor<float>(d0[1])(1.0), f(a,b)(a * b)), 9.999999974752427E-7, f(a,b)(a + b)), sum, d0), f(a,b)(a / b))\n" + "rankingExpression(default.output).input.type: tensor<float>(d0[3])\n" + "rankingExpression(default.output).type: tensor<float>(d0[3])\n"; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java index e270c81fe78..620b5883d29 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java @@ -38,9 +38,9 @@ public class SearchNodeTest { assertEquals(expected, cfg.basedir()); } - private void prepare(MockRoot root, SearchNode node) { + private void prepare(MockRoot root, SearchNode node, Boolean useFsync) { Host host = new Host(root, "mockhost"); - TransactionLogServer tls = new TransactionLogServer(root, "mycluster"); + TransactionLogServer tls = new TransactionLogServer(root, "mycluster", useFsync); tls.setHostResource(new HostResource(host)); tls.setBasePort(100); tls.initService(root.deployLogger()); @@ -61,10 +61,17 @@ public class SearchNodeTest { } @Test + public void requireThatSyncIsHonoured() { + assertTrue(getTlsConfig(new TestProperties(), null).usefsync()); + assertTrue(getTlsConfig(new TestProperties(), true).usefsync()); + assertFalse(getTlsConfig(new TestProperties(), false).usefsync()); + } + + @Test public void requireThatBasedirIsCorrectForElasticMode() { MockRoot root = new MockRoot(""); SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted(), false); - prepare(root, node); + prepare(root, node, true); assertBaseDir(Defaults.getDefaults().underVespaHome("var/db/vespa/search/cluster.mycluster/n3"), node); } @@ -92,10 +99,10 @@ public class SearchNodeTest { return new MockRoot("", new DeployState.Builder().properties(properties).build()); } - private TranslogserverConfig getTlsConfig(ModelContext.Properties properties) { + private TranslogserverConfig getTlsConfig(ModelContext.Properties properties, Boolean useFsync) { MockRoot root = createRoot(properties); SearchNode node = createSearchNode(root); - prepare(root, node); + prepare(root, node, useFsync); TranslogserverConfig.Builder tlsBuilder = new TranslogserverConfig.Builder(); node.getConfig(tlsBuilder); return tlsBuilder.build(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index eafbca09009..b72ae088484 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -52,7 +52,6 @@ public class VespaModelTester { private final Map<NodeResources, Collection<Host>> hostsByResources = new HashMap<>(); private ApplicationId applicationId = ApplicationId.defaultId(); private boolean useDedicatedNodeForLogserver = false; - private boolean dedicatedClusterControllerCluster = true; public VespaModelTester() { this(new NullConfigModelRegistry()); @@ -102,10 +101,6 @@ public class VespaModelTester { this.useDedicatedNodeForLogserver = useDedicatedNodeForLogserver; } - public void dedicatedClusterControllerCluster(boolean dedicatedClusterControllerCluster) { - this.dedicatedClusterControllerCluster = dedicatedClusterControllerCluster; - } - /** Creates a model which uses 0 as start index and fails on out of capacity */ public VespaModel createModel(String services, String ... retiredHostNames) { return createModel(Zone.defaultZone(), services, true, retiredHostNames); @@ -175,8 +170,7 @@ public class VespaModelTester { .setMultitenant(true) .setHostedVespa(hosted) .setApplicationId(applicationId) - .setUseDedicatedNodeForLogserver(useDedicatedNodeForLogserver) - .setDedicatedClusterControllerCluster(dedicatedClusterControllerCluster); + .setUseDedicatedNodeForLogserver(useDedicatedNodeForLogserver); DeployState.Builder deployState = deployStatebuilder .applicationPackage(appPkg) |