diff options
Diffstat (limited to 'config-model/src/test/java/com')
14 files changed, 403 insertions, 258 deletions
diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 96e228ca1f7..79e3e869b52 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -35,7 +35,6 @@ import com.yahoo.vespa.model.test.VespaModelTester; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import com.yahoo.yolean.Exceptions; -import org.junit.Ignore; import org.junit.Test; import java.io.StringReader; @@ -896,6 +895,89 @@ public class ModelProvisioningTest { } @Test + public void testLogForwarderNotInAdminCluster() { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <admin version='4.0'>" + + " <logservers>" + + " <nodes count='1' dedicated='true'/>" + + " </logservers>" + + " <logforwarding>" + + " <splunk deployment-server='bardeplserv:123' client-name='barclinam' phone-home-interval='987' />" + + " </logforwarding>" + + " </admin>" + + " <container version='1.0' id='foo'>" + + " <nodes count='1'/>" + + " </container>" + + "</services>"; + + int numberOfHosts = 2; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(numberOfHosts+1); + + VespaModel model = tester.createModel(Zone.defaultZone(), services, true); + assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); + + Admin admin = model.getAdmin(); + Logserver logserver = admin.getLogserver(); + HostResource hostResource = logserver.getHostResource(); + + assertNotNull(hostResource.getService("logserver")); + assertNull(hostResource.getService("container")); + assertNull(hostResource.getService("logforwarder")); + + var clist = model.getContainerClusters().get("foo").getContainers(); + assertThat(clist.size(), is(1)); + hostResource = clist.get(0).getHostResource(); + assertNull(hostResource.getService("logserver")); + assertNotNull(hostResource.getService("container")); + assertNotNull(hostResource.getService("logforwarder")); + } + + + @Test + public void testLogForwarderInAdminCluster() { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <admin version='4.0'>" + + " <logservers>" + + " <nodes count='1' dedicated='true'/>" + + " </logservers>" + + " <logforwarding include-admin='true'>" + + " <splunk deployment-server='bardeplserv:123' client-name='barclinam' phone-home-interval='987' />" + + " </logforwarding>" + + " </admin>" + + " <container version='1.0' id='foo'>" + + " <nodes count='1'/>" + + " </container>" + + "</services>"; + + int numberOfHosts = 2; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(numberOfHosts+1); + + VespaModel model = tester.createModel(Zone.defaultZone(), services, true); + assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); + + Admin admin = model.getAdmin(); + Logserver logserver = admin.getLogserver(); + HostResource hostResource = logserver.getHostResource(); + + assertNotNull(hostResource.getService("logserver")); + assertNull(hostResource.getService("container")); + assertNotNull(hostResource.getService("logforwarder")); + + var clist = model.getContainerClusters().get("foo").getContainers(); + assertThat(clist.size(), is(1)); + hostResource = clist.get(0).getHostResource(); + assertNull(hostResource.getService("logserver")); + assertNotNull(hostResource.getService("container")); + assertNotNull(hostResource.getService("logforwarder")); + } + + @Test public void testImplicitLogserverContainer() { String services = "<?xml version='1.0' encoding='utf-8' ?>\n" + @@ -1018,6 +1100,35 @@ public class ModelProvisioningTest { } @Test + public void testRedundancy2DownscaledToOneNodeButOneRetired() { + String services = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + " <content version='1.0' id='bar'>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='2'/>" + + " </content>" + + "</services>"; + + int numberOfHosts = 3; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(numberOfHosts); + VespaModel model = tester.createModel(services, false, false, true, "node-1-3-10-03"); + assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); + + ContentCluster cluster = model.getContentClusters().get("bar"); + assertEquals(2, cluster.getStorageNodes().getChildren().size()); + assertEquals(1, cluster.redundancy().effectiveInitialRedundancy()); + assertEquals(1, cluster.redundancy().effectiveFinalRedundancy()); + assertEquals(1, cluster.redundancy().effectiveReadyCopies()); + assertEquals(2, cluster.getRootGroup().getNodes().size()); + assertEquals(0, cluster.getRootGroup().getSubgroups().size()); + } + + @Test public void testUsingNodesCountAttributesAndGettingTooFewNodes() { String services = "<?xml version='1.0' encoding='utf-8' ?>" + @@ -1483,7 +1594,7 @@ public class ModelProvisioningTest { assertEquals("We get 1 node per cluster and no admin node apart from the dedicated cluster controller", 3, model.getHosts().size()); assertEquals(1, model.getContainerClusters().size()); assertEquals(1, model.getContainerClusters().get("foo").getContainers().size()); - assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes()); + assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes(true)); assertEquals(1, model.getAdmin().getClusterControllers().getContainers().size()); } @@ -1536,7 +1647,7 @@ public class ModelProvisioningTest { assertEquals(6, model.getRoot().hostSystem().getHosts().size()); assertEquals(3, model.getAdmin().getSlobroks().size()); assertEquals(2, model.getContainerClusters().get("foo").getContainers().size()); - assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes()); + assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes(true)); } @Test @@ -1606,7 +1717,7 @@ public class ModelProvisioningTest { assertEquals(1, model.getRoot().hostSystem().getHosts().size()); assertEquals(1, model.getAdmin().getSlobroks().size()); assertEquals(1, model.getContainerClusters().get("foo").getContainers().size()); - assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes()); + assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes(true)); } /** Recreate the combination used in some factory tests */ @@ -1889,7 +2000,7 @@ public class ModelProvisioningTest { assertTrue("Initial servers are not joining", config.build().server().stream().noneMatch(ZookeeperServerConfig.Server::joining)); } { - VespaModel nextModel = tester.createModel(Zone.defaultZone(), servicesXml.apply(5), true, false, 0, Optional.of(model), new DeployState.Builder()); + VespaModel nextModel = tester.createModel(Zone.defaultZone(), servicesXml.apply(5), true, false, false, 0, Optional.of(model), new DeployState.Builder()); ApplicationContainerCluster cluster = nextModel.getContainerClusters().get("zk"); ZookeeperServerConfig.Builder config = new ZookeeperServerConfig.Builder(); cluster.getContainers().forEach(c -> c.getConfig(config)); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java index 91599e6f607..f8d03d3574b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import com.yahoo.vespa.objects.FieldBase; import org.junit.Test; +import static com.yahoo.config.model.test.TestUtil.joinLines; import java.util.Collection; import java.util.List; @@ -25,21 +26,17 @@ public class SummaryTestCase { @Test public void testMemorySummary() throws ParseException { - String sd = - "search memorysummary {\n" + - "\n" + - " document memorysummary {\n" + - "\n" + - " field inmemory type string {\n" + - " indexing: attribute | summary\n" + - " }\n" + - " field ondisk type string {\n" + - " indexing: index # no summary, so ignored\n" + - " }\n" + - "\n" + - " }\n" + - "\n" + - "}"; + String sd = joinLines( + "search memorysummary {", + " document memorysummary {", + " field inmemory type string {", + " indexing: attribute | summary", + " }", + " field ondisk type string {", + " indexing: index # no summary, so ignored", + " }", + " }", + "}"); DeployLoggerStub logger = new DeployLoggerStub(); SearchBuilder.createFromString(sd, logger); assertTrue(logger.entries.isEmpty()); @@ -47,25 +44,21 @@ public class SummaryTestCase { @Test public void testDiskSummary() throws ParseException { - String sd = - "search disksummary {\n" + - "\n" + - " document-summary foobar {\n" + - " summary foo1 type string { source: inmemory }\n" + - " summary foo2 type string { source: ondisk }\n" + - " }\n" + - " document disksummary {\n" + - "\n" + - " field inmemory type string {\n" + - " indexing: attribute | summary\n" + - " }\n" + - " field ondisk type string {\n" + - " indexing: index | summary\n" + - " }\n" + - "\n" + - " }\n" + - "\n" + - "}"; + String sd = joinLines( + "search disksummary {", + " document-summary foobar {", + " summary foo1 type string { source: inmemory }", + " summary foo2 type string { source: ondisk }", + " }", + " document disksummary {", + " field inmemory type string {", + " indexing: attribute | summary", + " }", + " field ondisk type string {", + " indexing: index | summary", + " }", + " }", + "}"); DeployLoggerStub logger = new DeployLoggerStub(); SearchBuilder.createFromString(sd, logger); assertEquals(1, logger.entries.size()); @@ -78,27 +71,22 @@ public class SummaryTestCase { @Test public void testDiskSummaryExplicit() throws ParseException { - String sd = - "search disksummary {\n" + - "\n" + - " document disksummary {\n" + - "\n" + - " field inmemory type string {\n" + - " indexing: attribute | summary\n" + - " }\n" + - " field ondisk type string {\n" + - " indexing: index | summary\n" + - " }\n" + - "\n" + - " }\n" + - "\n" + - " document-summary foobar {\n" + - " summary foo1 type string { source: inmemory }\n" + - " summary foo2 type string { source: ondisk }\n" + - " from-disk\n" + - " }\n" + - "\n" + - "}"; + String sd = joinLines( + "search disksummary {", + " document disksummary {", + " field inmemory type string {", + " indexing: attribute | summary", + " }", + " field ondisk type string {", + " indexing: index | summary", + " }", + " }", + " document-summary foobar {", + " summary foo1 type string { source: inmemory }", + " summary foo2 type string { source: ondisk }", + " from-disk", + " }", + "}"); DeployLoggerStub logger = new DeployLoggerStub(); SearchBuilder.createFromString(sd, logger); assertTrue(logger.entries.isEmpty()); @@ -106,31 +94,30 @@ public class SummaryTestCase { @Test public void testStructMemorySummary() throws ParseException { - String sd = - "search structmemorysummary {\n" + - " document structmemorysummary {\n" + - " struct elem {\n" + - " field name type string {}\n" + - " field weight type int {}\n" + - " }\n" + - " field elem_array type array<elem> {\n" + - " indexing: summary\n" + - " struct-field name {\n" + - " indexing: attribute\n" + - " }\n" + - " struct-field weight {\n" + - " indexing: attribute\n" + - " }\n" + - " }\n" + - " }\n" + - " document-summary filtered {\n" + - " summary elem_array_filtered type array<elem> {\n" + - " source: elem_array\n" + - " matched-elements-only\n" + - " }\n" + - " }\n" + - "\n" + - "}"; + String sd = joinLines( + "search structmemorysummary {", + " document structmemorysummary {", + " struct elem {", + " field name type string {}", + " field weight type int {}", + " }", + " field elem_array type array<elem> {", + " indexing: summary", + " struct-field name {", + " indexing: attribute", + " }", + " struct-field weight {", + " indexing: attribute", + " }", + " }", + " }", + " document-summary filtered {", + " summary elem_array_filtered type array<elem> {", + " source: elem_array", + " matched-elements-only", + " }", + " }", + "}"); DeployLoggerStub logger = new DeployLoggerStub(); SearchBuilder.createFromString(sd, logger); assertTrue(logger.entries.isEmpty()); @@ -138,40 +125,35 @@ public class SummaryTestCase { @Test public void testInheritance() throws Exception { - String sd = - "search music {\n" + - "\n" + - " document music {\n" + - " field title type string {\n" + - " indexing: summary | attribute | index\n" + - " }\n" + - " \n" + - " field artist type string {\n" + - " indexing: summary | attribute | index\n" + - " }\n" + - " \n" + - " field album type string {\n" + - " indexing: summary | attribute | index\n" + - " }\n" + - " }\n" + - " \n" + - " document-summary title {\n" + - " summary title type string {\n" + - " source: title\n" + - " }\n" + - " }\n" + - " document-summary title_artist inherits title {\n" + - " summary artist type string {\n" + - " source: artist\n" + - " }\n" + - " }\n" + - " document-summary everything inherits title_artist {\n" + - " summary album type string {\n" + - " source: album\n" + - " }\n" + - " }\n" + - "\n" + - "}"; + String sd = joinLines( + "search music {", + " document music {", + " field title type string {", + " indexing: summary | attribute | index", + " }", + " field artist type string {", + " indexing: summary | attribute | index", + " }", + " field album type string {", + " indexing: summary | attribute | index", + " }", + " }", + " document-summary title {", + " summary title type string {", + " source: title", + " }", + " }", + " document-summary title_artist inherits title {", + " summary artist type string {", + " source: artist", + " }", + " }", + " document-summary everything inherits title_artist {", + " summary album type string {", + " source: album", + " }", + " }", + "}"); var logger = new DeployLoggerStub(); var search = SearchBuilder.createFromString(sd, logger).getSearch(); assertEquals(List.of(), logger.entries); @@ -202,30 +184,27 @@ public class SummaryTestCase { @Test public void testRedeclaringInheritedFieldFails() throws Exception { - String sd = - "search music {\n" + - "\n" + - " document music {\n" + - " field title type string {\n" + - " indexing: summary | attribute | index\n" + - " }\n" + - " field title_short type string {\n" + - " indexing: summary | attribute | index\n" + - " }\n" + - " }\n" + - " \n" + - " document-summary title {\n" + - " summary title type string {\n" + - " source: title\n" + - " }\n" + - " }\n" + - " document-summary title2 inherits title {\n" + - " summary title type string {\n" + - " source: title_short\n" + - " }\n" + - " }\n" + - " \n" + - "}"; + String sd = joinLines( + "search music {", + " document music {", + " field title type string {", + " indexing: summary | attribute | index", + " }", + " field title_short type string {", + " indexing: summary | attribute | index", + " }", + " }", + " document-summary title {", + " summary title type string {", + " source: title", + " }", + " }", + " document-summary title2 inherits title {", + " summary title type string {", + " source: title_short", + " }", + " }", + "}"); var logger = new DeployLoggerStub(); try { SearchBuilder.createFromString(sd, logger); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java index afbc9f52f6b..bfc738a4f87 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java @@ -23,6 +23,37 @@ import static org.junit.Assert.assertNull; public class SummaryTestCase extends SchemaTestCase { @Test + public void deriveRawAsBase64() throws ParseException { + String sd = joinLines( + "schema s {", + " raw-as-base64-in-summary", + " document s {", + " field raw_field type raw {", + " indexing: summary", + " }", + " }", + "}"); + Search search = SearchBuilder.createFromString(sd).getSearch(); + SummaryClass summary = new SummaryClass(search, search.getSummary("default"), new BaseDeployLogger()); + assertEquals(SummaryClassField.Type.RAW, summary.getField("raw_field").getType()); + } + + @Test + public void deriveRawAsLegacy() throws ParseException { + String sd = joinLines( + "schema s {", + " document s {", + " field raw_field type raw {", + " indexing: summary", + " }", + " }", + "}"); + Search search = SearchBuilder.createFromString(sd).getSearch(); + SummaryClass summary = new SummaryClass(search, search.getSummary("default"), new BaseDeployLogger()); + assertEquals(SummaryClassField.Type.DATA, summary.getField("raw_field").getType()); + } + + @Test public void testDeriving() throws IOException, ParseException { Search search = SearchBuilder.buildFromFile("src/test/examples/simple.sd"); SummaryClass summary = new SummaryClass(search, search.getSummary("default"), new BaseDeployLogger()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java index b149dafab95..12fe7e151c0 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java @@ -57,6 +57,56 @@ public class RankingExpressionTypeResolverTestCase { } } + + @Test + public void tensorFirstPhaseFromConstantMustProduceDouble() throws Exception { + try { + SearchBuilder builder = new SearchBuilder(); + builder.importString(joinLines( + "search test {", + " document test { ", + " field a type tensor(d0[3]) {", + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " function my_func() {", + " expression: x_tensor*2.0", + " }", + " function inline other_func() {", + " expression: z_tensor+3.0", + " }", + " first-phase {", + " expression: reduce(attribute(a),sum,d0)+y_tensor+my_func+other_func", + " }", + " constants {", + " x_tensor {", + " type: tensor(x{})", + " value: { {x:bar}:17 }", + " }", + " y_tensor {", + " type: tensor(y{})", + " value: { {y:foo}:42 }", + " }", + " z_tensor {", + " type: tensor(z{})", + " value: { {z:qux}:666 }", + " }", + " }", + " }", + "}" + )); + builder.build(); + fail("Expected exception"); + } + catch (IllegalArgumentException expected) { + assertEquals("In search definition 'test', rank profile 'my_rank_profile': The first-phase expression must produce a double (a tensor with no dimensions), but produces tensor(x{},y{},z{})", + Exceptions.toMessageString(expected)); + } + } + + + @Test public void tensorSecondPhaseMustProduceDouble() throws Exception { try { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java index a1231a1418b..10ba6eff169 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java @@ -33,6 +33,26 @@ public class RankingExpressionWithTensorTestCase { } @Test + public void requireConstantTensorCanBeReferredViaConstantFeature() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: sum(constant(my_tensor))\n" + + " }\n" + + " constants {\n" + + " my_tensor {\n" + + " value: { {x:1,y:2}:1, {x:2,y:1}:2 }\n" + + " type: tensor(x{},y{})\n" + + " }\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); + f.assertRankProperty("tensor(x{},y{}):{{x:1,y:2}:1.0,{x:2,y:1}:2.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{},y{})", "constant(my_tensor).type", "my_profile"); + } + + @Test public void requireThatMultiLineConstantTensorAndTypeCanBeParsed() throws ParseException { RankProfileSearchFixture f = new RankProfileSearchFixture( " rank-profile my_profile {\n" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java index 989ae87913d..cf142dae2c7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java @@ -13,7 +13,6 @@ import com.yahoo.config.model.api.Reindexing; import com.yahoo.config.model.application.provider.SimpleApplicationValidator; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.TestDriver; import com.yahoo.config.model.test.TestRoot; @@ -393,7 +392,7 @@ public class ClusterControllerTestCase extends DomBuilderTest { model.getConfig(qrBuilder, "admin/cluster-controllers/0/components/clustercontroller-bar-configurer"); QrStartConfig qrStartConfig = new QrStartConfig(qrBuilder); assertEquals(32, qrStartConfig.jvm().minHeapsize()); - assertEquals(256, qrStartConfig.jvm().heapsize()); + assertEquals(128, qrStartConfig.jvm().heapsize()); assertEquals(0, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory()); assertEquals(2, qrStartConfig.jvm().availableProcessors()); assertFalse(qrStartConfig.jvm().verbosegc()); @@ -407,44 +406,6 @@ public class ClusterControllerTestCase extends DomBuilderTest { } @Test - public void testQrStartConfigWithFeatureFlagForMaxHeap() throws Exception { - String xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + - "<services>\n" + - "\n" + - " <admin version=\"2.0\">\n" + - " <adminserver hostalias=\"configserver\" />\n" + - " <logserver hostalias=\"logserver\" />\n" + - " <slobroks>\n" + - " <slobrok hostalias=\"configserver\" />\n" + - " <slobrok hostalias=\"logserver\" />\n" + - " </slobroks>\n" + - " </admin>\n" + - " <content version='1.0' id='bar'>" + - " <redundancy>1</redundancy>\n" + - " <documents>" + - " <document type=\"type1\" mode=\"store-only\"/>\n" + - " </documents>\n" + - " <group>" + - " <node hostalias='node0' distribution-key='0' />" + - " </group>" + - " </content>" + - "\n" + - "</services>"; - - VespaModel model = createVespaModel(xml, new DeployState.Builder().properties(new TestProperties().clusterControllerMaxHeapSizeInMb(256))); - assertTrue(model.getService("admin/cluster-controllers/0").isPresent()); - - QrStartConfig.Builder qrBuilder = new QrStartConfig.Builder(); - model.getConfig(qrBuilder, "admin/cluster-controllers/0/components/clustercontroller-bar-configurer"); - QrStartConfig qrStartConfig = new QrStartConfig(qrBuilder); - // Taken from ContainerCluster - assertEquals(32, qrStartConfig.jvm().minHeapsize()); - // Overridden values from ClusterControllerContainerCluster - assertEquals(256, qrStartConfig.jvm().heapsize()); - assertFalse(qrStartConfig.jvm().verbosegc()); - } - - @Test public void testUnconfiguredNoContent() throws Exception { String xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + "<services>\n" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java index f1f794c5057..60672c7df07 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java @@ -147,7 +147,7 @@ public class DedicatedAdminV4Test { " <admin version='4.0'>" + " <slobroks><nodes count='2' dedicated='true'/></slobroks>" + " <logservers><nodes count='1' dedicated='true'/></logservers>" + - " <logforwarding>" + + " <logforwarding include-admin='true'>" + " <splunk deployment-server='foo:123' client-name='foocli' phone-home-interval='900'/>" + " </logforwarding>" + " </admin>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java index d92ace2939a..e99a92b530a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java @@ -26,7 +26,7 @@ public class QuotaValidatorTest { @Test public void test_deploy_under_quota() { var tester = new ValidationTester(8, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone)); - tester.deploy(null, getServices("testCluster", 5), Environment.prod, null); + tester.deploy(null, getServices("testCluster", 4), Environment.prod, null); } @Test @@ -54,7 +54,7 @@ public class QuotaValidatorTest { @Test public void test_deploy_above_quota_budget_in_publiccd() { - var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicCdZone)); + var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota.withBudget(BigDecimal.ONE)).setZone(publicCdZone)); try { tester.deploy(null, getServices("testCluster", 10), Environment.prod, null); fail(); @@ -65,6 +65,19 @@ public class QuotaValidatorTest { } @Test + public void test_deploy_max_resources_above_quota() { + var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicCdZone)); + try { + tester.deploy(null, getServices("testCluster", 10), Environment.prod, null); + fail(); + } catch (RuntimeException e) { + assertEquals("publiccd: Please free up some capacity! This deployment's quota use ($-.--) exceeds reserved quota ($-.--)!", + ValidationTester.censorNumbers(e.getMessage())); + + } + } + + @Test public void test_deploy_with_negative_budget() { var quota = Quota.unlimited().withBudget(BigDecimal.valueOf(-1)); var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone)); @@ -88,7 +101,7 @@ public class QuotaValidatorTest { " <document type='music' mode='index'/>" + " </documents>" + " <nodes count='" + nodeCount + "'>" + - " <resources vcpu=\"[0.5, 1]\" memory=\"[1Gb, 3Gb]\" disk=\"[1Gb, 9Gb]\"/>\n" + + " <resources vcpu=\"[0.5, 2]\" memory=\"[1Gb, 6Gb]\" disk=\"[1Gb, 18Gb]\"/>\n" + " </nodes>" + " </content>" + "</services>"; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java index 20f5a9c841c..b1fda081b64 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java @@ -2,14 +2,12 @@ package com.yahoo.vespa.model.application.validation.change.search; import com.yahoo.config.application.api.ValidationId; -import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; import com.yahoo.vespa.model.application.validation.change.VespaReindexAction; import org.junit.Test; -import java.time.Instant; import java.util.Arrays; import java.util.List; @@ -130,7 +128,7 @@ public class IndexingScriptChangeValidatorTest { } @Test - public void requireThatAddingIndexFieldIsOk() throws Exception { + public void requireThatAddingDocumentIndexFieldIsOk() throws Exception { new Fixture("", "field f1 type string { indexing: index | summary }"). assertValidation(); } @@ -142,12 +140,22 @@ public class IndexingScriptChangeValidatorTest { } @Test - public void requireThatAddingFieldIsOk() throws Exception { + public void requireThatAddingDocumentFieldIsOk() throws Exception { new Fixture("", FIELD + " { indexing: attribute | summary }"). assertValidation(); } @Test + public void requireThatAddingExtraFieldRequiresReindexing() throws Exception { + new Fixture(" field f1 type string { indexing: index }", + " field f1 type string { indexing: index } } " + + " field f2 type string { indexing: input f1 | summary ") + .assertValidation(VespaReindexAction.of(ClusterSpec.Id.from("test"), + null, + "Non-document field 'f2' added; this may be populated by reindexing")); + } + + @Test public void requireThatAddingSummaryAspectIsOk() throws Exception { new Fixture(FIELD + " { indexing: attribute }", FIELD + " { indexing: attribute | summary }"). diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 3a3dde0cf87..ad4603e5c6b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -824,55 +824,22 @@ public class ContentBuilderTest extends DomBuilderTest { verifyThatFeatureFlagControlsVisibilityDelayDefault(0.6, 0.6); } - private void verifyThatFeatureFlagControlsUseBucketExecutorForLidSpaceCompact(boolean flag) { - DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().useBucketExecutorForLidSpaceCompact(flag)); + private void verifyThatFeatureFlagControlsUseBucketExecutorForPruneRemoved(boolean flag) { + DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().useBucketExecutorForPruneRemoved(flag)); VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() .withServices(singleNodeContentXml()) .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) .build()) .create(deployStateBuilder); ProtonConfig config = getProtonConfig(model.getContentClusters().values().iterator().next()); - assertEquals(flag, config.lidspacecompaction().usebucketexecutor()); + assertEquals(flag, config.pruneremoveddocuments().usebucketexecutor()); } - private void verifyThatFeatureFlagControlsUseBucketExecutorForBucketMove(boolean flag) { - DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().useBucketExecutorForBucketMove(flag)); - VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() - .withServices(singleNodeContentXml()) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) - .build()) - .create(deployStateBuilder); - ProtonConfig config = getProtonConfig(model.getContentClusters().values().iterator().next()); - assertEquals(flag, config.bucketmove().usebucketexecutor()); - } - - private void verifyThatFeatureFlagControlsMaxpendingMoveOps(int moveOps) { - DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().setMaxPendingMoveOps(moveOps)); - VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() - .withServices(singleNodeContentXml()) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) - .build()) - .create(deployStateBuilder); - ProtonConfig config = getProtonConfig(model.getContentClusters().values().iterator().next()); - assertEquals(moveOps, config.maintenancejobs().maxoutstandingmoveops()); - } - - @Test - public void verifyMaxPendingMoveOps() { - verifyThatFeatureFlagControlsMaxpendingMoveOps(13); - verifyThatFeatureFlagControlsMaxpendingMoveOps(107); - } - - @Test - public void verifyUseBucketExecutorForLidSpaceCompact() { - verifyThatFeatureFlagControlsUseBucketExecutorForLidSpaceCompact(true); - verifyThatFeatureFlagControlsUseBucketExecutorForLidSpaceCompact(false); - } @Test - public void verifyUseBucketExecutorForBucketMove() { - verifyThatFeatureFlagControlsUseBucketExecutorForBucketMove(true); - verifyThatFeatureFlagControlsUseBucketExecutorForBucketMove(false); + public void verifyUseBucketExecutorForPruneRemoved() { + verifyThatFeatureFlagControlsUseBucketExecutorForPruneRemoved(true); + verifyThatFeatureFlagControlsUseBucketExecutorForPruneRemoved(false); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 46026a5528c..ee9c9ccb681 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -175,7 +175,7 @@ public class ContainerClusterTest { cluster.getConfig(qrBuilder); QrStartConfig qrStartConfig = new QrStartConfig(qrBuilder); assertEquals(32, qrStartConfig.jvm().minHeapsize()); - assertEquals(256, qrStartConfig.jvm().heapsize()); + assertEquals(128, qrStartConfig.jvm().heapsize()); assertEquals(32, qrStartConfig.jvm().compressedClassSpaceSize()); assertEquals(0, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory()); root.freezeModelTopology(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 4aadc0e3f05..953c42243a6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1005,22 +1005,6 @@ public class ContentClusterTest extends ContentBaseTest { assertTrue(resolveThreePhaseUpdateConfigWithFeatureFlag(true)); } - private double resolveMaxDeadBytesRatio(double maxDeadBytesRatio) { - VespaModel model = createEnd2EndOneNode(new TestProperties().maxDeadBytesRatio(maxDeadBytesRatio)); - ContentCluster cc = model.getContentClusters().get("storage"); - ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); - cc.getSearch().getConfig(protonBuilder); - ProtonConfig protonConfig = new ProtonConfig(protonBuilder); - assertEquals(1, protonConfig.documentdb().size()); - return protonConfig.documentdb(0).allocation().max_dead_bytes_ratio(); - } - - @Test - public void default_max_dead_bytes_ratio_config_controlled_by_properties() { - assertEquals(0.2, resolveMaxDeadBytesRatio(0.2), 1e-5); - assertEquals(0.1, resolveMaxDeadBytesRatio(0.1), 1e-5); - } - void assertZookeeperServerImplementation(String expectedClassName, ClusterControllerContainerCluster clusterControllerCluster) { for (ClusterControllerContainer c : clusterControllerCluster.getContainers()) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java index fc6a4ee2783..d0196ace766 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java @@ -15,6 +15,7 @@ import com.yahoo.path.Path; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.ApplicationContainerCluster; @@ -95,6 +96,10 @@ public class ModelEvaluationTest { cluster.getConfig(cb); RankingConstantsConfig constantsConfig = new RankingConstantsConfig(cb); + OnnxModelsConfig.Builder ob = new OnnxModelsConfig.Builder(); + cluster.getConfig(ob); + OnnxModelsConfig onnxModelsConfig = new OnnxModelsConfig(ob); + assertEquals(4, config.rankprofile().size()); Set<String> modelNames = config.rankprofile().stream().map(v -> v.name()).collect(Collectors.toSet()); assertTrue(modelNames.contains("xgboost_2_2")); @@ -109,7 +114,7 @@ public class ModelEvaluationTest { assertEquals(profile, sb.toString()); ModelsEvaluator evaluator = new ModelsEvaluator(new ToleratingMissingConstantFilesRankProfilesConfigImporter(MockFileAcquirer.returnFile(null)) - .importFrom(config, constantsConfig)); + .importFrom(config, constantsConfig, onnxModelsConfig)); assertEquals(4, evaluator.models().size()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index b72ae088484..ba975e52d1a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -52,6 +52,7 @@ public class VespaModelTester { private final Map<NodeResources, Collection<Host>> hostsByResources = new HashMap<>(); private ApplicationId applicationId = ApplicationId.defaultId(); private boolean useDedicatedNodeForLogserver = false; + private HostProvisioner provisioner; public VespaModelTester() { this(new NullConfigModelRegistry()); @@ -61,6 +62,12 @@ public class VespaModelTester { this.configModelRegistry = configModelRegistry; } + public HostProvisioner provisioner() { + if (provisioner instanceof ProvisionerAdapter) + return ((ProvisionerAdapter)provisioner).provisioner(); + return provisioner; + } + /** Adds some nodes with resources 1, 3, 10 */ public Hosts addHosts(int count) { return addHosts(InMemoryProvisioner.defaultResources, count); } @@ -108,37 +115,43 @@ public class VespaModelTester { /** Creates a model which uses 0 as start index */ public VespaModel createModel(String services, boolean failOnOutOfCapacity, String ... retiredHostNames) { - return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, 0, + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, false, 0, Optional.empty(), new DeployState.Builder(), retiredHostNames); } /** Creates a model which uses 0 as start index */ public VespaModel createModel(String services, boolean failOnOutOfCapacity, DeployState.Builder builder) { - return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, 0, Optional.empty(), builder); + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, false, 0, Optional.empty(), builder); } /** Creates a model which uses 0 as start index */ public VespaModel createModel(String services, boolean failOnOutOfCapacity, boolean useMaxResources, String ... retiredHostNames) { - return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, useMaxResources, 0, + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, useMaxResources, false, 0, + Optional.empty(), new DeployState.Builder(), retiredHostNames); + } + + /** Creates a model which uses 0 as start index */ + public VespaModel createModel(String services, boolean failOnOutOfCapacity, boolean useMaxResources, boolean alwaysReturnOneNode, String ... retiredHostNames) { + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, useMaxResources, alwaysReturnOneNode, 0, Optional.empty(), new DeployState.Builder(), retiredHostNames); } /** Creates a model which uses 0 as start index */ public VespaModel createModel(String services, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) { - return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, startIndexForClusters, + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, false, startIndexForClusters, Optional.empty(), new DeployState.Builder(), retiredHostNames); } /** Creates a model which uses 0 as start index */ public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, String ... retiredHostNames) { - return createModel(zone, services, failOnOutOfCapacity, false, 0, + return createModel(zone, services, failOnOutOfCapacity, false, false, 0, Optional.empty(), new DeployState.Builder(), retiredHostNames); } /** Creates a model which uses 0 as start index */ public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, DeployState.Builder deployStateBuilder, String ... retiredHostNames) { - return createModel(zone, services, failOnOutOfCapacity, false, 0, + return createModel(zone, services, failOnOutOfCapacity, false, false, 0, Optional.empty(), deployStateBuilder, retiredHostNames); } @@ -152,15 +165,16 @@ public class VespaModelTester { * @return the resulting model */ public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, boolean useMaxResources, + boolean alwaysReturnOneNode, int startIndexForClusters, Optional<VespaModel> previousModel, DeployState.Builder deployStatebuilder, String ... retiredHostNames) { VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(null, services, ApplicationPackageUtils.generateSearchDefinition("type1")); ApplicationPackage appPkg = modelCreatorWithMockPkg.appPkg; - HostProvisioner provisioner = hosted ? - new ProvisionerAdapter(new InMemoryProvisioner(hostsByResources, + provisioner = hosted ? new ProvisionerAdapter(new InMemoryProvisioner(hostsByResources, failOnOutOfCapacity, useMaxResources, + alwaysReturnOneNode, false, startIndexForClusters, retiredHostNames)) : @@ -184,12 +198,14 @@ public class VespaModelTester { /** To verify that we don't call allocateHost(alias) in hosted environments */ private static class ProvisionerAdapter implements HostProvisioner { - private final HostProvisioner provisioner; + private final InMemoryProvisioner provisioner; - public ProvisionerAdapter(HostProvisioner provisioner) { + public ProvisionerAdapter(InMemoryProvisioner provisioner) { this.provisioner = provisioner; } + public InMemoryProvisioner provisioner() { return provisioner; } + @Override public HostSpec allocateHost(String alias) { throw new UnsupportedOperationException("Allocating hosts using <node> tags is not supported in hosted environments, " + |