From 0683a03e8acad571b476e6ed96534e903f8c4b1e Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 28 Jan 2021 10:00:00 +0100 Subject: Revert "Revert "Jonmv/reapply document protocol super config [run-systemtest]"" This reverts commit 2e2e2edeb3ea99f2c04925070cf44601e2cd94fb. --- .../vespa/model/content/ContentClusterTest.java | 38 ++++++++++- .../content/IndexedHierarchicDistributionTest.java | 6 +- .../vespa/model/content/StorageGroupTest.java | 73 ++++++++++++++++++---- .../vespa/model/routing/test/RoutingTestCase.java | 14 ++++- 4 files changed, 111 insertions(+), 20 deletions(-) (limited to 'config-model/src/test/java') diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 9f17a1c4142..095434c8e04 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -14,6 +14,7 @@ import com.yahoo.container.ComponentsConfig; import com.yahoo.messagebus.routing.RoutingTableSpec; import com.yahoo.metrics.MetricsmanagerConfig; import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; +import com.yahoo.vespa.config.content.DistributionConfig; import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.config.content.StorDistributionConfig; import com.yahoo.vespa.config.content.StorFilestorConfig; @@ -99,12 +100,22 @@ public class ContentClusterTest extends ContentBaseTest { " " + "" ); + DistributionConfig.Builder distributionBuilder = new DistributionConfig.Builder(); + cc.getConfig(distributionBuilder); + DistributionConfig distributionConfig = distributionBuilder.build(); + assertEquals(3, distributionConfig.cluster("storage").ready_copies()); + assertEquals(15, distributionConfig.cluster("storage").initial_redundancy()); + assertEquals(15, distributionConfig.cluster("storage").redundancy()); + assertEquals(4, distributionConfig.cluster("storage").group().size()); + assertEquals(1, distributionConfig.cluster().size()); + StorDistributionConfig.Builder storBuilder = new StorDistributionConfig.Builder(); cc.getConfig(storBuilder); StorDistributionConfig storConfig = new StorDistributionConfig(storBuilder); assertEquals(15, storConfig.initial_redundancy()); assertEquals(15, storConfig.redundancy()); assertEquals(3, storConfig.ready_copies()); + ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); cc.getSearch().getConfig(protonBuilder); ProtonConfig protonConfig = new ProtonConfig(protonBuilder); @@ -132,12 +143,20 @@ public class ContentClusterTest extends ContentBaseTest { " " + "" ); + DistributionConfig.Builder distributionBuilder = new DistributionConfig.Builder(); + cc.getConfig(distributionBuilder); + DistributionConfig distributionConfig = distributionBuilder.build(); + assertEquals(3, distributionConfig.cluster("storage").ready_copies()); + assertEquals(4, distributionConfig.cluster("storage").initial_redundancy()); + assertEquals(5, distributionConfig.cluster("storage").redundancy()); + StorDistributionConfig.Builder storBuilder = new StorDistributionConfig.Builder(); cc.getConfig(storBuilder); StorDistributionConfig storConfig = new StorDistributionConfig(storBuilder); assertEquals(4, storConfig.initial_redundancy()); assertEquals(5, storConfig.redundancy()); assertEquals(3, storConfig.ready_copies()); + ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); cc.getSearch().getConfig(protonBuilder); ProtonConfig protonConfig = new ProtonConfig(protonBuilder); @@ -163,8 +182,7 @@ public class ContentClusterTest extends ContentBaseTest { @Test public void testRedundancyDefaults() { - StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); - parse( + ContentCluster cc = parse( "\n" + " " + " " + @@ -173,8 +191,15 @@ public class ContentClusterTest extends ContentBaseTest { " \"" + " " + "" - ).getConfig(builder); + ); + + DistributionConfig.Builder distributionBuilder = new DistributionConfig.Builder(); + cc.getConfig(distributionBuilder); + DistributionConfig distributionConfig = distributionBuilder.build(); + assertEquals(3, distributionConfig.cluster("storage").redundancy()); + StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); + cc.getConfig(builder); StorDistributionConfig config = new StorDistributionConfig(builder); assertEquals(2, config.initial_redundancy()); assertEquals(3, config.redundancy()); @@ -548,6 +573,13 @@ public class ContentClusterTest extends ContentBaseTest { "" ); + DistributionConfig.Builder bob = new DistributionConfig.Builder(); + cluster.getConfig(bob); + DistributionConfig.Cluster.Group group = bob.build().cluster("test").group(0); + assertEquals("invalid", group.name()); + assertEquals("invalid", group.index()); + assertEquals(2, group.nodes().size()); + StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); cluster.getConfig(builder); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java index d290d4ec953..80ab6745b79 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java @@ -2,10 +2,6 @@ package com.yahoo.vespa.model.content; import com.yahoo.vespa.config.content.StorDistributionConfig; -import com.yahoo.config.model.test.MockRoot; -import com.yahoo.vespa.model.Host; -import com.yahoo.vespa.model.HostResource; -import com.yahoo.vespa.model.SimpleConfigProducer; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.search.DispatchGroup; import com.yahoo.vespa.model.search.SearchInterface; @@ -17,9 +13,9 @@ import java.util.List; import java.util.Optional; import static com.yahoo.config.model.test.TestUtil.joinLines; -import static org.hamcrest.Matchers.containsString; import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createCluster; import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createClusterXml; +import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageGroupTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageGroupTest.java index cb457cabf6c..b9495d45e08 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageGroupTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageGroupTest.java @@ -2,13 +2,14 @@ package com.yahoo.vespa.model.content; import com.yahoo.config.model.test.MockRoot; +import com.yahoo.vespa.config.content.DistributionConfig; import com.yahoo.vespa.config.content.StorDistributionConfig; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import org.junit.Test; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Test for storage groups. @@ -21,7 +22,6 @@ public class StorageGroupTest { @Test public void testSingleGroup() throws Exception { - StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); ContentCluster cluster = parse( "\n" + " " + @@ -32,8 +32,6 @@ public class StorageGroupTest { "" ); - cluster.getConfig(builder); - assertEquals("content", cluster.getStorageNodes().getChildren().get("0").getServicePropertyString("clustertype")); assertEquals("storage", cluster.getStorageNodes().getChildren().get("0").getServicePropertyString("clustername")); assertEquals("0", cluster.getStorageNodes().getChildren().get("0").getServicePropertyString("index")); @@ -42,6 +40,8 @@ public class StorageGroupTest { assertEquals("storage", cluster.getDistributorNodes().getChildren().get("0").getServicePropertyString("clustername")); assertEquals("0", cluster.getDistributorNodes().getChildren().get("0").getServicePropertyString("index")); + StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); + cluster.getConfig(builder); StorDistributionConfig config = new StorDistributionConfig(builder); assertEquals(1, config.group().size()); @@ -51,6 +51,17 @@ public class StorageGroupTest { assertEquals(0, config.group(0).nodes(0).index()); assertEquals(1, config.group(0).nodes(1).index()); //assertNotNull(cluster.getRootGroup().getNodes().get(0).getHost()); + + DistributionConfig.Builder distributionBuilder = new DistributionConfig.Builder(); + cluster.getConfig(distributionBuilder); + DistributionConfig.Cluster clusterConfig = distributionBuilder.build().cluster("storage"); + + assertEquals(1, clusterConfig.group().size()); + assertEquals("invalid", clusterConfig.group(0).index()); + assertEquals("invalid", clusterConfig.group(0).name()); + assertEquals(2, clusterConfig.group(0).nodes().size()); + assertEquals(0, clusterConfig.group(0).nodes(0).index()); + assertEquals(1, clusterConfig.group(0).nodes(1).index()); } @Test @@ -70,15 +81,14 @@ public class StorageGroupTest { " \n" + "" ); - assertTrue(false); + fail(); } catch (Exception e) { } } @Test public void testNestedGroups() throws Exception { - StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); - parse( + ContentCluster cluster = parse( "\n" + " 4" + " " + @@ -101,8 +111,10 @@ public class StorageGroupTest { " \n" + " \n" + "" - ).getConfig(builder); + ); + StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); + cluster.getConfig(builder); StorDistributionConfig config = new StorDistributionConfig(builder); assertEquals(5, config.group().size()); @@ -128,12 +140,39 @@ public class StorageGroupTest { assertEquals(5, config.group(4).nodes(1).index()); assertEquals("1|*", config.group(0).partitions()); + + DistributionConfig.Builder distributionBuilder = new DistributionConfig.Builder(); + cluster.getConfig(distributionBuilder); + DistributionConfig.Cluster clusterConfig = distributionBuilder.build().cluster("storage"); + + assertEquals(5, clusterConfig.group().size()); + assertEquals("invalid", clusterConfig.group(0).index()); + assertEquals("0", clusterConfig.group(1).index()); + assertEquals("1", clusterConfig.group(2).index()); + assertEquals("1.0", clusterConfig.group(3).index()); + assertEquals("1.1", clusterConfig.group(4).index()); + assertEquals("invalid", clusterConfig.group(0).name()); + assertEquals("sub1", clusterConfig.group(1).name()); + assertEquals("sub2", clusterConfig.group(2).name()); + assertEquals("sub3", clusterConfig.group(3).name()); + assertEquals("sub4", clusterConfig.group(4).name()); + assertEquals(2, clusterConfig.group(1).nodes().size()); + assertEquals(0, clusterConfig.group(1).nodes(0).index()); + assertEquals(1, clusterConfig.group(1).nodes(1).index()); + assertEquals(0, clusterConfig.group(2).nodes().size()); + assertEquals(2, clusterConfig.group(3).nodes().size()); + assertEquals(2, clusterConfig.group(3).nodes(0).index()); + assertEquals(3, clusterConfig.group(3).nodes(1).index()); + assertEquals(2, clusterConfig.group(4).nodes().size()); + assertEquals(4, clusterConfig.group(4).nodes(0).index()); + assertEquals(5, clusterConfig.group(4).nodes(1).index()); + + assertEquals("1|*", clusterConfig.group(0).partitions()); } @Test public void testGroupCapacity() throws Exception { - StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); - parse( + ContentCluster cluster = parse( "\n" + " 2" + " " + @@ -149,13 +188,25 @@ public class StorageGroupTest { " \n" + " \n" + "" - ).getConfig(builder); + ); + StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); + cluster.getConfig(builder); StorDistributionConfig config = new StorDistributionConfig(builder); assertEquals(3, config.group().size()); assertEquals(5.5, config.group(0).capacity(), 0.001); assertEquals(2, config.group(1).capacity(), 0.001); assertEquals(3.5, config.group(2).capacity(), 0.001); + + DistributionConfig.Builder distributionBuilder = new DistributionConfig.Builder(); + cluster.getConfig(distributionBuilder); + DistributionConfig.Cluster clusterConfig = distributionBuilder.build().cluster("storage"); + + assertEquals(3, clusterConfig.group().size()); + assertEquals(5.5, clusterConfig.group(0).capacity(), 0.001); + assertEquals(2, clusterConfig.group(1).capacity(), 0.001); + assertEquals(3.5, clusterConfig.group(2).capacity(), 0.001); } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java index 9e7370a933c..2663ad1f348 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java @@ -5,12 +5,20 @@ import com.yahoo.config.ConfigInstance; import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; import com.yahoo.io.IOUtils; import com.yahoo.messagebus.MessagebusConfig; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; import org.junit.Ignore; import org.junit.Test; -import java.io.*; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.PrintWriter; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -96,6 +104,10 @@ public class RoutingTestCase { model.getConfig(drB, ""); DocumentrouteselectorpolicyConfig dr = new DocumentrouteselectorpolicyConfig(drB); assertConfigFileContains(application, files, "documentrouteselectorpolicy.cfg", dr); + + DocumentProtocolPoliciesConfig.Builder policies = new DocumentProtocolPoliciesConfig.Builder(); + model.getConfig(policies, ""); + assertConfigFileContains(application, files, "document-protocol-policies.cfg", policies.build()); } else { StringBuilder msg = new StringBuilder(); for (String error : errors) { -- cgit v1.2.3 From 9e5ae1833e4f9638efd8736e2257690eb77e826b Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 28 Jan 2021 10:00:22 +0100 Subject: Revert "Update test and abi spec" This reverts commit e18ceeacf60f6d850a22ed8b771d6ea972a231ca. --- .../test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'config-model/src/test/java') diff --git a/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java index 2663ad1f348..d2c66b92834 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java @@ -5,7 +5,7 @@ import com.yahoo.config.ConfigInstance; import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; import com.yahoo.io.IOUtils; import com.yahoo.messagebus.MessagebusConfig; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; +import com.yahoo.messagebus.documentapi.DocumentProtocolPoliciesConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; import org.junit.Ignore; -- cgit v1.2.3 From 393145ec4ff274d7510a0a88573db4511c039158 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 28 Jan 2021 10:24:38 +0100 Subject: Update abi spec and update CMakeLists --- .../test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java | 2 +- configdefinitions/src/vespa/CMakeLists.txt | 4 ++++ .../com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java | 1 - .../documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java | 1 - .../com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java | 1 - 5 files changed, 5 insertions(+), 4 deletions(-) (limited to 'config-model/src/test/java') diff --git a/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java index d2c66b92834..2663ad1f348 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java @@ -5,7 +5,7 @@ import com.yahoo.config.ConfigInstance; import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; import com.yahoo.io.IOUtils; import com.yahoo.messagebus.MessagebusConfig; -import com.yahoo.messagebus.documentapi.DocumentProtocolPoliciesConfig; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; import org.junit.Ignore; diff --git a/configdefinitions/src/vespa/CMakeLists.txt b/configdefinitions/src/vespa/CMakeLists.txt index c6e4d612a0a..45a7fa29da8 100644 --- a/configdefinitions/src/vespa/CMakeLists.txt +++ b/configdefinitions/src/vespa/CMakeLists.txt @@ -56,6 +56,10 @@ vespa_generate_config(configdefinitions slobroks.def) install_config_definition(slobroks.def cloud.config.slobroks.def) vespa_generate_config(configdefinitions specialtokens.def) install_config_definition(specialtokens.def vespa.configdefinition.specialtokens.def) +vespa_generate_config(configdefinitions document-protocol-policies.def) +install_config_definition(document-protocol-policies.def documentapi.messagebus.protocol.document-protocol-policies.def) +vespa_generate_config(configdefinitions distribution.def) +install_config_definition(distribution.def vespa.config.content.distribution.def) vespa_generate_config(configdefinitions stor-distribution.def) install_config_definition(stor-distribution.def vespa.config.content.stor-distribution.def) vespa_generate_config(configdefinitions stor-filestor.def) diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java index 3f0ebf93d04..547d7f76dc5 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java @@ -11,7 +11,6 @@ import com.yahoo.messagebus.ErrorCode; import com.yahoo.messagebus.Protocol; import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.Routable; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; import com.yahoo.messagebus.routing.RoutingContext; import com.yahoo.messagebus.routing.RoutingNodeIterator; import com.yahoo.messagebus.routing.RoutingPolicy; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java index 879226d8229..07fd098c9b4 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java @@ -7,7 +7,6 @@ import com.yahoo.document.select.DocumentSelector; import com.yahoo.document.select.Result; import com.yahoo.document.select.parser.ParseException; import com.yahoo.messagebus.Message; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; import com.yahoo.messagebus.routing.Route; import com.yahoo.messagebus.routing.RoutingContext; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java index 16a5d760816..34124cf48db 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java @@ -2,7 +2,6 @@ package com.yahoo.documentapi.messagebus.protocol; import com.yahoo.config.subscription.ConfigSubscriber; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; import com.yahoo.messagebus.routing.Route; import com.yahoo.messagebus.routing.RoutingContext; import com.yahoo.vespa.config.content.MessagetyperouteselectorpolicyConfig; -- cgit v1.2.3