From fd040bb34edc8224cf8cb4f874d7c19f70e50318 Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Wed, 23 Aug 2017 14:23:25 +0200 Subject: Rewrite to use joinLines(). --- .../vespa/model/content/cluster/ClusterTest.java | 63 +++++++++++----------- 1 file changed, 30 insertions(+), 33 deletions(-) (limited to 'config-model') diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java index 1339fb5a5a6..60d13c3e4c7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java @@ -17,6 +17,7 @@ import org.junit.Test; import java.util.List; +import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -27,11 +28,10 @@ public class ClusterTest { @Test public void requireThatContentSearchIsApplied() throws ParseException { - ContentCluster cluster = newContentCluster( - "" + - " 1.1" + - " 2.3" + - ""); + ContentCluster cluster = newContentCluster(joinLines("", + " 1.1", + " 2.3", + "")); IndexedSearchCluster searchCluster = cluster.getSearch().getIndexed(); assertNotNull(searchCluster); assertEquals(1.1, searchCluster.getQueryTimeout(), 1E-6); @@ -44,14 +44,13 @@ public class ClusterTest { @Test public void requireThatSearchCoverageIsApplied() throws ParseException { - ContentCluster cluster = newContentCluster( - "" + - " " + - " 0.11" + - " 0.23" + - " 0.58" + - " " + - ""); + ContentCluster cluster = newContentCluster(joinLines("", + " ", + " 0.11", + " 0.23", + " 0.58", + " ", + "")); for (Dispatch tld : cluster.getSearch().getIndexed().getTLDs()) { PartitionsConfig.Builder builder = new PartitionsConfig.Builder(); tld.getConfig(builder); @@ -64,26 +63,24 @@ public class ClusterTest { private static ContentCluster newContentCluster(String contentSearchXml) throws ParseException { ApplicationPackage app = new MockApplicationPackage.Builder() - .withHosts( - "" + - " my_host" + - "") - .withServices( - "" + - " " + - " " + - " " + - " " + - " " + - " " + - " " + - " " + - " " + - " " + - " " + - contentSearchXml + - " " + - "") + .withHosts(joinLines("", + " my_host", + "")) + .withServices(joinLines("", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + contentSearchXml, + " ", + "")) .withSearchDefinitions(ApplicationPackageUtils.generateSearchDefinition("my_document")) .build(); List contents = new TestDriver().buildModel(app).getConfigModels(Content.class); -- cgit v1.2.3 From 8bb6723b6b4deeac34d8ad75930c6adf2dba924b Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Wed, 23 Aug 2017 14:43:30 +0200 Subject: Set visibility delay to zero for all global document types. This is to ensure consistency between parent and child document types in the search core when using parent-child features. --- .../vespa/model/content/ContentSearchCluster.java | 6 ++++- .../vespa/model/content/cluster/ClusterTest.java | 30 +++++++++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) (limited to 'config-model') diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 10f9983bdfb..4481c53e248 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -279,16 +279,20 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot for (NewDocumentType type : documentDefinitions.values()) { ProtonConfig.Documentdb.Builder ddbB = new ProtonConfig.Documentdb.Builder(); String docTypeName = type.getFullName().getName(); + boolean globalDocType = isGloballyDistributed(type); ddbB.inputdoctypename(docTypeName) .configid(getConfigId()) .visibilitydelay(visibilityDelay) - .global(isGloballyDistributed(type)); + .global(globalDocType); Optional ssc = findStreamingCluster(docTypeName); if (ssc.isPresent()) { ddbB.inputdoctypename(type.getFullName().getName()).configid(ssc.get().getDocumentDBConfigId()); } else if (hasIndexedCluster()) { getIndexed().fillDocumentDBConfig(type.getFullName().getName(), ddbB); } + if (globalDocType) { + ddbB.visibilitydelay(0.0); + } builder.documentdb(ddbB); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java index 60d13c3e4c7..eb62788380f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java @@ -36,9 +36,7 @@ public class ClusterTest { assertNotNull(searchCluster); assertEquals(1.1, searchCluster.getQueryTimeout(), 1E-6); assertEquals(2.3, searchCluster.getVisibilityDelay(), 1E-6); - ProtonConfig.Builder builder = new ProtonConfig.Builder(); - cluster.getSearch().getConfig(builder); - ProtonConfig proton = new ProtonConfig(builder); + ProtonConfig proton = getProtonConfig(cluster); assertEquals(searchCluster.getVisibilityDelay(), proton.documentdb(0).visibilitydelay(), 1E-6); } @@ -61,7 +59,20 @@ public class ClusterTest { } } + @Test + public void requireThatVisibilityDelayIsZeroForGlobalDocumentType() throws ParseException { + ContentCluster cluster = newContentCluster(joinLines("", + " 2.3", + ""), true); + ProtonConfig proton = getProtonConfig(cluster); + assertEquals(0.0, proton.documentdb(0).visibilitydelay(), 1E-6); + } + private static ContentCluster newContentCluster(String contentSearchXml) throws ParseException { + return newContentCluster(contentSearchXml, false); + } + + private static ContentCluster newContentCluster(String contentSearchXml, boolean globalDocType) throws ParseException { ApplicationPackage app = new MockApplicationPackage.Builder() .withHosts(joinLines("", " my_host", @@ -72,7 +83,7 @@ public class ClusterTest { " ", " ", " ", - " ", + " " + getDocumentXml(globalDocType), " ", " ", " ", @@ -88,10 +99,21 @@ public class ClusterTest { return contents.get(0).getCluster(); } + private static String getDocumentXml(boolean globalDocType) { + return ""; + } + private static SearchDefinition newSearchDefinition(String name) throws ParseException { SearchBuilder builder = new SearchBuilder(); builder.importString("search " + name + " { document " + name + " { } }"); builder.build(); return new SearchDefinition(name, builder.getSearch(name)); } + + private static ProtonConfig getProtonConfig(ContentCluster cluster) { + ProtonConfig.Builder builder = new ProtonConfig.Builder(); + cluster.getSearch().getConfig(builder); + return new ProtonConfig(builder); + } + } -- cgit v1.2.3 From 1bac4f808e6b5cc9d49cd65dda2479c6cf1faf28 Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Thu, 24 Aug 2017 14:17:54 +0200 Subject: Sort list of document types in topological order (according to document references) when producing proton config. This is to ensure that config is applied in the right order in the search backend. --- .../vespa/model/content/ContentSearchCluster.java | 2 +- .../content/TopologicalDocumentTypeSorter.java | 47 +++++++++++++++ .../model/content/ContentSearchClusterTest.java | 28 +++++++++ .../content/TopologicalDocumentTypeSorterTest.java | 68 ++++++++++++++++++++++ 4 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 config-model/src/main/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorter.java create mode 100644 config-model/src/test/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorterTest.java (limited to 'config-model') diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 4481c53e248..b54978f52d3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -276,7 +276,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot @Override public void getConfig(ProtonConfig.Builder builder) { double visibilityDelay = hasIndexedCluster() ? getIndexed().getVisibilityDelay() : 0.0; - for (NewDocumentType type : documentDefinitions.values()) { + for (NewDocumentType type : TopologicalDocumentTypeSorter.sort(documentDefinitions.values())) { ProtonConfig.Documentdb.Builder ddbB = new ProtonConfig.Documentdb.Builder(); String docTypeName = type.getFullName().getName(); boolean globalDocType = isGloballyDistributed(type); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorter.java b/config-model/src/main/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorter.java new file mode 100644 index 00000000000..9dc07275151 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorter.java @@ -0,0 +1,47 @@ +package com.yahoo.vespa.model.content; + +import com.yahoo.documentmodel.NewDocumentType; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Class that sorts a list of document types in topological order + * according to the document references between the types. + * + * Document types without any outgoing document references are considered + * to be first in the topological order. + * + * @author geirst + */ +public class TopologicalDocumentTypeSorter { + + private final Map unsortedTypes = new LinkedHashMap<>(); + private final Map sortedTypes = new LinkedHashMap<>(); + + private TopologicalDocumentTypeSorter(Collection documentTypes) { + documentTypes.stream().forEach(docType -> unsortedTypes.put(docType.getName(), docType)); + unsortedTypes.values().stream().forEach(docType -> depthFirstTraverse(docType)); + } + + private void depthFirstTraverse(NewDocumentType docType) { + // Note that cycles are not allowed and detected earlier in DocumentGraphValidator. + if (sortedTypes.containsKey(docType.getName())) { + return; + } + for (NewDocumentType.Name referenceDocTypeName : docType.getDocumentReferences()) { + NewDocumentType referenceDocType = unsortedTypes.get(referenceDocTypeName.getName()); + assert (referenceDocType != null); + depthFirstTraverse(referenceDocType); + } + sortedTypes.put(docType.getName(), docType); + } + + public static List sort(Collection documentTypes) { + TopologicalDocumentTypeSorter sorter = new TopologicalDocumentTypeSorter(documentTypes); + return sorter.sortedTypes.values().stream().collect(Collectors.toList()); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java index d2166b170da..5f18b28d6ce 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java @@ -4,10 +4,14 @@ package com.yahoo.vespa.model.content; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; +import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder; import org.junit.Test; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; +import static com.yahoo.config.model.test.TestUtil.joinLines; import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createCluster; import static com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder.createSearchDefinitions; import static junit.framework.TestCase.assertEquals; @@ -86,4 +90,28 @@ public class ContentSearchClusterTest { assertEquals(expGlobal, db.global()); } + @Test + public void require_that_document_types_with_references_are_topologically_sorted() throws Exception { + ProtonConfig cfg = getProtonConfig(createClusterWithThreeDocumentTypes()); + assertEquals(3, cfg.documentdb().size()); + assertDocumentDb("c", true, cfg.documentdb(0)); + assertDocumentDb("b", true, cfg.documentdb(1)); + assertDocumentDb("a", false, cfg.documentdb(2)); + } + + private static ContentCluster createClusterWithThreeDocumentTypes() throws Exception { + List searchDefinitions = new ArrayList<>(); + searchDefinitions.add(new SearchDefinitionBuilder().name("a") + .content(joinLines("field ref_to_b type reference { indexing: attribute }", + "field ref_to_c type reference { indexing: attribute }")).build()); + searchDefinitions.add(new SearchDefinitionBuilder().name("b") + .content("field ref_to_c type reference { indexing: attribute }").build()); + searchDefinitions.add(new SearchDefinitionBuilder().name("c").build()); + return createCluster(new ContentClusterBuilder().docTypes(Arrays.asList( + new ContentClusterBuilder.DocType("a"), + new ContentClusterBuilder.DocType("b", true), + new ContentClusterBuilder.DocType("c", true))).getXml(), + searchDefinitions); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorterTest.java new file mode 100644 index 00000000000..ddac6562612 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorterTest.java @@ -0,0 +1,68 @@ +package com.yahoo.vespa.model.content; + +import com.yahoo.documentmodel.NewDocumentType; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; + +/** + * @author geirst + */ +public class TopologicalDocumentTypeSorterTest { + + @Test + public void require_that_types_without_references_are_returned_in_input_order() { + assertOrder(Arrays.asList("a"), new DocumentTypesBuilder().add("a")); + assertOrder(Arrays.asList("a", "c", "b"), + new DocumentTypesBuilder().add("a").add("c").add("b")); + } + + @Test + public void require_that_types_with_references_are_sorted_in_topological_order() { + assertOrder(Arrays.asList("b", "a"), new DocumentTypesBuilder() + .add("a", Arrays.asList("b")) + .add("b")); + assertOrder(Arrays.asList("c", "b", "a"), new DocumentTypesBuilder() + .add("a", Arrays.asList("b", "c")) + .add("b", Arrays.asList("c")) + .add("c")); + assertOrder(Arrays.asList("b", "a", "d", "c"), new DocumentTypesBuilder() + .add("a", Arrays.asList("b")) + .add("b") + .add("c", Arrays.asList("d")) + .add("d")); + } + + private void assertOrder(List expOrder, DocumentTypesBuilder builder) { + List sortedDocTypes = TopologicalDocumentTypeSorter.sort(builder.build()); + List actOrder = sortedDocTypes.stream().map(NewDocumentType::getName).collect(Collectors.toList()); + assertEquals(expOrder, actOrder); + } + + private static class DocumentTypesBuilder { + + private final List result = new ArrayList<>(); + + public DocumentTypesBuilder add(String docTypeName) { + return add(docTypeName, Collections.emptyList()); + } + + public DocumentTypesBuilder add(String docTypeName, List docTypeNameReferences) { + Set documentReferences = + docTypeNameReferences.stream().map(NewDocumentType.Name::new).collect(Collectors.toSet()); + result.add(new NewDocumentType(new NewDocumentType.Name(docTypeName), documentReferences)); + return this; + } + + public List build() { + return result; + } + } +} -- cgit v1.2.3 From c7214bac1d5d0b89fcd001893a23fe107b056619 Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Fri, 25 Aug 2017 13:05:50 +0200 Subject: Use forEach() directly, no need to go via stream. --- .../com/yahoo/vespa/model/content/TopologicalDocumentTypeSorter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'config-model') diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorter.java b/config-model/src/main/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorter.java index 9dc07275151..4de4d498722 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorter.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorter.java @@ -23,8 +23,8 @@ public class TopologicalDocumentTypeSorter { private final Map sortedTypes = new LinkedHashMap<>(); private TopologicalDocumentTypeSorter(Collection documentTypes) { - documentTypes.stream().forEach(docType -> unsortedTypes.put(docType.getName(), docType)); - unsortedTypes.values().stream().forEach(docType -> depthFirstTraverse(docType)); + documentTypes.forEach(docType -> unsortedTypes.put(docType.getName(), docType)); + unsortedTypes.values().forEach(docType -> depthFirstTraverse(docType)); } private void depthFirstTraverse(NewDocumentType docType) { -- cgit v1.2.3 From de5afdf9b4066dcae7a48f3ba59a883a8d7a82fd Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Fri, 25 Aug 2017 13:07:19 +0200 Subject: Simplify return statement. --- .../com/yahoo/vespa/model/content/TopologicalDocumentTypeSorter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'config-model') diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorter.java b/config-model/src/main/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorter.java index 4de4d498722..0183258f13d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorter.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorter.java @@ -2,11 +2,11 @@ package com.yahoo.vespa.model.content; import com.yahoo.documentmodel.NewDocumentType; +import java.util.ArrayList; import java.util.Collection; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; /** * Class that sorts a list of document types in topological order @@ -42,6 +42,6 @@ public class TopologicalDocumentTypeSorter { public static List sort(Collection documentTypes) { TopologicalDocumentTypeSorter sorter = new TopologicalDocumentTypeSorter(documentTypes); - return sorter.sortedTypes.values().stream().collect(Collectors.toList()); + return new ArrayList<>(sorter.sortedTypes.values()); } } -- cgit v1.2.3 From c34d8ea5a1528c7f8098c1b61d0c7b8a4354fe1d Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Fri, 25 Aug 2017 13:24:03 +0200 Subject: Remove assert. --- .../com/yahoo/vespa/model/content/TopologicalDocumentTypeSorter.java | 1 - 1 file changed, 1 deletion(-) (limited to 'config-model') diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorter.java b/config-model/src/main/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorter.java index 0183258f13d..2f749a28f2d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorter.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorter.java @@ -34,7 +34,6 @@ public class TopologicalDocumentTypeSorter { } for (NewDocumentType.Name referenceDocTypeName : docType.getDocumentReferences()) { NewDocumentType referenceDocType = unsortedTypes.get(referenceDocTypeName.getName()); - assert (referenceDocType != null); depthFirstTraverse(referenceDocType); } sortedTypes.put(docType.getName(), docType); -- cgit v1.2.3