diff options
7 files changed, 312 insertions, 12 deletions
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java index 7c594a4e836..575d6789232 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -1,7 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentmodel; -import com.yahoo.document.*; +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.Field; +import com.yahoo.document.StructDataType; +import com.yahoo.document.StructuredDataType; import com.yahoo.document.annotation.AnnotationType; import com.yahoo.document.annotation.AnnotationTypeRegistry; import com.yahoo.document.datatypes.FieldValue; @@ -10,13 +15,24 @@ import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.FieldSet; import com.yahoo.searchdefinition.processing.BuiltInFieldSets; -import java.util.*; +import java.util.Collection; +import java.util.Collections; +import java.util.Deque; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.Map; +import java.util.Set; + +import static java.util.Collections.emptySet; /** * @author baldersheim */ public final class NewDocumentType extends StructuredDataType implements DataTypeCollection { + public static final class Name { // TODO: privatize @@ -53,13 +69,26 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp private final StructDataType header; private final StructDataType body; private final Set<FieldSet> fieldSets = new LinkedHashSet<>(); + private final Set<Name> documentReferences; public NewDocumentType(Name name) { - this(name, - new StructDataType(name.getName() + ".header"), - new StructDataType(name.getName() + ".body"), new FieldSets()); + this(name, emptySet()); + } + + public NewDocumentType(Name name, Set<Name> documentReferences) { + this( + name, + new StructDataType(name.getName() + ".header"), + new StructDataType(name.getName() + ".body"), + new FieldSets(), + documentReferences); } - public NewDocumentType(Name name, StructDataType header, StructDataType body, FieldSets fs) { + + public NewDocumentType(Name name, + StructDataType header, + StructDataType body, + FieldSets fs, + Set<Name> documentReferences) { super(name.getName()); this.name = name; this.header = header; @@ -73,6 +102,7 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp } } } + this.documentReferences = documentReferences; } public Name getFullName() { @@ -353,9 +383,13 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp /** * The field sets defined for this type and its {@link Search} - * @return fieldsets + * @return fieldsets */ public Set<FieldSet> getFieldSets() { return Collections.unmodifiableSet(fieldSets); } + + public Set<Name> getDocumentReferences() { + return documentReferences; + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java index 91d655e825a..153d324e57e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java @@ -1,23 +1,42 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; -import com.yahoo.document.*; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; +import com.yahoo.document.MapDataType; +import com.yahoo.document.ReferenceDataType; +import com.yahoo.document.StructDataType; +import com.yahoo.document.StructuredDataType; +import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; import com.yahoo.documentmodel.DataTypeCollection; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.documentmodel.VespaDocumentType; +import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; -import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferenceDataType; -import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.documentmodel.DocumentModel; import com.yahoo.vespa.documentmodel.FieldView; import com.yahoo.vespa.documentmodel.SearchDef; import com.yahoo.vespa.documentmodel.SearchField; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static java.util.Collections.emptySet; +import static java.util.stream.Collectors.toSet; /** * @author baldersheim @@ -312,7 +331,8 @@ public class DocumentModelBuilder { NewDocumentType dt = new NewDocumentType(new NewDocumentType.Name(sdoc.getName()), sdoc.getDocumentType().getHeaderType(), sdoc.getDocumentType().getBodyType(), - sdoc.getFieldSets()); + sdoc.getFieldSets(), + convertDocumentRferences(sdoc.getDocumentReferences())); for (SDDocumentType n : sdoc.getInheritedTypes()) { NewDocumentType.Name name = new NewDocumentType.Name(n.getName()); NewDocumentType inherited = model.getDocumentManager().getDocumentType(name); @@ -367,6 +387,17 @@ public class DocumentModelBuilder { extractDataTypesFromFields(dt, sdoc.fieldSet()); return dt; } + + private static Set<NewDocumentType.Name> convertDocumentRferences(Optional<DocumentReferences> documentReferences) { + if (!documentReferences.isPresent()) { + return emptySet(); + } + return documentReferences.get().referenceMap().values().stream() + .map(documentReference -> documentReference.search().getDocument()) + .map(documentType -> new NewDocumentType.Name(documentType.getName())) + .collect(toSet()); + } + private static void extractDataTypesFromFields(NewDocumentType dt, Collection<Field> fields) { for (Field f : fields) { DataType type = f.getDataType(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java new file mode 100644 index 00000000000..72a2f41fb6e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java @@ -0,0 +1,76 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.documentmodel.NewDocumentType; + +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toSet; + +/** + * Performs the following validations: + * - Verify that all global documents have required redundancy + * - Verify that all referred documents are present in services.xml + * - Verify that all referred documents are global + * + * @author bjorncs + */ +public class GlobalDistributionValidator { + + public void validate(Map<String, NewDocumentType> documentDefinitions, + Set<NewDocumentType> globallyDistributedDocuments, + Redundancy redundancy) { + + verifyGlobalDocumentsHaveRequiredRedundancy(globallyDistributedDocuments, redundancy); + verifyReferredDocumentsArePresent(documentDefinitions); + verifyReferredDocumentsAreGlobal(documentDefinitions, globallyDistributedDocuments); + } + + private static void verifyGlobalDocumentsHaveRequiredRedundancy(Set<NewDocumentType> globallyDistributedDocuments, + Redundancy redundancy) { + if (!globallyDistributedDocuments.isEmpty() && !redundancy.isEffectivelyGloballyDistributed()) { + throw new IllegalArgumentException("The following document types are marked as global, " + + "but do not have high enough redundancy to make the documents globally distributed: " + + asPrintableString(toDocumentNameStream(globallyDistributedDocuments))); + } + } + + private static void verifyReferredDocumentsArePresent(Map<String, NewDocumentType> documentDefinitions) { + Set<NewDocumentType.Name> unknowDocuments = getReferencedDocuments(documentDefinitions) + .filter(name -> !documentDefinitions.containsKey(name.toString())) + .collect(toSet()); + if (!unknowDocuments.isEmpty()) { + throw new IllegalArgumentException("The following document types are referenced from other documents, " + + "but are not listed in services.xml: " + asPrintableString(unknowDocuments.stream())); + } + } + + private static void verifyReferredDocumentsAreGlobal(Map<String, NewDocumentType> documentDefinitions, + Set<NewDocumentType> globallyDistributedDocuments) { + Set<NewDocumentType> nonGlobalReferencedDocuments = getReferencedDocuments(documentDefinitions) + .map(name -> documentDefinitions.get(name.toString())) + .filter(documentType -> !globallyDistributedDocuments.contains(documentType)) + .collect(toSet()); + if (!nonGlobalReferencedDocuments.isEmpty()) { + throw new IllegalArgumentException("The following document types are referenced from other documents, " + + "but are not globally distributed: " + asPrintableString(toDocumentNameStream(nonGlobalReferencedDocuments))); + } + } + + private static Stream<NewDocumentType.Name> getReferencedDocuments(Map<String, NewDocumentType> documentDefinitions) { + return documentDefinitions.values().stream() + .map(NewDocumentType::getDocumentReferences) + .flatMap(Set::stream); + } + + private static Stream<NewDocumentType.Name> toDocumentNameStream(Set<NewDocumentType> globallyDistributedDocuments) { + return globallyDistributedDocuments.stream().map(NewDocumentType::getFullName); + } + + private static String asPrintableString(Stream<NewDocumentType.Name> documentTypes) { + return documentTypes.map(name -> "'" + name + "'").collect(joining(", ")); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java index 918bdcb8cb7..ecc41f5156b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java @@ -51,6 +51,10 @@ public class Redundancy implements StorDistributionConfig.Producer, ProtonConfig public int effectiveFinalRedundancy() { return Math.min(totalNodes, finalRedundancy * implicitGroups); } public int effectiveReadyCopies() { return Math.min(totalNodes, readyCopies * implicitGroups); } + public boolean isEffectivelyGloballyDistributed() { + return totalNodes == effectiveFinalRedundancy(); + } + @Override public void getConfig(StorDistributionConfig.Builder builder) { builder.initial_redundancy(effectiveInitialRedundancy()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 885d35f288e..c42f44b46f1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -602,6 +602,7 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri throw new IllegalArgumentException("In indexed content cluster '" + search.getClusterName() + "': Using multi-level dispatch setup is not supported when using hierarchical distribution."); } } + new GlobalDistributionValidator().validate(documentDefinitions, globallyDistributedDocuments, redundancy); } public static Map<String, Integer> METRIC_INDEX_MAP = new TreeMap<>(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java new file mode 100644 index 00000000000..9d53e52e31e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java @@ -0,0 +1,125 @@ +package com.yahoo.vespa.model.content; + +import com.yahoo.documentmodel.NewDocumentType; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; +import static java.util.Collections.singleton; +import static java.util.Collections.singletonMap; +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author bjorncs + */ +public class GlobalDistributionValidatorTest { + + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void throws_exception_if_redudancy_does_not_imply_global_distribution() { + Map<String, NewDocumentType> documentTypes = Stream.of("foo", "bar") + .collect(toMap(identity(), name -> new NewDocumentType(new NewDocumentType.Name(name)))); + HashSet<NewDocumentType> globallyDistributedDocuments = new HashSet<>(documentTypes.values()); + Redundancy redundancy = createRedundancyWithGlobalDistributionValue(false); + + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage( + "The following document types are marked as global, " + + "but do not have high enough redundancy to make the documents globally distributed: 'bar', 'foo'"); + new GlobalDistributionValidator() + .validate(documentTypes, globallyDistributedDocuments, redundancy); + } + + @Test + public void validation_succeeds_when_globally_distributed() { + Redundancy redundancy = createRedundancyWithGlobalDistributionValue(true); + + NewDocumentType document = new NewDocumentType(new NewDocumentType.Name("foo")); + Map<String, NewDocumentType> documentTypes = singletonMap("foo", document); + Set<NewDocumentType> globallyDistributedDocuments = singleton(document); + + new GlobalDistributionValidator() + .validate(documentTypes, globallyDistributedDocuments, redundancy); + } + + @Test + public void validation_succeeds_on_no_documents() { + new GlobalDistributionValidator() + .validate(emptyMap(), emptySet(), createRedundancyWithGlobalDistributionValue(false)); + } + + @Test + public void validation_succeeds_on_no_global_documents() { + NewDocumentType document = new NewDocumentType(new NewDocumentType.Name("foo")); + Map<String, NewDocumentType> documentTypes = singletonMap("foo", document); + Redundancy redundancy = createRedundancyWithGlobalDistributionValue(false); + + new GlobalDistributionValidator() + .validate(documentTypes, emptySet(), redundancy); + } + + @Test + public void throws_exception_if_referenced_document_not_global() { + NewDocumentType parentDocument = new NewDocumentType(new NewDocumentType.Name("parent")); + NewDocumentType childDocument = new NewDocumentType( + new NewDocumentType.Name("child"), singleton(parentDocument.getFullName())); + + Map<String, NewDocumentType> documentTypes = Stream.of(parentDocument, childDocument) + .collect(toMap(doc -> doc.getFullName().toString(), identity())); + Redundancy redundancy = createRedundancyWithGlobalDistributionValue(false); + + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage( + "The following document types are referenced from other documents, but are not globally distributed: 'parent'"); + new GlobalDistributionValidator() + .validate(documentTypes, emptySet(), redundancy); + } + + @Test + public void validation_succeeds_if_referenced_document_is_global() { + NewDocumentType parentDocument = new NewDocumentType(new NewDocumentType.Name("parent")); + NewDocumentType childDocument = new NewDocumentType( + new NewDocumentType.Name("child"), singleton(parentDocument.getFullName())); + + Map<String, NewDocumentType> documentTypes = Stream.of(parentDocument, childDocument) + .collect(toMap(doc -> doc.getFullName().toString(), identity())); + Set<NewDocumentType> globallyDistributedDocuments = singleton(parentDocument); + Redundancy redundancy = createRedundancyWithGlobalDistributionValue(true); + + new GlobalDistributionValidator() + .validate(documentTypes, globallyDistributedDocuments, redundancy); + } + + @Test + public void throws_exception_on_unknown_document() { + NewDocumentType childDocument = new NewDocumentType( + new NewDocumentType.Name("child"), singleton(new NewDocumentType.Name("unknown"))); + Map<String, NewDocumentType> documentTypes = singletonMap(childDocument.getName(), childDocument); + Redundancy redundancy = createRedundancyWithGlobalDistributionValue(true); + + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage( + "The following document types are referenced from other documents, but are not listed in services.xml: 'unknown'"); + new GlobalDistributionValidator() + .validate(documentTypes, emptySet(), redundancy); + + } + + private static Redundancy createRedundancyWithGlobalDistributionValue(boolean isGloballyDistributed) { + Redundancy redundancy = mock(Redundancy.class); + when(redundancy.isEffectivelyGloballyDistributed()).thenReturn(isGloballyDistributed); + return redundancy; + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/RedundancyTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/RedundancyTest.java new file mode 100644 index 00000000000..e93295ca12a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/RedundancyTest.java @@ -0,0 +1,29 @@ +package com.yahoo.vespa.model.content; + +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author bjorncs + */ +public class RedundancyTest { + + @Test + public void effectively_globally_distributed_is_correct() { + assertFalse(createRedundancy(4, 2, 10).isEffectivelyGloballyDistributed()); + assertFalse(createRedundancy(5, 1, 10).isEffectivelyGloballyDistributed()); + assertFalse(createRedundancy(5, 2, 12).isEffectivelyGloballyDistributed()); + assertTrue(createRedundancy(5, 2, 10).isEffectivelyGloballyDistributed()); + assertTrue(createRedundancy(5, 3, 10).isEffectivelyGloballyDistributed()); + } + + private static Redundancy createRedundancy(int redundancy, int implicitGroups, int totalNodes) { + Redundancy r = new Redundancy(1, redundancy, 1); + r.setImplicitGroups(implicitGroups); + r.setTotalNodes(totalNodes); + return r; + } + +} |