diff options
115 files changed, 1812 insertions, 1177 deletions
diff --git a/config-lib/src/main/java/com/yahoo/config/FileReference.java b/config-lib/src/main/java/com/yahoo/config/FileReference.java index 5f0bc275bad..b3dbb80f51f 100755 --- a/config-lib/src/main/java/com/yahoo/config/FileReference.java +++ b/config-lib/src/main/java/com/yahoo/config/FileReference.java @@ -7,18 +7,20 @@ import java.util.Collection; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; /** * An immutable file reference that can only be created from classes within the same package. * This is to prevent clients from creating arbitrary and invalid file references. * - * @author tonytv + * @author Tony Vaagenes */ public final class FileReference { private final String value; public FileReference(String value) { + Objects.requireNonNull(value); this.value = value; } diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index 1857cc33d64..3ef9925510c 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -14,30 +14,29 @@ <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>config-model-api</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-provisioning</artifactId> + <artifactId>fat-model-dependencies</artifactId> <version>${project.version}</version> - <scope>provided</scope> + <type>pom</type> </dependency> + <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-model</artifactId> - <version>${project.version}</version> + <!-- TODO: remove, we probably don't need version 13. --> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <version>13.0.1</version> </dependency> + <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>config-lib</artifactId> + <artifactId>config-model-api</artifactId> <version>${project.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>provided-dependencies</artifactId> + <artifactId>config-provisioning</artifactId> <version>${project.version}</version> + <scope>provided</scope> <exclusions> <exclusion> <groupId>com.google.inject</groupId> @@ -45,6 +44,8 @@ </exclusion> </exclusions> </dependency> + + <!-- TODO: remove all test deps, should not be needed --> <dependency> <groupId>org.hamcrest</groupId> <artifactId>hamcrest-all</artifactId> @@ -60,35 +61,6 @@ <artifactId>guava-testlib</artifactId> <version>17.0</version> <scope>test</scope> - </dependency> - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - <version>13.0.1</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>configdefinitions</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-application-package</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>configgen</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-bundle</artifactId> - <version>${project.version}</version> <exclusions> <exclusion> <groupId>com.yahoo.vespa</groupId> @@ -102,109 +74,9 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>simplemetrics</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>metrics</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-disc</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespajlib</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>yolean</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>testutil</artifactId> <version>${project.version}</version> <scope>test</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>documentapi</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vdslib</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>messagebus</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>document</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-core</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>linguistics</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespalog</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>statistics</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>messagebus-disc</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-messagebus</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>searchlib</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>processing</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>chain</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>docproc</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-search</artifactId> - <version>${project.version}</version> <exclusions> <exclusion> <!-- OPTIMIZATION: very large (44 MB) and only used for query sorting --> @@ -213,50 +85,6 @@ </exclusion> </exclusions> </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-search-and-docproc</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>logd</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>searchcore</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>storage</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vsm</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>indexinglanguage</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>searchsummary</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.scalatest</groupId> - <artifactId>scalatest_${scala.major-version}</artifactId> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>jdisc_http_service</artifactId> - <version>${project.version}</version> - </dependency> </dependencies> <build> diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index 72ba6de7022..c7ca1a33ff2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -2,8 +2,11 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.subscription.ConfigInstanceUtil; +import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; +import com.yahoo.document.Field; import com.yahoo.document.PositionDataType; +import com.yahoo.document.StructDataType; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.ImmutableSDField; @@ -41,18 +44,53 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce /** Derives everything from a field */ @Override protected void derive(ImmutableSDField field, Search search) { + boolean fieldIsArrayOfSimpleStruct = isArrayOfSimpleStruct(field); if (field.usesStructOrMap() && + !fieldIsArrayOfSimpleStruct && !field.getDataType().equals(PositionDataType.INSTANCE) && !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))) { return; // Ignore struct fields for indexed search (only implemented for streaming search) } if (field.isImportedField()) { deriveImportedAttributes(field); + } else if (fieldIsArrayOfSimpleStruct) { + deriveArrayOfSimpleStruct(field); } else { deriveAttributes(field); } } + private static boolean isArrayOfSimpleStruct(ImmutableSDField field) { + DataType fieldType = field.getDataType(); + if (fieldType instanceof ArrayDataType) { + ArrayDataType arrayType = (ArrayDataType)fieldType; + DataType nestedType = arrayType.getNestedType(); + if (nestedType instanceof StructDataType && + !(nestedType.equals(PositionDataType.INSTANCE))) { + StructDataType structType = (StructDataType)nestedType; + for (Field innerField : structType.getFields()) { + if (!isPrimitiveType(innerField.getDataType())) { + return false; + } + } + return true; + } else { + return false; + } + } else { + return false; + } + } + + private static boolean isPrimitiveType(DataType dataType) { + return dataType.equals(DataType.BYTE) || + dataType.equals(DataType.INT) || + dataType.equals(DataType.LONG) || + dataType.equals(DataType.FLOAT) || + dataType.equals(DataType.DOUBLE) || + dataType.equals(DataType.STRING); + } + /** Returns an attribute by name, or null if it doesn't exist */ public Attribute getAttribute(String attributeName) { return attributes.get(attributeName); @@ -98,6 +136,16 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce } } + private void deriveArrayOfSimpleStruct(ImmutableSDField field) { + for (ImmutableSDField structField : field.getStructFields()) { + for (Attribute attribute : structField.getAttributes().values()) { + if (structField.getName().equals(attribute.getName())) { + attributes.put(attribute.getName(), attribute.convertToArray()); + } + } + } + } + /** Returns a read only attribute iterator */ public Iterator attributeIterator() { return attributes().iterator(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java index f932265cb93..81e44850e71 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java @@ -147,6 +147,12 @@ public final class Attribute implements Cloneable, Serializable { this.referenceDocumentType = referenceDocumentType; } + public Attribute convertToArray() { + Attribute result = clone(); + result.collectionType = CollectionType.ARRAY; + return result; + } + /** * <p>Returns whether this attribute should be included in the "attributeprefetch" summary * which is returned to the Qrs by prefetchAttributes, used by blending, uniquing etc. @@ -181,6 +187,7 @@ public final class Attribute implements Cloneable, Serializable { public long upperBound() { return upperBound; } public double densePostingListThreshold() { return densePostingListThreshold; } public Optional<TensorType> tensorType() { return tensorType; } + public Optional<StructuredDataType> referenceDocumentType() { return referenceDocumentType; } public Sorting getSorting() { return sorting; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java index 4bdef0607a2..0661ef67131 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java @@ -21,45 +21,11 @@ import static java.util.stream.Collectors.toSet; public class GlobalDistributionValidator { public void validate(Map<String, NewDocumentType> documentDefinitions, - Set<NewDocumentType> globallyDistributedDocuments, - Redundancy redundancy, - boolean enableMultipleBucketSpaces) { - if (!enableMultipleBucketSpaces) { - verifyGlobalDocumentsHaveRequiredRedundancy(globallyDistributedDocuments, redundancy); - verifySearchableCopiesIsSameAsRedundancy(globallyDistributedDocuments, redundancy); - } + Set<NewDocumentType> globallyDistributedDocuments) { verifyReferredDocumentsArePresent(documentDefinitions); verifyReferredDocumentsAreGlobal(documentDefinitions, globallyDistributedDocuments); } - private static void verifyGlobalDocumentsHaveRequiredRedundancy(Set<NewDocumentType> globallyDistributedDocuments, - Redundancy redundancy) { - if (!globallyDistributedDocuments.isEmpty() && !redundancy.isEffectivelyGloballyDistributed()) { - throw new IllegalArgumentException( - String.format( - "The following document types are marked as global, " + - "but do not have high enough redundancy to make the documents globally distributed: %s. " + - "Redundancy is %d, expected %d.", - asPrintableString(toDocumentNameStream(globallyDistributedDocuments)), - redundancy.effectiveFinalRedundancy(), - redundancy.totalNodes())); - } - } - - private static void verifySearchableCopiesIsSameAsRedundancy(Set<NewDocumentType> globallyDistributedDocuments, - Redundancy redundancy) { - if (!globallyDistributedDocuments.isEmpty() && - redundancy.effectiveReadyCopies() != redundancy.effectiveFinalRedundancy()) { - throw new IllegalArgumentException( - String.format( - "The following document types have the number of searchable copies less than redundancy: %s. " + - "Searchable copies is %d, while redundancy is %d.", - asPrintableString(toDocumentNameStream(globallyDistributedDocuments)), - redundancy.effectiveReadyCopies(), - redundancy.effectiveFinalRedundancy())); - } - } - private static void verifyReferredDocumentsArePresent(Map<String, NewDocumentType> documentDefinitions) { Set<NewDocumentType.Name> unknowDocuments = getReferencedDocuments(documentDefinitions) .filter(name -> !documentDefinitions.containsKey(name.toString())) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 0119bced095..68ae4d2b242 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -61,14 +61,12 @@ public class ContentCluster extends AbstractConfigProducer implements MessagetyperouteselectorpolicyConfig.Producer, BucketspacesConfig.Producer { - // TODO: Make private private String documentSelection; private ContentSearchCluster search; private final boolean isHostedVespa; private final Map<String, NewDocumentType> documentDefinitions; private final Set<NewDocumentType> globallyDistributedDocuments; - // Experimental flag (TODO: remove when feature is enabled by default) - private boolean enableMultipleBucketSpaces = false; + private boolean forceEnableMultipleBucketSpaces = false; private com.yahoo.vespa.model.content.StorageGroup rootGroup; private StorageCluster storageNodes; private DistributorCluster distributorNodes; @@ -250,7 +248,7 @@ public class ContentCluster extends AbstractConfigProducer implements private void setupExperimental(ContentCluster cluster, ModelElement experimental) { Boolean enableMultipleBucketSpaces = experimental.childAsBoolean("enable-multiple-bucket-spaces"); if (enableMultipleBucketSpaces != null) { - cluster.enableMultipleBucketSpaces = enableMultipleBucketSpaces; + cluster.forceEnableMultipleBucketSpaces = enableMultipleBucketSpaces; } } @@ -596,13 +594,13 @@ public class ContentCluster extends AbstractConfigProducer implements builder.min_distributor_up_ratio(0); builder.min_storage_up_ratio(0); } - builder.enable_multiple_bucket_spaces(enableMultipleBucketSpaces); + builder.enable_multiple_bucket_spaces(true); // Telling the controller whether we actually _have_ global document types lets // it selectively enable or disable constraints that aren't needed when these // are not are present, even if full protocol and backend support is enabled // for multiple bucket spaces. Basically, if you don't use it, you don't // pay for it. - builder.cluster_has_global_document_types(enableMultipleBucketSpaces && !globallyDistributedDocuments.isEmpty()); + builder.cluster_has_global_document_types(!globallyDistributedDocuments.isEmpty()); } @Override @@ -646,7 +644,7 @@ public class ContentCluster extends AbstractConfigProducer implements } } new ReservedDocumentTypeNameValidator().validate(documentDefinitions); - new GlobalDistributionValidator().validate(documentDefinitions, globallyDistributedDocuments, redundancy, enableMultipleBucketSpaces); + new GlobalDistributionValidator().validate(documentDefinitions, globallyDistributedDocuments); } public static Map<String, Integer> METRIC_INDEX_MAP = new TreeMap<>(); @@ -727,11 +725,13 @@ public class ContentCluster extends AbstractConfigProducer implements for (NewDocumentType docType : getDocumentDefinitions().values()) { BucketspacesConfig.Documenttype.Builder docTypeBuilder = new BucketspacesConfig.Documenttype.Builder(); docTypeBuilder.name(docType.getName()); - String bucketSpace = ((enableMultipleBucketSpaces && isGloballyDistributed(docType)) - ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE); + String bucketSpace = (isGloballyDistributed(docType) ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE); docTypeBuilder.bucketspace(bucketSpace); builder.documenttype(docTypeBuilder); } - builder.enable_multiple_bucket_spaces(enableMultipleBucketSpaces); + // NOTE: this config is kept around to allow the use of multiple bucket spaces + // on older versions of Vespa. It is for all intents and purposes a no-op in + // newer versions where multiple bucket spaces are enabled by default. + builder.enable_multiple_bucket_spaces(forceEnableMultipleBucketSpaces); } } diff --git a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg new file mode 100644 index 00000000000..9e6b5cea55e --- /dev/null +++ b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg @@ -0,0 +1,40 @@ +attribute[].name "elem_array.name" +attribute[].datatype STRING +attribute[].collectiontype ARRAY +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].name "elem_array.weight" +attribute[].datatype INT32 +attribute[].collectiontype ARRAY +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false diff --git a/config-model/src/test/derived/array_of_struct_attribute/summary.cfg b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg new file mode 100644 index 00000000000..c1679c57d1a --- /dev/null +++ b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg @@ -0,0 +1,11 @@ +defaultsummaryid 252850086 +classes[].id 252850086 +classes[].name "default" +classes[].fields[].name "elem_array" +classes[].fields[].type "jsonstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "documentid" +classes[].fields[].type "longstring" diff --git a/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg b/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg new file mode 100644 index 00000000000..8956a146b74 --- /dev/null +++ b/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg @@ -0,0 +1,7 @@ +defaultoutputclass -1 +override[].field "rankfeatures" +override[].command "rankfeatures" +override[].arguments "" +override[].field "summaryfeatures" +override[].command "summaryfeatures" +override[].arguments "" diff --git a/config-model/src/test/derived/array_of_struct_attribute/test.sd b/config-model/src/test/derived/array_of_struct_attribute/test.sd new file mode 100644 index 00000000000..5b2d50cbdba --- /dev/null +++ b/config-model/src/test/derived/array_of_struct_attribute/test.sd @@ -0,0 +1,17 @@ +search test { + document test { + struct elem { + field name type string {} + field weight type int {} + } + field elem_array type array<elem> { + indexing: summary + struct-field name { + indexing: attribute + } + struct-field weight { + indexing: attribute + } + } + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java index 95d5832b70d..4ee33abfc08 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java @@ -1,12 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.document.StructDataType; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.Sorting; import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.tensor.TensorType; import org.junit.Test; import java.io.IOException; +import java.util.Optional; import static org.hamcrest.core.Is.is; import static org.junit.Assert.*; @@ -88,4 +92,48 @@ public class AttributeSettingsTestCase extends SearchDefinitionTestCase { assertTrue(attr.isFastAccess()); } + @Test + public void attribute_convert_to_array_copies_internal_state() { + StructDataType refType = new StructDataType("my_struct"); + Attribute single = new Attribute("foo", Attribute.Type.STRING, Attribute.CollectionType.SINGLE, + Optional.of(TensorType.fromSpec("tensor(x{})")), Optional.of(refType)); + single.setRemoveIfZero(true); + single.setCreateIfNonExistent(true); + single.setPrefetch(Boolean.TRUE); + single.setEnableBitVectors(true); + single.setEnableOnlyBitVector(true); + single.setFastSearch(true); + single.setHuge(true); + single.setFastAccess(true); + single.setPosition(true); + single.setArity(5); + single.setLowerBound(7); + single.setUpperBound(11); + single.setDensePostingListThreshold(13.3); + single.getSorting().setAscending(); + single.getAliases().add("foo"); + + Attribute array = single.convertToArray(); + assertEquals("foo", array.getName()); + assertEquals(Attribute.Type.STRING, array.getType()); + assertEquals(Attribute.CollectionType.ARRAY, array.getCollectionType()); + assertEquals(Optional.of(TensorType.fromSpec("tensor(x{})")), array.tensorType()); + assertSame(single.referenceDocumentType(), array.referenceDocumentType()); + assertTrue(array.isRemoveIfZero()); + assertTrue(array.isCreateIfNonExistent()); + assertTrue(array.isPrefetch()); + assertTrue(array.isEnabledBitVectors()); + assertTrue(array.isEnabledOnlyBitVector()); + assertTrue(array.isFastSearch()); + assertTrue(array.isHuge()); + assertTrue(array.isFastAccess()); + assertTrue(array.isPosition()); + assertEquals(5, array.arity()); + assertEquals(7, array.lowerBound()); + assertEquals(11, array.upperBound()); + assertEquals(13.3, array.densePostingListThreshold(), 0.00001); + assertSame(single.getSorting(), array.getSorting()); + assertSame(single.getAliases(), array.getAliases()); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java index 86f30ba3c11..990ebe7f993 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java @@ -11,6 +11,7 @@ import org.junit.Test; import java.io.IOException; import java.util.Iterator; +import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -68,4 +69,36 @@ public class AttributeListTestCase extends SearchDefinitionTestCase { assertTrue(!attributes.hasNext()); } + @Test + public void array_of_struct_field_is_derived_into_array_attributes() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/derived/array_of_struct_attribute/test.sd"); + Iterator<Attribute> attributes = new AttributeFields(search).attributeIterator(); + + assertAttribute("elem_array.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next()); + assertAttribute("elem_array.weight", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, attributes.next()); + assertTrue(!attributes.hasNext()); + } + + private static void assertAttribute(String name, Attribute.Type type, Attribute.CollectionType collection, Attribute attr) { + assertEquals(name, attr.getName()); + assertEquals(type, attr.getType()); + assertEquals(collection, attr.getCollectionType()); + } + + @Test + public void only_zcurve_attribute_is_derived_from_array_of_position_field() throws ParseException { + Search search = SearchBuilder.createFromString( + joinLines("search test {", + " document test {", + " field pos_array type array<position> {", + " indexing: attribute", + " }", + " }", + "}")).getSearch(); + Iterator<Attribute> attributes = new AttributeFields(search).attributeIterator(); + + assertAttribute("pos_array_zcurve", Attribute.Type.LONG, Attribute.CollectionType.ARRAY, attributes.next()); + assertTrue(!attributes.hasNext()); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index 4600f6ae4c6..dc2d3b7cea1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -139,4 +139,9 @@ public class ExportingTestCase extends AbstractExportingTestCase { assertCorrectDeriving("tensor"); } + @Test + public void testArrayOfStructAttribute() throws IOException, ParseException { + assertCorrectDeriving("array_of_struct_attribute"); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java index 98177b4ada0..0156128f7ca 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java @@ -173,17 +173,17 @@ public class ContentSearchClusterTest { } @Test - public void require_that_all_document_types_belong_to_default_bucket_space_by_default() throws Exception { + public void require_that_document_types_belong_to_correct_bucket_spaces() throws Exception { BucketspacesConfig config = getBucketspacesConfig(createClusterWithGlobalType()); assertEquals(2, config.documenttype().size()); - assertDocumentType("global", "default", config.documenttype(0)); + assertDocumentType("global", "global", config.documenttype(0)); assertDocumentType("regular", "default", config.documenttype(1)); // Safeguard against flipping the switch assertFalse(config.enable_multiple_bucket_spaces()); } @Test - public void require_that_multiple_bucket_spaces_can_be_enabled() throws Exception { + public void require_that_multiple_bucket_spaces_can_be_force_enabled() throws Exception { ContentCluster cluster = createClusterWithMultipleBucketSpacesEnabled(); { BucketspacesConfig config = getBucketspacesConfig(cluster); @@ -210,9 +210,9 @@ public class ContentSearchClusterTest { } @Test - public void controller_global_documents_config_forced_to_false_if_multiple_spaces_not_enabled() throws Exception { + public void controller_global_documents_config_always_enabled_even_without_experimental_flag_set() throws Exception { ContentCluster cluster = createClusterWithGlobalDocsButNotMultipleSpacesEnabled(); - assertFalse(getFleetcontrollerConfig(cluster).cluster_has_global_document_types()); + assertTrue(getFleetcontrollerConfig(cluster).cluster_has_global_document_types()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java index b8252f2f081..6506f7a08a8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java @@ -26,64 +26,16 @@ public class GlobalDistributionValidatorTest { public final ExpectedException exceptionRule = ExpectedException.none(); @Test - public void throws_exception_if_redudancy_does_not_imply_global_distribution() { - Fixture fixture = new Fixture() - .addGlobalDocument(createDocumentType("foo")) - .addGlobalDocument(createDocumentType("bar")); - Redundancy redundancy = createRedundancyWithoutGlobalDistribution(); - - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage( - "The following document types are marked as global, " + - "but do not have high enough redundancy to make the documents globally distributed: " + - "'bar', 'foo'. Redundancy is 2, expected 3."); - validate(fixture, redundancy); - } - - @Test - public void validation_of_redundancy_is_deactivated_if_multiple_bucket_spaces_is_enabled() { - Fixture fixture = new Fixture() - .addGlobalDocument(createDocumentType("foo")) - .addGlobalDocument(createDocumentType("bar")); - Redundancy redundancy = createRedundancyWithoutGlobalDistributionAndTooFewSearchableCopies(); - - validate(fixture, redundancy, true); - } - - @Test - public void throws_exception_if_searchable_copies_too_low() { - Fixture fixture = new Fixture() - .addGlobalDocument(createDocumentType("foo")) - .addGlobalDocument(createDocumentType("bar")); - Redundancy redundancy = createRedundancyWithTooFewSearchableCopies(); - - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage( - "The following document types have the number of searchable copies less than redundancy: " + - "'bar', 'foo'. Searchable copies is 1, while redundancy is 2."); - validate(fixture, redundancy); - } - - @Test - public void validation_succeeds_when_globally_distributed_and_enough_searchable_copies() { - Fixture fixture = new Fixture() - .addGlobalDocument(createDocumentType("foo")); - Redundancy redundancy = createRedundancyWithGlobalDistribution(); - validate(fixture, redundancy); - } - - @Test public void validation_succeeds_on_no_documents() { new GlobalDistributionValidator() - .validate(emptyMap(), emptySet(), createRedundancyWithoutGlobalDistribution(), false); + .validate(emptyMap(), emptySet()); } @Test public void validation_succeeds_on_no_global_documents() { Fixture fixture = new Fixture() .addNonGlobalDocument(createDocumentType("foo")); - Redundancy redundancy = createRedundancyWithoutGlobalDistribution(); - validate(fixture, redundancy); + validate(fixture); } @Test @@ -92,11 +44,10 @@ public class GlobalDistributionValidatorTest { Fixture fixture = new Fixture() .addNonGlobalDocument(parent) .addNonGlobalDocument(createDocumentType("child", parent)); - Redundancy redundancy = createRedundancyWithoutGlobalDistribution(); exceptionRule.expect(IllegalArgumentException.class); exceptionRule.expectMessage( "The following document types are referenced from other documents, but are not globally distributed: 'parent'"); - validate(fixture, redundancy); + validate(fixture); } @Test @@ -105,8 +56,7 @@ public class GlobalDistributionValidatorTest { Fixture fixture = new Fixture() .addGlobalDocument(parent) .addNonGlobalDocument(createDocumentType("child", parent)); - Redundancy redundancy = createRedundancyWithGlobalDistribution(); - validate(fixture, redundancy); + validate(fixture); } @Test @@ -115,11 +65,10 @@ public class GlobalDistributionValidatorTest { NewDocumentType child = createDocumentType("child", unknown); Fixture fixture = new Fixture() .addNonGlobalDocument(child); - Redundancy redundancy = createRedundancyWithGlobalDistribution(); exceptionRule.expect(IllegalArgumentException.class); exceptionRule.expectMessage( "The following document types are referenced from other documents, but are not listed in services.xml: 'unknown'"); - validate(fixture, redundancy); + validate(fixture); } @Test @@ -130,42 +79,14 @@ public class GlobalDistributionValidatorTest { new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/global_distribution_validation/").create(); } - private static Redundancy createRedundancyWithGlobalDistribution() { - Redundancy redundancy = new Redundancy(2, 2, 2); - redundancy.setTotalNodes(2); - return redundancy; - } - - private static Redundancy createRedundancyWithoutGlobalDistribution() { - Redundancy redundancy = new Redundancy(2, 2, 2); - redundancy.setTotalNodes(3); - return redundancy; - } - - private static Redundancy createRedundancyWithTooFewSearchableCopies() { - Redundancy redundancy = new Redundancy(2, 2, 1); - redundancy.setTotalNodes(2); - return redundancy; - } - - private static Redundancy createRedundancyWithoutGlobalDistributionAndTooFewSearchableCopies() { - Redundancy redundancy = new Redundancy(2, 2, 1); - redundancy.setTotalNodes(3); - return redundancy; - } - private static NewDocumentType createDocumentType(String name, NewDocumentType... references) { Set<NewDocumentType.Name> documentReferences = Stream.of(references).map(NewDocumentType::getFullName).collect(toSet()); return new NewDocumentType(new NewDocumentType.Name(name), documentReferences); } - private static void validate(Fixture fixture, Redundancy redundancy) { - validate(fixture, redundancy, false); - } - - private static void validate(Fixture fixture, Redundancy redundancy, boolean enableMultipleBucketSpaces) { + private static void validate(Fixture fixture) { new GlobalDistributionValidator() - .validate(fixture.getDocumentTypes(), fixture.getGloballyDistributedDocuments(), redundancy, enableMultipleBucketSpaces); + .validate(fixture.getDocumentTypes(), fixture.getGloballyDistributedDocuments()); } private static class Fixture { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/SameElementItem.java b/container-search/src/main/java/com/yahoo/prelude/query/SameElementItem.java index e1b5842529f..70e9357e7cf 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/SameElementItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/SameElementItem.java @@ -1,10 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.query; - import com.google.common.annotations.Beta; import com.yahoo.protect.Validator; +import java.nio.ByteBuffer; import java.util.Iterator; /** @@ -14,34 +14,30 @@ import java.util.Iterator; * @author baldersheim */ @Beta -public class SameElementItem extends CompositeIndexedItem { +public class SameElementItem extends CompositeItem { + + private final String fieldName; public SameElementItem(String commonPath) { - setIndexName(commonPath); + Validator.ensureNonEmpty("Field name", commonPath); + this.fieldName = commonPath; } @Override - public String getIndexedString() { - StringBuilder buf = new StringBuilder(); - - for (Iterator<Item> i = getItemIterator(); i.hasNext();) { - IndexedItem indexedItem = (IndexedItem) i.next(); - - buf.append(indexedItem.getIndexedString()); - if (i.hasNext()) { - buf.append(' '); - } - } - return buf.toString(); + protected void encodeThis(ByteBuffer buffer) { + super.encodeThis(buffer); + putString(fieldName, buffer); } + @Override protected void appendHeadingString(StringBuilder buffer) { } + @Override protected void appendBodyString(StringBuilder buffer) { - appendIndexString(buffer); + buffer.append(fieldName).append(':'); buffer.append('{'); for (Iterator<Item> i = getItemIterator(); i.hasNext();) { TermItem term = (TermItem) i.next(); - buffer.append(term.getIndexName()).append(':').append(term.getIndexedString()); + buffer.append(extractSubFieldName(term)).append(':').append(term.getIndexedString()); if (i.hasNext()) { buffer.append(' '); } @@ -50,16 +46,14 @@ public class SameElementItem extends CompositeIndexedItem { } @Override - public int getNumWords() { - return getItemCount(); - } - - @Override protected void adding(Item item) { Validator.ensureInstanceOf("Child item", item, TermItem.class); TermItem asTerm = (TermItem) item; Validator.ensureNonEmpty("Struct fieldname", asTerm.getIndexName()); Validator.ensureNonEmpty("Query term", asTerm.getIndexedString()); + Validator.ensure("Struct fieldname starts with '" + getFieldName() + ".'", + !asTerm.getIndexName().startsWith(fieldName+".")); + item.setIndexName(fieldName + '.' + asTerm.getIndexName()); } @Override public ItemType getItemType() { @@ -70,4 +64,8 @@ public class SameElementItem extends CompositeIndexedItem { public String getName() { return getItemType().toString(); } + public String getFieldName() { return fieldName; } + public String extractSubFieldName(TermItem full) { + return full.getIndexName().substring(getFieldName().length()+1); + } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java index 0384dfdca12..708c48f0954 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java @@ -42,11 +42,7 @@ public final class WeakAndItem extends NonReducibleCompositeItem { **/ public WeakAndItem(String index, int N) { this.N = N; - if (index == null) { - this.index = ""; - } else { - this.index = index; - } + this.index = (index == null) ? "" : index; } public WeakAndItem(int N) { this("", N); @@ -54,12 +50,7 @@ public final class WeakAndItem extends NonReducibleCompositeItem { /** Sets the index name of all subitems of this */ public void setIndexName(String index) { - String toSet; - if (index == null) { - toSet = ""; - } else { - toSet = index; - } + String toSet = (index == null) ? "" : index; super.setIndexName(toSet); this.index = toSet; } diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index 4c5dfaabed7..12aec81a5f8 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -594,26 +594,22 @@ public class VespaSerializer { static boolean serialize(StringBuilder destination, Item item, boolean includeField) { - SameElementItem phrase = (SameElementItem) item; - String annotations = leafAnnotations(phrase); + SameElementItem sameElement = (SameElementItem) item; if (includeField) { - destination.append(normalizeIndexName(phrase.getIndexName())).append(" contains "); - - } - if (annotations.length() > 0) { - destination.append("([{").append(annotations).append("}]"); + destination.append(normalizeIndexName(sameElement.getFieldName())).append(" contains "); } destination.append(SAME_ELEMENT).append('('); - for (int i = 0; i < phrase.getItemCount(); ++i) { + for (int i = 0; i < sameElement.getItemCount(); ++i) { if (i > 0) { destination.append(", "); } - Item current = phrase.getItem(i); + Item current = sameElement.getItem(i); if (current instanceof WordItem) { - new WordSerializer().serialize(destination, current); - + WordItem modified = (WordItem)current.clone(); + modified.setIndexName(sameElement.extractSubFieldName(modified)); + new WordSerializer().serialize(destination, modified); } else { throw new IllegalArgumentException( "Serializing of " + current.getClass().getSimpleName() @@ -621,9 +617,7 @@ public class VespaSerializer { } } destination.append(')'); - if (annotations.length() > 0) { - destination.append(')'); - } + return false; } diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 292bb6d0f5a..c6097b1bc73 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -542,7 +542,7 @@ public class YqlParser implements Parser { for (OperatorNode<ExpressionOperator> word : ast.<List<OperatorNode<ExpressionOperator>>> getArgument(1)) { sameElement.addItem(buildTermSearch(word)); } - return leafStyleSettings(ast, sameElement); + return sameElement; } @NonNull diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/SameElementItemTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/SameElementItemTestCase.java index ff3ca53319f..01c03fcd802 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/test/SameElementItemTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/SameElementItemTestCase.java @@ -22,6 +22,19 @@ public class SameElementItemTestCase { s.addItem(new WordItem("b", "f1")); s.addItem(new WordItem("c")); } + @Test + public void requireAllowCommonPrefix() { + SameElementItem s = new SameElementItem("structa"); + s.addItem(new WordItem("b", "f1")); + s.addItem(new WordItem("c", "structaf2")); + assertEquals("structa:{f1:b structaf2:c}", s.toString()); + } + @Test(expected = IllegalArgumentException.class) + public void requireNoChildrenHasCommonPrefixWithDot() { + SameElementItem s = new SameElementItem("structa"); + s.addItem(new WordItem("b", "f1")); + s.addItem(new WordItem("c", "structa.f2")); + } @Test(expected = IllegalArgumentException.class) public void requireAllChildrenHaveNonEmptyTerm() { SameElementItem s = new SameElementItem("structa"); diff --git a/fat-model-dependencies/OWNERS b/fat-model-dependencies/OWNERS new file mode 100644 index 00000000000..d34761f1ba5 --- /dev/null +++ b/fat-model-dependencies/OWNERS @@ -0,0 +1,2 @@ +gjoranv +hmusum diff --git a/fat-model-dependencies/README b/fat-model-dependencies/README new file mode 100644 index 00000000000..ba71b189db9 --- /dev/null +++ b/fat-model-dependencies/README @@ -0,0 +1,4 @@ +This module contains all dependencies that must be embedded in the config-model-fat bundle. +This artifact should be depended on by config-model-fat and all amended versions of the +fat config model. This allows pulling in the same set of dependencies without duplication +in pom.xml. diff --git a/fat-model-dependencies/pom.xml b/fat-model-dependencies/pom.xml new file mode 100644 index 00000000000..1415ca6e5aa --- /dev/null +++ b/fat-model-dependencies/pom.xml @@ -0,0 +1,223 @@ +<?xml version="1.0"?> +<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>fat-model-dependencies</artifactId> + <packaging>pom</packaging> + <version>6-SNAPSHOT</version> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-model</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <!-- Large, and installed separately as part of Vespa --> + <groupId>org.tensorflow</groupId> + <artifactId>libtensorflow_jni</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-lib</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>provided-dependencies</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>configdefinitions</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-application-package</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>configgen</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-bundle</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jrt</artifactId> + </exclusion> + <exclusion> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-lib</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>simplemetrics</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>metrics</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-disc</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>yolean</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>documentapi</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vdslib</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>messagebus</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>document</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>linguistics</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespalog</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>statistics</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>messagebus-disc</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-messagebus</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>searchlib</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>processing</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>chain</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>docproc</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-search</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <!-- OPTIMIZATION: very large (44 MB) and only used for query sorting --> + <groupId>com.ibm.icu</groupId> + <artifactId>icu4j</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-search-and-docproc</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>logd</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>searchcore</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>storage</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vsm</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>indexinglanguage</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>searchsummary</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.scalatest</groupId> + <artifactId>scalatest_${scala.major-version}</artifactId> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jdisc_http_service</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> +</project> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java index d2c09aae22a..71e55c36284 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java @@ -2,31 +2,33 @@ package com.yahoo.vespa.hosted.node.admin.component; /** - * This class is thread unsafe: All method calls MUST be exclusive and serialized. + * <p>This class is thread unsafe: All method calls MUST be exclusive and serialized.</p> * - * In a specialized environment it is possible to provide a richer context than TaskContext: - * - Define a subclass T of TaskContext with the additional functionality. - * - Define task classes that implement IdempotentTask<T>. + * <dl> + * <dt>In a specialized environment it is possible to provide a richer context than TaskContext:</dt> + * <dd>- Define a subclass T of TaskContext with the additional functionality.</dd> + * <dd>- Define task classes that implement IdempotentTask<T>.</dd> + * </dl> */ public interface IdempotentTask<T extends TaskContext> { /** - * A short id of the task to e.g. identify the task in the log. + * <p>A short id of the task to e.g. identify the task in the log.</p> * - * Prefer PascalCase and without white-space. + * <p>Prefer PascalCase and without white-space.</p> * - * Example: "EnableDocker" + * <p>Example: "EnableDocker"</p> */ default String name() { return getClass().getSimpleName(); } /** - * Execute an administrative task to converge towards some ideal state, whether it is - * system state or in-memory Java state. + * <p>Execute an administrative task to converge towards some ideal state, whether it is + * system state or in-memory Java state.</p> * - * converge() must be idempotent: it may be called any number of times, or - * interrupted at any time e.g. by `kill -9`. + * <p>converge() must be idempotent: it may be called any number of times, or + * interrupted at any time e.g. by `kill -9`.</p> * - * converge() is not thread safe: The caller must ensure there is at most one invocation - * of converge() at any given time. + * <p>converge() is not thread safe: The caller must ensure there is at most one invocation + * of converge() at any given time.</p> * * @return false if already converged, i.e. was a no-op. A typical sequence of converge() * calls on a IdempotentTask will consist of: @@ -38,4 +40,22 @@ public interface IdempotentTask<T extends TaskContext> { * @throws RuntimeException (or a subclass) if the task is unable to converge. */ boolean converge(T context); + + /** + * <p>Converge the task towards some state where it can be suspended. The + * TaskContext should provide enough to determine what kind of suspend is wanted, e.g. + * suspension of only the task, or the task and the resources/processes it manages.</p> + * + * <p>convergeSuspend() must be idempotent: it may be called any number of times, or + * interrupted at any time e.g. by `kill -9`.</p> + * + * <p>convergeSuspend() is not thread safe: The caller must ensure there is at most one + * invocation of convergeSuspend() at any given time.</p> + * + * @return false if already converged, i.e. was a no-op + * @throws RuntimeException (or a subclass) if the task is unable to suspend. + */ + default boolean convergeSuspend(T context) { + return false; + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index a41afa2225f..f7e9c3ca1d8 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -171,7 +171,7 @@ public class AthenzCredentialsMaintainer { } private boolean isCertificateExpired(Instant expiry, Instant now) { - return expiry.minus(EXPIRY_MARGIN).isAfter(now); + return now.isAfter(expiry.minus(EXPIRY_MARGIN)); } private void registerIdentity(VespaUniqueInstanceId instanceId, Set<String> ipAddresses) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java index 98394f52857..427588c287d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java @@ -43,6 +43,7 @@ import static com.yahoo.vespa.hosted.node.admin.provider.NodeAdminStateUpdater.S */ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { static final Duration FREEZE_CONVERGENCE_TIMEOUT = Duration.ofMinutes(5); + static final String TRANSITION_EXCEPTION_MESSAGE = "NodeAdminStateUpdater has not run since current wanted state was set"; private final AtomicBoolean terminated = new AtomicBoolean(false); private State currentState = SUSPENDED_NODE_ADMIN; @@ -50,6 +51,7 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { private boolean workToDoNow = true; private final Object monitor = new Object(); + private RuntimeException lastConvergenceException; private final Logger log = Logger.getLogger(NodeAdminStateUpdater.class.getName()); private final ScheduledExecutorService specVerifierScheduler = @@ -143,15 +145,19 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { } @Override - public boolean setResumeStateAndCheckIfResumed(State wantedState) { + public void setResumeStateAndCheckIfResumed(State wantedState) { synchronized (monitor) { if (this.wantedState != wantedState) { log.info("Wanted state change: " + this.wantedState + " -> " + wantedState); this.wantedState = wantedState; + setLastConvergenceException(null); signalWorkToBeDone(); } - return currentState == wantedState; + if (currentState != wantedState) { + throw Optional.ofNullable(lastConvergenceException) + .orElseGet(() -> new RuntimeException(TRANSITION_EXCEPTION_MESSAGE)); + } } } @@ -187,9 +193,12 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { try { convergeState(wantedStateCopy); + setLastConvergenceException(null); } catch (OrchestratorException | ConvergenceException | HttpException e) { + setLastConvergenceException(e); log.info("Unable to converge to " + wantedStateCopy + ": " + e.getMessage()); - } catch (Exception e) { + } catch (RuntimeException e) { + setLastConvergenceException(e); log.log(LogLevel.ERROR, "Error while trying to converge to " + wantedStateCopy, e); } @@ -206,6 +215,12 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { fetchContainersToRunFromNodeRepository(); } + private void setLastConvergenceException(RuntimeException exception) { + synchronized (monitor) { + lastConvergenceException = exception; + } + } + /** * This method attempts to converge node-admin w/agents to a {@link State} * with respect to: freeze, Orchestrator, and services running. diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminStateUpdater.java index 841f464e014..8a926b511e6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminStateUpdater.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminStateUpdater.java @@ -8,9 +8,11 @@ public interface NodeAdminStateUpdater extends NodeAdminDebugHandler { enum State { TRANSITIONING, RESUMED, SUSPENDED_NODE_ADMIN, SUSPENDED} /** - * Set the wanted state, and return whether the current state equals it. + * Set the wanted state, and assert whether the current state equals it. * Typically, this method should be called repeatedly until current state * has converged. + * + * @throws RuntimeException (or a subclass) if the state has not converged yet. */ - boolean setResumeStateAndCheckIfResumed(State wantedState); + void setResumeStateAndCheckIfResumed(State wantedState); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java index a8dcde02ca6..8411df09c70 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java @@ -9,6 +9,7 @@ import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.vespa.hosted.dockerapi.metrics.DimensionMetrics; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.provider.NodeAdminStateUpdater; +import com.yahoo.yolean.Exceptions; import javax.inject.Inject; import javax.ws.rs.core.MediaType; @@ -94,9 +95,13 @@ public class RestApiHandler extends LoggingRequestHandler{ } if (wantedState != null) { - return refresher.setResumeStateAndCheckIfResumed(wantedState) ? - new SimpleResponse(200, "ok") : - new SimpleResponse(409, "fail"); + try { + refresher.setResumeStateAndCheckIfResumed(wantedState); + return new SimpleResponse(200, "ok"); + } catch (RuntimeException e) { + return new SimpleResponse(409, "Failed to converge to " + wantedState + ": " + + Exceptions.toMessageString(e)); + } } return new SimpleResponse(400, "unknown path " + path); } @@ -107,7 +112,7 @@ public class RestApiHandler extends LoggingRequestHandler{ SimpleResponse(int code, String message) { super(code); ObjectNode objectNode = objectMapper.createObjectNode(); - objectNode.put("jsonMessage", message); + objectNode.put("message", message); this.jsonMessage = objectNode.toString(); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java index a83efc03fbe..db14efdd5d2 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java @@ -42,8 +42,8 @@ public class RebootTest { "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=null, dockerImage=dockerImage, vespaVersion='null'}"); NodeAdminStateUpdaterImpl updater = dockerTester.nodeAdminStateUpdater; - assertThat(updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED), - is(Optional.of("Not all node agents are frozen."))); +// assertThat(updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED), +// is(Optional.of("Not all node agents are frozen."))); updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java index 607dc080a90..02baf5959c9 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java @@ -19,8 +19,9 @@ import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.IntStream; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdaterImpl.TRANSITION_EXCEPTION_MESSAGE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; @@ -62,7 +63,7 @@ public class NodeAdminStateUpdaterImplTest { suspendHostnames.add(parentHostname); // Initially everything is frozen to force convergence - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED)); + assertResumeStateError(NodeAdminStateUpdater.State.RESUMED, TRANSITION_EXCEPTION_MESSAGE); when(nodeAdmin.setFrozen(eq(false))).thenReturn(true); doNothing().when(orchestrator).resume(parentHostname); tickAfter(0); // The first tick should unfreeze @@ -70,35 +71,36 @@ public class NodeAdminStateUpdaterImplTest { verify(orchestrator, times(1)).resume(parentHostname); // Everything is running and we want to continue running, therefore we have converged - assertTrue(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED)); + refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED); tickAfter(35); tickAfter(35); - assertTrue(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED)); + refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED); verify(refresher, never()).signalWorkToBeDone(); // No attempt in changing state verify(orchestrator, times(1)).resume(parentHostname); // Already resumed // Lets try to suspend node admin only, immediately we get false back, and need to wait until next // tick before any change can happen - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN)); + assertResumeStateError(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN, TRANSITION_EXCEPTION_MESSAGE); verify(refresher, times(1)).signalWorkToBeDone(); - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN)); // Still no change + assertResumeStateError(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN, TRANSITION_EXCEPTION_MESSAGE); // Still no change verify(refresher, times(1)).signalWorkToBeDone(); // We already notified of work, dont need to do it again when(nodeAdmin.setFrozen(eq(true))).thenReturn(false); when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofSeconds(1)); tickAfter(0); - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN)); + assertResumeStateError(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN, "NodeAdmin is not yet frozen"); verify(refresher, times(1)).signalWorkToBeDone(); // No change in desired state // First orchestration failure happens within the freeze convergence timeout, // and so should not call setFrozen(false) + final String exceptionMessage = "Cannot allow to suspend because some reason"; verify(nodeAdmin, times(1)).setFrozen(eq(false)); when(nodeAdmin.setFrozen(eq(true))).thenReturn(true); when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofSeconds(1)); - doThrow(new RuntimeException("Cannot allow to suspend because some reason")) + doThrow(new RuntimeException(exceptionMessage)) .when(orchestrator).suspend(eq(parentHostname)); tickAfter(35); - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN)); + assertResumeStateError(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN, exceptionMessage); verify(refresher, times(1)).signalWorkToBeDone(); verify(nodeAdmin, times(1)).setFrozen(eq(false)); @@ -106,22 +108,22 @@ public class NodeAdminStateUpdaterImplTest { // and so SHOULD call setFrozen(false) when(nodeAdmin.setFrozen(eq(true))).thenReturn(true); when(nodeAdmin.subsystemFreezeDuration()).thenReturn(NodeAdminStateUpdaterImpl.FREEZE_CONVERGENCE_TIMEOUT.plusMinutes(1)); - doThrow(new RuntimeException("Cannot allow to suspend because some reason")).doNothing() + doThrow(new RuntimeException(exceptionMessage)).doNothing() .when(orchestrator).suspend(eq(parentHostname)); tickAfter(35); - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN)); + assertResumeStateError(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN, exceptionMessage); verify(refresher, times(1)).signalWorkToBeDone(); verify(nodeAdmin, times(2)).setFrozen(eq(false)); // +1, since freeze convergence have timed out tickAfter(35); - assertTrue(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN)); + refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN); verify(nodeAdmin, times(2)).setFrozen(eq(false)); // At this point orchestrator will say its OK to suspend, but something goes wrong when we try to stop services verify(orchestrator, times(0)).suspend(eq(parentHostname), eq(suspendHostnames)); doThrow(new RuntimeException("Failed to stop services")).doNothing().when(nodeAdmin).stopNodeAgentServices(eq(activeHostnames)); when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofSeconds(1)); - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED)); + assertResumeStateError(NodeAdminStateUpdater.State.SUSPENDED, TRANSITION_EXCEPTION_MESSAGE); tickAfter(0); // Change in wanted state, no need to wait verify(orchestrator, times(1)).suspend(eq(parentHostname), eq(suspendHostnames)); verify(refresher, times(2)).signalWorkToBeDone(); // No change in desired state @@ -130,58 +132,69 @@ public class NodeAdminStateUpdaterImplTest { // Finally we are successful in transitioning to frozen tickAfter(35); - assertTrue(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED)); + refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED); // We are in desired state, no changes will happen reset(nodeAdmin); tickAfter(35); tickAfter(35); - assertTrue(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED)); + refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED); verify(refresher, times(2)).signalWorkToBeDone(); // No change in desired state verifyNoMoreInteractions(nodeAdmin); // Lets try going back to resumed when(nodeAdmin.setFrozen(eq(false))).thenReturn(false).thenReturn(true); // NodeAgents not converged to yet - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED)); + assertResumeStateError(NodeAdminStateUpdater.State.RESUMED, TRANSITION_EXCEPTION_MESSAGE); tickAfter(35); - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED)); + assertResumeStateError(NodeAdminStateUpdater.State.RESUMED, "NodeAdmin is not yet unfrozen"); doThrow(new OrchestratorException("Cannot allow to suspend " + parentHostname)).doNothing() .when(orchestrator).resume(parentHostname); tickAfter(35); - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED)); + assertResumeStateError(NodeAdminStateUpdater.State.RESUMED, "Cannot allow to suspend basehost1.test.yahoo.com"); tickAfter(35); - assertTrue(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED)); + refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED); } @Test public void half_transition_revert() { + final String exceptionMsg = "Cannot allow to suspend because some reason"; mockNodeRepo(3); // Initially everything is frozen to force convergence when(nodeAdmin.setFrozen(eq(false))).thenReturn(true); doNothing().when(orchestrator).resume(parentHostname); tickAfter(0); // The first tick should unfreeze - assertTrue(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED)); + refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED); verify(nodeAdmin, times(1)).setFrozen(eq(false)); // Let's start suspending, we are able to freeze the nodes, but orchestrator denies suspension when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofSeconds(1)); when(nodeAdmin.setFrozen(eq(true))).thenReturn(true); - doThrow(new RuntimeException("Cannot allow to suspend because some reason")) + doThrow(new RuntimeException(exceptionMsg)) .when(orchestrator).suspend(eq(parentHostname)); - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN)); + assertResumeStateError(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN, TRANSITION_EXCEPTION_MESSAGE); tickAfter(0); verify(nodeAdmin, times(1)).setFrozen(eq(true)); + assertResumeStateError(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN, exceptionMsg); // We change our mind, want to remain resumed - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED)); + assertResumeStateError(NodeAdminStateUpdater.State.RESUMED, TRANSITION_EXCEPTION_MESSAGE); tickAfter(0); - assertTrue(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED)); + refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED); verify(nodeAdmin, times(2)).setFrozen(eq(false)); // Make sure that we unfreeze! } + private void assertResumeStateError(NodeAdminStateUpdater.State targetState, String reason) { + try { + refresher.setResumeStateAndCheckIfResumed(targetState); + fail("Expected set resume state to fail with \"" + reason + "\", but it succeeded without error"); + } catch (RuntimeException e) { + assertEquals(reason, e.getMessage()); + } + } + private void mockNodeRepo(int numberOfNodes) { List<NodeSpec> containersToRun = IntStream.range(0, numberOfNodes) .mapToObj(i -> new NodeSpec.Builder() diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 4497f55cb85..62e954afba3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostLivenessTracker; import com.yahoo.config.provision.Provisioner; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -65,7 +66,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor, Zone zone, Clock clock, Orchestrator orchestrator, Metric metric, ConfigserverConfig configserverConfig) { - DefaultTimes defaults = new DefaultTimes(zone.environment()); + DefaultTimes defaults = new DefaultTimes(zone); jobControl = new JobControl(nodeRepository.database()); infrastructureVersions = new InfrastructureVersions(nodeRepository.database()); @@ -151,7 +152,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final NodeFailer.ThrottlePolicy throttlePolicy; - DefaultTimes(Environment environment) { + DefaultTimes(Zone zone) { failGrace = Duration.ofMinutes(60); periodicRedeployInterval = Duration.ofMinutes(30); operatorChangeRedeployInterval = Duration.ofMinutes(1); @@ -164,13 +165,14 @@ public class NodeRepositoryMaintenance extends AbstractComponent { infrastructureProvisionInterval = Duration.ofMinutes(3); throttlePolicy = NodeFailer.ThrottlePolicy.hosted; + Environment environment = zone.environment(); if (environment.isTest()) retiredExpiry = Duration.ofMinutes(1); // fast turnaround as test envs don't have persistent data else retiredExpiry = Duration.ofDays(4); // give up migrating data after 4 days - if (environment.equals(Environment.prod)) { + if (environment.equals(Environment.prod) && zone.system() == SystemName.main) { inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy retiredInterval = Duration.ofMinutes(29); dirtyExpiry = Duration.ofHours(2); // enough time to clean the node @@ -72,6 +72,7 @@ <module>documentapi</module> <module>document</module> <module>documentgen-test</module> + <module>fat-model-dependencies</module> <module>fileacquirer</module> <module>filedistribution</module> <module>fsa</module> diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index 5c0edce0b94..153b1ae2867 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -2,6 +2,7 @@ #include <vespa/persistence/spi/result.h> #include <vespa/document/update/assignvalueupdate.h> +#include <vespa/document/repo/documenttyperepo.h> #include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h> #include <vespa/searchcore/proton/test/bucketfactory.h> #include <vespa/searchcore/proton/common/feedtoken.h> @@ -35,6 +36,7 @@ LOG_SETUP("feedhandler_test"); using document::BucketId; using document::Document; using document::DocumentId; +using document::DocumentType; using document::DocumentTypeRepo; using document::DocumentUpdate; using document::GlobalId; @@ -181,7 +183,9 @@ struct MyFeedView : public test::DummyFeedView { int prune_removed_count; int update_count; SerialNum update_serial; - MyFeedView(const std::shared_ptr<const DocumentTypeRepo> &dtr); + const DocumentType *documentType; + MyFeedView(const std::shared_ptr<const DocumentTypeRepo> &dtr, + const DocTypeName &docTypeName); ~MyFeedView() override; void resetPutLatch(uint32_t count) { putLatch.reset(new vespalib::CountDownLatch(count)); } void preparePut(PutOperation &op) override { @@ -203,6 +207,8 @@ struct MyFeedView : public test::DummyFeedView { if (usePutRdz) { putRdz.run(); } + EXPECT_EQUAL(_docTypeRepo.get(), putOp.getDocument()->getRepo()); + EXPECT_EQUAL(documentType, &putOp.getDocument()->getType()); ++put_count; put_serial = putOp.getSerialNum(); metaStore.allocate(putOp.getDocument()->getId().getGlobalId()); @@ -216,6 +222,7 @@ struct MyFeedView : public test::DummyFeedView { void handleUpdate(FeedToken token, const UpdateOperation &op) override { (void) token; + EXPECT_EQUAL(documentType, &op.getUpdate()->getType()); ++update_count; update_serial = op.getSerialNum(); } @@ -237,7 +244,7 @@ struct MyFeedView : public test::DummyFeedView { } }; -MyFeedView::MyFeedView(const std::shared_ptr<const DocumentTypeRepo> &dtr) +MyFeedView::MyFeedView(const std::shared_ptr<const DocumentTypeRepo> &dtr, const DocTypeName &docTypeName) : test::DummyFeedView(dtr), putRdz(), usePutRdz(false), @@ -250,7 +257,8 @@ MyFeedView::MyFeedView(const std::shared_ptr<const DocumentTypeRepo> &dtr) move_count(0), prune_removed_count(0), update_count(0), - update_serial(0) + update_serial(0), + documentType(dtr->getDocumentType(docTypeName.getName())) {} MyFeedView::~MyFeedView() {} @@ -294,6 +302,13 @@ struct DocumentContext { } }; +struct TwoFieldsSchemaContext : public SchemaContext { + TwoFieldsSchemaContext() + : SchemaContext() + { + addField("i2"); + } +}; struct UpdateContext { DocumentUpdate::SP update; @@ -433,7 +448,7 @@ struct FeedHandlerFixture owner(), _state(), replayConfig(), - feedView(schema.getRepo()), + feedView(schema.getRepo(), schema.getDocType()), _bucketDB(), _bucketDBHandler(_bucketDB), handler(writeService, tlsSpec, schema.getDocType(), _state, owner, @@ -714,15 +729,13 @@ TEST_F("require that update with same document type repo is ok", FeedHandlerFixt TEST_F("require that update with different document type repo can be ok", FeedHandlerFixture) { - SchemaContext schema; - schema.addField("i2"); + TwoFieldsSchemaContext schema; checkUpdate(f, schema, "i1", false, true); } TEST_F("require that update with different document type repo can be rejected", FeedHandlerFixture) { - SchemaContext schema; - schema.addField("i2"); + TwoFieldsSchemaContext schema; checkUpdate(f, schema, "i2", true, true); } @@ -733,18 +746,31 @@ TEST_F("require that update with same document type repo is ok, fallback to crea TEST_F("require that update with different document type repo can be ok, fallback to create document", FeedHandlerFixture) { - SchemaContext schema; - schema.addField("i2"); + TwoFieldsSchemaContext schema; checkUpdate(f, schema, "i1", false, false); } TEST_F("require that update with different document type repo can be rejected, preventing fallback to create document", FeedHandlerFixture) { - SchemaContext schema; - schema.addField("i2"); + TwoFieldsSchemaContext schema; checkUpdate(f, schema, "i2", true, false); } +TEST_F("require that put with different document type repo is ok", FeedHandlerFixture) +{ + TwoFieldsSchemaContext schema; + DocumentContext doc_context("doc:test:foo", *schema.builder); + auto op = std::make_unique<PutOperation>(doc_context.bucketId, + Timestamp(10), doc_context.doc); + FeedTokenContext token_context; + EXPECT_EQUAL(schema.getRepo().get(), op->getDocument()->getRepo()); + EXPECT_NOT_EQUAL(f.schema.getRepo().get(), op->getDocument()->getRepo()); + EXPECT_NOT_EQUAL(f.feedView.documentType, &op->getDocument()->getType()); + f.handler.performOperation(std::move(token_context.token), std::move(op)); + EXPECT_EQUAL(1, f.feedView.put_count); + EXPECT_EQUAL(1, f.tls_writer.store_count); +} + } // namespace TEST_MAIN() diff --git a/searchcore/src/tests/proton/matching/query_test.cpp b/searchcore/src/tests/proton/matching/query_test.cpp index eb49603f71d..7875e7ec4aa 100644 --- a/searchcore/src/tests/proton/matching/query_test.cpp +++ b/searchcore/src/tests/proton/matching/query_test.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for query. -#include <vespa/document/datatype/positiondatatype.h> #include <vespa/searchcore/proton/matching/fakesearchcontext.h> #include <vespa/searchcore/proton/matching/matchdatareservevisitor.h> #include <vespa/searchcore/proton/matching/blueprintbuilder.h> @@ -26,8 +25,9 @@ #include <vespa/searchlib/queryeval/simpleresult.h> #include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/searchlib/queryeval/termasstring.h> +#include <vespa/document/datatype/positiondatatype.h> + #include <vespa/vespalib/testkit/testapp.h> -#include <vector> using document::PositionDataType; using search::fef::FieldInfo; @@ -62,8 +62,7 @@ using std::vector; namespace fef_test = search::fef::test; using CollectionType = FieldInfo::CollectionType; -namespace proton { -namespace matching { +namespace proton::matching { namespace { class Test : public vespalib::TestApp { @@ -175,6 +174,12 @@ Node::UP buildQueryTree(const ViewResolver &resolver, query_builder.addPhrase(2, field, 7, Weight(0)); query_builder.addStringTerm(phrase_term, field, 8, Weight(0)); query_builder.addStringTerm(phrase_term, field, 9, Weight(0)); +#if 0 + //Todo add testing when SameElement blueprints are ready + query_builder.addSameElement(2, field); + query_builder.addStringTerm(string_term, field, 10, Weight(0)); + query_builder.addStringTerm(prefix_term, field, 11, Weight(0)); +#endif Node::UP node = query_builder.build(); ResolveViewVisitor visitor(resolver, idxEnv); @@ -222,19 +227,19 @@ public: EXPECT_EQUAL((double)estimatedHitCount / doc_count, n.field(0).getDocFreq()); } - virtual void visit(ProtonNumberTerm &n) override { checkNode(n, 1, false); } - virtual void visit(ProtonLocationTerm &n) override { checkNode(n, 0, true); } - virtual void visit(ProtonPrefixTerm &n) override { checkNode(n, 1, false); } - virtual void visit(ProtonRangeTerm &n) override { checkNode(n, 2, false); } - virtual void visit(ProtonStringTerm &n) override { checkNode(n, 2, false); } - virtual void visit(ProtonSubstringTerm &n) override { checkNode(n, 0, true); } - virtual void visit(ProtonSuffixTerm &n) override { checkNode(n, 2, false); } - virtual void visit(ProtonPhrase &n) override { checkNode(n, 0, true); } - virtual void visit(ProtonWeightedSetTerm &) override {} - virtual void visit(ProtonDotProduct &) override {} - virtual void visit(ProtonWandTerm &) override {} - virtual void visit(ProtonPredicateQuery &) override {} - virtual void visit(ProtonRegExpTerm &) override {} + void visit(ProtonNumberTerm &n) override { checkNode(n, 1, false); } + void visit(ProtonLocationTerm &n) override { checkNode(n, 0, true); } + void visit(ProtonPrefixTerm &n) override { checkNode(n, 1, false); } + void visit(ProtonRangeTerm &n) override { checkNode(n, 2, false); } + void visit(ProtonStringTerm &n) override { checkNode(n, 2, false); } + void visit(ProtonSubstringTerm &n) override { checkNode(n, 0, true); } + void visit(ProtonSuffixTerm &n) override { checkNode(n, 2, false); } + void visit(ProtonPhrase &n) override { checkNode(n, 0, true); } + void visit(ProtonWeightedSetTerm &) override {} + void visit(ProtonDotProduct &) override {} + void visit(ProtonWandTerm &) override {} + void visit(ProtonPredicateQuery &) override {} + void visit(ProtonRegExpTerm &) override {} }; void Test::requireThatTermsAreLookedUp() { @@ -354,12 +359,12 @@ class SetUpTermDataTestCheckerVisitor int Main() { return 0; } public: - virtual void visit(ProtonNumberTerm &) override {} - virtual void visit(ProtonLocationTerm &) override {} - virtual void visit(ProtonPrefixTerm &) override {} - virtual void visit(ProtonRangeTerm &) override {} + void visit(ProtonNumberTerm &) override {} + void visit(ProtonLocationTerm &) override {} + void visit(ProtonPrefixTerm &) override {} + void visit(ProtonRangeTerm &) override {} - virtual void visit(ProtonStringTerm &n) override { + void visit(ProtonStringTerm &n) override { const ITermData &term_data = n; EXPECT_EQUAL(string_weight.percent(), term_data.getWeight().percent()); @@ -375,17 +380,17 @@ public: } } - virtual void visit(ProtonSubstringTerm &) override {} - virtual void visit(ProtonSuffixTerm &) override {} - virtual void visit(ProtonPhrase &n) override { + void visit(ProtonSubstringTerm &) override {} + void visit(ProtonSuffixTerm &) override {} + void visit(ProtonPhrase &n) override { const ITermData &term_data = n; EXPECT_EQUAL(2u, term_data.getPhraseLength()); } - virtual void visit(ProtonWeightedSetTerm &) override {} - virtual void visit(ProtonDotProduct &) override {} - virtual void visit(ProtonWandTerm &) override {} - virtual void visit(ProtonPredicateQuery &) override {} - virtual void visit(ProtonRegExpTerm &) override {} + void visit(ProtonWeightedSetTerm &) override {} + void visit(ProtonDotProduct &) override {} + void visit(ProtonWandTerm &) override {} + void visit(ProtonPredicateQuery &) override {} + void visit(ProtonRegExpTerm &) override {} }; void Test::requireThatTermDataIsFilledIn() { @@ -855,7 +860,7 @@ Test::requireThatWhiteListBlueprintCanBeUsed() EXPECT_EQUAL(exp, act); } -Test::~Test() {} +Test::~Test() = default; int Test::Main() @@ -877,6 +882,7 @@ Test::Main() TEST_CALL(requireThatNearIteratorsCanBeBuilt); TEST_CALL(requireThatONearIteratorsCanBeBuilt); TEST_CALL(requireThatPhraseIteratorsCanBeBuilt); + //TODO Add SameElement testing TEST_CALL(requireThatUnknownFieldActsEmpty); TEST_CALL(requireThatIllegalFieldsAreIgnored); TEST_CALL(requireThatQueryGluesEverythingTogether); @@ -893,7 +899,6 @@ Test::Main() } // namespace -} // namespace matching -} // namespace proton +} // namespace proton::matching TEST_APPHOOK(proton::matching::Test); diff --git a/searchcore/src/tests/proton/matching/querynodes_test.cpp b/searchcore/src/tests/proton/matching/querynodes_test.cpp index f8a419ba15b..7b6fdd1ae88 100644 --- a/searchcore/src/tests/proton/matching/querynodes_test.cpp +++ b/searchcore/src/tests/proton/matching/querynodes_test.cpp @@ -1,11 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for querynodes. -#include <vespa/log/log.h> -LOG_SETUP("querynodes_test"); - #include <vespa/searchcore/proton/matching/querynodes.h> - #include <vespa/searchcore/proton/matching/fakesearchcontext.h> #include <vespa/searchcore/proton/matching/blueprintbuilder.h> #include <vespa/searchcore/proton/matching/matchdatareservevisitor.h> @@ -33,11 +29,12 @@ LOG_SETUP("querynodes_test"); #include <vespa/searchlib/queryeval/fake_search.h> #include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/vespalib/testkit/testapp.h> -#include <cstdarg> -#include <string> -#include <vector> + #include <vespa/searchlib/attribute/singlenumericattribute.hpp> +#include <vespa/log/log.h> +LOG_SETUP("querynodes_test"); + using search::fef::FieldInfo; using search::fef::FieldType; using search::fef::MatchData; @@ -210,9 +207,8 @@ public: }; typedef QueryBuilder<ProtonNodeTypes> QB; -struct Phrase { - void addToBuilder(QB& b) { b.addPhrase(2, view, id, weight); } -}; +struct Phrase { void addToBuilder(QB& b) { b.addPhrase(2, view, id, weight); }}; +struct SameElement { void addToBuilder(QB& b) { b.addSameElement(2, view); }}; struct Near { void addToBuilder(QB& b) { b.addNear(2, distance); } }; struct ONear { void addToBuilder(QB& b) { b.addONear(2, distance); } }; struct Or { void addToBuilder(QB& b) { b.addOr(2); } }; @@ -466,6 +462,11 @@ TEST("requireThatPhrasesGetProperBlending") { TEST_DO(checkProperBlending<Phrase>()); } +TEST("requireThatSameElementGetProperBlending") { + //TODO SameEelement needs proper testing/implementation + //TEST_DO(checkProperBlending<SameElement>()); +} + TEST("requireThatNearGetProperBlending") { TEST_DO(checkProperBlendingWithParent<Near>()); } diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.cpp index a6d34b696ca..efac7297c67 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.cpp @@ -49,6 +49,15 @@ PutOperation::deserialize(vespalib::nbostream &is, _serializedDocSize = oldSize - is.size(); } +void +PutOperation::deserializeDocument(const DocumentTypeRepo &repo) +{ + vespalib::nbostream stream; + _doc->serialize(stream); + auto fixedDoc = std::make_shared<Document>(repo, stream); + _doc = std::move(fixedDoc); +} + vespalib::string PutOperation::toString() const { diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.h index 7e9dee2cbc0..33330692fab 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.h @@ -21,6 +21,7 @@ public: virtual void serialize(vespalib::nbostream &os) const override; virtual void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) override; + void deserializeDocument(const document::DocumentTypeRepo &repo); virtual vespalib::string toString() const override; }; diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp index fa5bbee8e6d..beaf719dc5c 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp @@ -8,6 +8,7 @@ LOG_SETUP(".proton.feedoperation.updateoperation"); using document::BucketId; +using document::DocumentType; using document::DocumentTypeRepo; using document::DocumentUpdate; using storage::spi::Timestamp; @@ -46,12 +47,9 @@ UpdateOperation::UpdateOperation(const BucketId &bucketId, { } - void -UpdateOperation::serialize(vespalib::nbostream &os) const +UpdateOperation::serializeUpdate(vespalib::nbostream &os) const { - assertValidBucketId(_upd->getId()); - DocumentOperation::serialize(os); if (getType() == FeedOperation::UPDATE_42) { _upd->serialize42(os); } else { @@ -59,19 +57,33 @@ UpdateOperation::serialize(vespalib::nbostream &os) const } } +void +UpdateOperation::deserializeUpdate(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) +{ + document::ByteBuffer buf(is.peek(), is.size()); + using Version = DocumentUpdate::SerializeVersion; + Version version = ((getType() == FeedOperation::UPDATE_42) ? Version::SERIALIZE_42 : Version::SERIALIZE_HEAD); + DocumentUpdate::SP update(std::make_shared<DocumentUpdate>(repo, buf, version)); + is.adjustReadPos(buf.getPos()); + _upd = std::move(update); +} + +void +UpdateOperation::serialize(vespalib::nbostream &os) const +{ + assertValidBucketId(_upd->getId()); + DocumentOperation::serialize(os); + serializeUpdate(os); +} + void UpdateOperation::deserialize(vespalib::nbostream &is, const DocumentTypeRepo &repo) { DocumentOperation::deserialize(is, repo); - document::ByteBuffer buf(is.peek(), is.size()); - using Version = DocumentUpdate::SerializeVersion; - Version version = ((getType() == FeedOperation::UPDATE_42) ? Version::SERIALIZE_42 : Version::SERIALIZE_HEAD); try { - DocumentUpdate::SP update(std::make_shared<DocumentUpdate>(repo, buf, version)); - is.adjustReadPos(buf.getPos()); - _upd = update; + deserializeUpdate(is, repo); } catch (document::DocumentTypeNotFoundException &e) { LOG(warning, "Failed deserialize update operation using unknown document type '%s'", e.getDocumentTypeName().c_str()); @@ -80,6 +92,14 @@ UpdateOperation::deserialize(vespalib::nbostream &is, } } +void +UpdateOperation::deserializeUpdate(const DocumentTypeRepo &repo) +{ + vespalib::nbostream stream; + serializeUpdate(stream); + deserializeUpdate(stream, repo); +} + vespalib::string UpdateOperation::toString() const { return make_string("%s(%s, %s)", ((getType() == FeedOperation::UPDATE_42) ? "Update42" : "Update"), diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h index 6e061f79f30..7886231af82 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h @@ -3,7 +3,10 @@ #include "documentoperation.h" -namespace document { class DocumentUpdate; } +namespace document { +class DocumentTypeRepo; +class DocumentUpdate; +} namespace proton { @@ -16,6 +19,8 @@ private: const document::BucketId &bucketId, const storage::spi::Timestamp ×tamp, const DocumentUpdateSP &upd); + void serializeUpdate(vespalib::nbostream &os) const; + void deserializeUpdate(vespalib::nbostream &is, const document::DocumentTypeRepo &repo); public: UpdateOperation(); UpdateOperation(Type type); @@ -26,6 +31,7 @@ public: const DocumentUpdateSP &getUpdate() const { return _upd; } void serialize(vespalib::nbostream &os) const override; void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) override; + void deserializeUpdate(const document::DocumentTypeRepo &repo); virtual vespalib::string toString() const override; static UpdateOperation makeOldUpdate(const document::BucketId &bucketId, const storage::spi::Timestamp ×tamp, diff --git a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp index 97fc1905f50..165fc67179a 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp @@ -10,8 +10,7 @@ using namespace search::queryeval; -namespace proton { -namespace matching { +namespace proton::matching { namespace { @@ -124,29 +123,31 @@ private: } protected: - virtual void visit(ProtonAnd &n) override { buildIntermediate(new AndBlueprint(), n); } - virtual void visit(ProtonAndNot &n) override { buildIntermediate(new AndNotBlueprint(), n); } - virtual void visit(ProtonOr &n) override { buildIntermediate(new OrBlueprint(), n); } - virtual void visit(ProtonWeakAnd &n) override { buildWeakAnd(n); } - virtual void visit(ProtonEquiv &n) override { buildEquiv(n); } - virtual void visit(ProtonRank &n) override { buildIntermediate(new RankBlueprint(), n); } - virtual void visit(ProtonNear &n) override { buildIntermediate(new NearBlueprint(n.getDistance()), n); } - virtual void visit(ProtonONear &n) override { buildIntermediate(new ONearBlueprint(n.getDistance()), n); } - - virtual void visit(ProtonWeightedSetTerm &n) override { buildTerm(n); } - virtual void visit(ProtonDotProduct &n) override { buildTerm(n); } - virtual void visit(ProtonWandTerm &n) override { buildTerm(n); } - - virtual void visit(ProtonPhrase &n) override { buildTerm(n); } - virtual void visit(ProtonNumberTerm &n) override { buildTerm(n); } - virtual void visit(ProtonLocationTerm &n) override { buildTerm(n); } - virtual void visit(ProtonPrefixTerm &n) override { buildTerm(n); } - virtual void visit(ProtonRangeTerm &n) override { buildTerm(n); } - virtual void visit(ProtonStringTerm &n) override { buildTerm(n); } - virtual void visit(ProtonSubstringTerm &n) override { buildTerm(n); } - virtual void visit(ProtonSuffixTerm &n) override { buildTerm(n); } - virtual void visit(ProtonPredicateQuery &n) override { buildTerm(n); } - virtual void visit(ProtonRegExpTerm &n) override { buildTerm(n); } + void visit(ProtonAnd &n) override { buildIntermediate(new AndBlueprint(), n); } + void visit(ProtonAndNot &n) override { buildIntermediate(new AndNotBlueprint(), n); } + void visit(ProtonOr &n) override { buildIntermediate(new OrBlueprint(), n); } + void visit(ProtonWeakAnd &n) override { buildWeakAnd(n); } + void visit(ProtonEquiv &n) override { buildEquiv(n); } + void visit(ProtonRank &n) override { buildIntermediate(new RankBlueprint(), n); } + void visit(ProtonNear &n) override { buildIntermediate(new NearBlueprint(n.getDistance()), n); } + void visit(ProtonONear &n) override { buildIntermediate(new ONearBlueprint(n.getDistance()), n); } + void visit(ProtonSameElement &n) override { buildIntermediate(nullptr /*new SameElementBlueprint())*/, n); } + + + void visit(ProtonWeightedSetTerm &n) override { buildTerm(n); } + void visit(ProtonDotProduct &n) override { buildTerm(n); } + void visit(ProtonWandTerm &n) override { buildTerm(n); } + + void visit(ProtonPhrase &n) override { buildTerm(n); } + void visit(ProtonNumberTerm &n) override { buildTerm(n); } + void visit(ProtonLocationTerm &n) override { buildTerm(n); } + void visit(ProtonPrefixTerm &n) override { buildTerm(n); } + void visit(ProtonRangeTerm &n) override { buildTerm(n); } + void visit(ProtonStringTerm &n) override { buildTerm(n); } + void visit(ProtonSubstringTerm &n) override { buildTerm(n); } + void visit(ProtonSuffixTerm &n) override { buildTerm(n); } + void visit(ProtonPredicateQuery &n) override { buildTerm(n); } + void visit(ProtonRegExpTerm &n) override { buildTerm(n); } public: BlueprintBuilderVisitor(const IRequestContext & requestContext, ISearchContext &context) : @@ -174,5 +175,4 @@ BlueprintBuilder::build(const IRequestContext & requestContext, return result; } -} // namespace matching -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp index c1c4387cf41..1efb74b96ba 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp @@ -141,7 +141,7 @@ MatchThread::try_share(DocidRange &docid_range, uint32_t next_docid) { template <typename Strategy, bool do_rank, bool do_limit, bool do_share_work> uint32_t -MatchThread::inner_match_loop(Context &context, MatchTools &tools, DocidRange docid_range) +MatchThread::inner_match_loop(Context &context, MatchTools &tools, DocidRange &docid_range) { SearchIterator *search = &tools.search(); search->initRange(docid_range.begin, docid_range.end); @@ -175,12 +175,14 @@ MatchThread::match_loop(MatchTools &tools, HitCollector &hits) uint32_t docsCovered = 0; Context context(matchParams.rankDropLimit, tools, hits, num_threads); for (DocidRange docid_range = scheduler.first_range(thread_id); - !docid_range.empty() && ! softDoomed; + !docid_range.empty(); docid_range = scheduler.next_range(thread_id)) { - uint32_t lastCovered = inner_match_loop<Strategy, do_rank, do_limit, do_share_work>(context, tools, docid_range); - softDoomed = (lastCovered < docid_range.end); - docsCovered += std::min(lastCovered, docid_range.end) - docid_range.begin; + if (!softDoomed) { + uint32_t lastCovered = inner_match_loop<Strategy, do_rank, do_limit, do_share_work>(context, tools, docid_range); + softDoomed = (lastCovered < docid_range.end); + docsCovered += std::min(lastCovered, docid_range.end) - docid_range.begin; + } } uint32_t matches = context.matches; if (do_limit && context.isBelowLimit()) { diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h index b7ecf149001..5c78ee59b1d 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h @@ -75,7 +75,7 @@ private: bool try_share(DocidRange &docid_range, uint32_t next_docid) __attribute__((noinline)); template <typename Strategy, bool do_rank, bool do_limit, bool do_share_work> - uint32_t inner_match_loop(Context &context, MatchTools &tools, DocidRange docid_range) __attribute__((noinline)); + uint32_t inner_match_loop(Context &context, MatchTools &tools, DocidRange &docid_range) __attribute__((noinline)); template <typename Strategy, bool do_rank, bool do_limit, bool do_share_work> void match_loop(MatchTools &tools, HitCollector &hits) __attribute__((noinline)); diff --git a/searchcore/src/vespa/searchcore/proton/matching/matchdatareservevisitor.h b/searchcore/src/vespa/searchcore/proton/matching/matchdatareservevisitor.h index ebe4e9b2ad7..a20f648ca1c 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matchdatareservevisitor.h +++ b/searchcore/src/vespa/searchcore/proton/matching/matchdatareservevisitor.h @@ -22,13 +22,14 @@ public: template <class TermNode> void visitTerm(TermNode &n) { n.allocateTerms(_mdl); } - virtual void visit(ProtonNodeTypes::Equiv &n) override { + void visit(ProtonNodeTypes::Equiv &n) override { MatchDataReserveVisitor subAllocator(n.children_mdl); for (size_t i = 0; i < n.getChildren().size(); ++i) { n.getChildren()[i]->accept(subAllocator); } n.allocateTerms(_mdl); } + void visit(ProtonNodeTypes::SameElement &) override { } MatchDataReserveVisitor(search::fef::MatchDataLayout &mdl) : _mdl(mdl) {} }; diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.cpp b/searchcore/src/vespa/searchcore/proton/matching/query.cpp index 9181205bb19..8896e4d35c8 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/query.cpp @@ -78,8 +78,8 @@ void AddLocationNode(const string &location_str, Node::UP &query_tree, Location } } // namespace -Query::Query() {} -Query::~Query() {} +Query::Query() = default; +Query::~Query() = default; bool Query::buildTree(const vespalib::stringref &stack, const string &location, diff --git a/searchcore/src/vespa/searchcore/proton/matching/querynodes.cpp b/searchcore/src/vespa/searchcore/proton/matching/querynodes.cpp index 63302424db6..b0d45ee0684 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/querynodes.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/querynodes.cpp @@ -30,7 +30,7 @@ namespace proton::matching { ProtonTermData::ProtonTermData() = default; ProtonTermData::ProtonTermData(const ProtonTermData &) = default; ProtonTermData & ProtonTermData::operator = (const ProtonTermData &) = default; -ProtonTermData::~ProtonTermData() { } +ProtonTermData::~ProtonTermData() = default; void ProtonTermData::setDocumentFrequency(double freq) @@ -42,9 +42,9 @@ ProtonTermData::setDocumentFrequency(double freq) void ProtonTermData::resolve(const ViewResolver &resolver, - const IIndexEnvironment &idxEnv, - const string &view, - bool forceFilter) + const IIndexEnvironment &idxEnv, + const string &view, + bool forceFilter) { std::vector<string> fields; resolver.resolve(((view == "") ? "default" : view), fields); diff --git a/searchcore/src/vespa/searchcore/proton/matching/querynodes.h b/searchcore/src/vespa/searchcore/proton/matching/querynodes.h index ca6c8c75310..986c5d2dde9 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/querynodes.h +++ b/searchcore/src/vespa/searchcore/proton/matching/querynodes.h @@ -104,13 +104,15 @@ struct ProtonTerm : public ProtonTermBase<Base> { ~ProtonTerm() {} }; -typedef search::query::SimpleAnd ProtonAnd; -typedef search::query::SimpleAndNot ProtonAndNot; -typedef search::query::SimpleNear ProtonNear; -typedef search::query::SimpleONear ProtonONear; -typedef search::query::SimpleOr ProtonOr; -typedef search::query::SimpleRank ProtonRank; -typedef search::query::SimpleWeakAnd ProtonWeakAnd; +typedef search::query::SimpleAnd ProtonAnd; +typedef search::query::SimpleAndNot ProtonAndNot; +typedef search::query::SimpleNear ProtonNear; +typedef search::query::SimpleONear ProtonONear; +typedef search::query::SimpleOr ProtonOr; +typedef search::query::SimpleRank ProtonRank; +typedef search::query::SimpleWeakAnd ProtonWeakAnd; +typedef search::query::SimpleSameElement ProtonSameElement; + struct ProtonEquiv final : public ProtonTermBase<search::query::Equiv> { @@ -121,6 +123,7 @@ struct ProtonEquiv final : public ProtonTermBase<search::query::Equiv> typedef ProtonTerm<search::query::LocationTerm> ProtonLocationTerm; typedef ProtonTerm<search::query::NumberTerm> ProtonNumberTerm; typedef ProtonTerm<search::query::Phrase> ProtonPhrase; + typedef ProtonTerm<search::query::PrefixTerm> ProtonPrefixTerm; typedef ProtonTerm<search::query::RangeTerm> ProtonRangeTerm; typedef ProtonTerm<search::query::StringTerm> ProtonStringTerm; @@ -142,6 +145,7 @@ struct ProtonNodeTypes { typedef ProtonONear ONear; typedef ProtonOr Or; typedef ProtonPhrase Phrase; + typedef ProtonSameElement SameElement; typedef ProtonPrefixTerm PrefixTerm; typedef ProtonRangeTerm RangeTerm; typedef ProtonRank Rank; diff --git a/searchcore/src/vespa/searchcore/proton/matching/termdatafromnode.cpp b/searchcore/src/vespa/searchcore/proton/matching/termdatafromnode.cpp index 26b765cf3cf..3fd4000bf9f 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/termdatafromnode.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/termdatafromnode.cpp @@ -4,8 +4,7 @@ #include "querynodes.h" #include <vespa/searchlib/query/tree/customtypevisitor.h> -namespace proton { -namespace matching { +namespace proton::matching { namespace { struct TermDataFromTermVisitor @@ -19,29 +18,30 @@ struct TermDataFromTermVisitor data = &n; } - virtual void visit(ProtonAnd &) override {} - virtual void visit(ProtonAndNot &) override {} - virtual void visit(ProtonNear &) override {} - virtual void visit(ProtonONear &) override {} - virtual void visit(ProtonOr &) override {} - virtual void visit(ProtonRank &) override {} - virtual void visit(ProtonWeakAnd &) override {} - - virtual void visit(ProtonWeightedSetTerm &n) override { visitTerm(n); } - virtual void visit(ProtonDotProduct &n) override { visitTerm(n); } - virtual void visit(ProtonWandTerm &n) override { visitTerm(n); } - virtual void visit(ProtonPhrase &n) override { visitTerm(n); } - virtual void visit(ProtonEquiv &n) override { visitTerm(n); } - - virtual void visit(ProtonNumberTerm &n) override { visitTerm(n); } - virtual void visit(ProtonLocationTerm &n) override { visitTerm(n); } - virtual void visit(ProtonPrefixTerm &n) override { visitTerm(n); } - virtual void visit(ProtonRangeTerm &n) override { visitTerm(n); } - virtual void visit(ProtonStringTerm &n) override { visitTerm(n); } - virtual void visit(ProtonSubstringTerm &n) override { visitTerm(n); } - virtual void visit(ProtonSuffixTerm &n) override { visitTerm(n); } - virtual void visit(ProtonPredicateQuery &) override { } - virtual void visit(ProtonRegExpTerm &n) override { visitTerm(n); } + void visit(ProtonAnd &) override {} + void visit(ProtonAndNot &) override {} + void visit(ProtonNear &) override {} + void visit(ProtonONear &) override {} + void visit(ProtonOr &) override {} + void visit(ProtonRank &) override {} + void visit(ProtonWeakAnd &) override {} + void visit(ProtonSameElement &) override { } + + void visit(ProtonWeightedSetTerm &n) override { visitTerm(n); } + void visit(ProtonDotProduct &n) override { visitTerm(n); } + void visit(ProtonWandTerm &n) override { visitTerm(n); } + void visit(ProtonPhrase &n) override { visitTerm(n); } + void visit(ProtonEquiv &n) override { visitTerm(n); } + + void visit(ProtonNumberTerm &n) override { visitTerm(n); } + void visit(ProtonLocationTerm &n) override { visitTerm(n); } + void visit(ProtonPrefixTerm &n) override { visitTerm(n); } + void visit(ProtonRangeTerm &n) override { visitTerm(n); } + void visit(ProtonStringTerm &n) override { visitTerm(n); } + void visit(ProtonSubstringTerm &n) override { visitTerm(n); } + void visit(ProtonSuffixTerm &n) override { visitTerm(n); } + void visit(ProtonPredicateQuery &) override { } + void visit(ProtonRegExpTerm &n) override { visitTerm(n); } }; } // namespace @@ -53,5 +53,4 @@ termDataFromNode(const search::query::Node &node) return visitor.data; } -} // namespace matching -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/termdatafromnode.h b/searchcore/src/vespa/searchcore/proton/matching/termdatafromnode.h index 3e78ace1900..e445dbcfc50 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/termdatafromnode.h +++ b/searchcore/src/vespa/searchcore/proton/matching/termdatafromnode.h @@ -2,16 +2,13 @@ #pragma once - namespace search { namespace query { class Node; } } -namespace proton { -namespace matching { +namespace proton::matching { class ProtonTermData; const ProtonTermData *termDataFromNode(const search::query::Node &node); -} // namespace matching -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index ad5372b4af6..90b9bbc7f34 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -96,6 +96,14 @@ void FeedHandler::performPut(FeedToken token, PutOperation &op) { } return; } + /* + * Check if document type repos are equal. DocumentTypeRepoFactory::make + * returns the same document type repo if document type configs are equal, + * thus we can just perform a cheaper identity check here. + */ + if (_repo != op.getDocument()->getRepo()) { + op.deserializeDocument(*_repo); + } storeOperation(op, token); if (token) { token->setResult(make_unique<Result>(), false); @@ -344,6 +352,8 @@ FeedHandler::FeedHandler(IThreadingService &writeService, _feedLock(), _feedState(make_shared<InitState>(getDocTypeName())), _activeFeedView(nullptr), + _repo(nullptr), + _documentType(nullptr), _bucketDBHandler(nullptr), _syncLock(), _syncedSerialNum(0), @@ -408,6 +418,14 @@ void FeedHandler::changeToNormalFeedState() { changeFeedState(make_shared<NormalState>(*this)); } +void +FeedHandler::setActiveFeedView(IFeedView *feedView) +{ + _activeFeedView = feedView; + _repo = feedView->getDocumentTypeRepo().get(); + _documentType = _repo->getDocumentType(_docTypeName.getName()); +} + bool FeedHandler::isDoingReplay() const { return _tlsMgr.isDoingReplay(); @@ -492,22 +510,17 @@ FeedHandler::considerWriteOperationForRejection(FeedToken & token, const FeedOpe } bool -FeedHandler::considerUpdateOperationForRejection(FeedToken &token, const UpdateOperation &op) +FeedHandler::considerUpdateOperationForRejection(FeedToken &token, UpdateOperation &op) { - const auto *repo = _activeFeedView->getDocumentTypeRepo().get(); const auto &update = *op.getUpdate(); /* * Check if document types are equal. DocumentTypeRepoFactory::make returns * the same document type repo if document type configs are equal, thus we * can just perform a cheaper identity check here. */ - if (repo->getDocumentType(_docTypeName.getName()) != &update.getType()) { + if (_documentType != &update.getType()) { try { - vespalib::nbostream stream; - op.serialize(stream); - UpdateOperation checkOp(op.getType()); - vespalib::nbostream checkStream(stream.peek(), stream.size()); - checkOp.deserialize(stream, *repo); + op.deserializeUpdate(*_repo); } catch (document::FieldNotFoundException &e) { if (token) { auto message = make_string("Update operation rejected for document '%s' of type '%s': 'Field not found'", @@ -516,6 +529,14 @@ FeedHandler::considerUpdateOperationForRejection(FeedToken &token, const UpdateO token->fail(); } return true; + } catch (document::DocumentTypeNotFoundException &e) { + auto message = make_string("Update operation rejected for document '%s' of type '%s': 'Uknown document type', expected '%s'", + update.getId().toString().c_str(), + e.getDocumentTypeName().c_str(), + _docTypeName.toString().c_str()); + token->setResult(make_unique<UpdateResult>(Result::TRANSIENT_ERROR, message), false); + token->fail(); + return true; } } return false; diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index faf909080d9..34f979b7115 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -90,6 +90,8 @@ private: FeedStateSP _feedState; // used by master write thread tasks IFeedView *_activeFeedView; + const document::DocumentTypeRepo *_repo; + const document::DocumentType *_documentType; bucketdb::IBucketDBHandler *_bucketDBHandler; std::mutex _syncLock; SerialNum _syncedSerialNum; @@ -102,7 +104,7 @@ private: void doHandleOperation(FeedToken token, FeedOperationUP op); bool considerWriteOperationForRejection(FeedToken & token, const FeedOperation &op); - bool considerUpdateOperationForRejection(FeedToken &token, const UpdateOperation &op); + bool considerUpdateOperationForRejection(FeedToken &token, UpdateOperation &op); /** * Delayed execution of feed operations against feed view, in @@ -203,9 +205,7 @@ public: * Update the active feed view. * Always called by the master write thread so locking is not needed. */ - void setActiveFeedView(IFeedView *feedView) { - _activeFeedView = feedView; - } + void setActiveFeedView(IFeedView *feedView); void setBucketDBHandler(bucketdb::IBucketDBHandler *bucketDBHandler) { _bucketDBHandler = bucketDBHandler; diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexcollection.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexcollection.cpp index 8c8ea8e6f28..de4cde956d0 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexcollection.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexcollection.cpp @@ -179,28 +179,30 @@ private: _result = mixer.mix(); } - virtual void visit(And &) override { } - virtual void visit(AndNot &) override { } - virtual void visit(Or &) override { } - virtual void visit(WeakAnd &) override { } - virtual void visit(Equiv &) override { } - virtual void visit(Rank &) override { } - virtual void visit(Near &) override { } - virtual void visit(ONear &) override { } - - virtual void visit(WeightedSetTerm &n) override { visitTerm(n); } - virtual void visit(DotProduct &n) override { visitTerm(n); } - virtual void visit(WandTerm &n) override { visitTerm(n); } - virtual void visit(Phrase &n) override { visitTerm(n); } - virtual void visit(NumberTerm &n) override { visitTerm(n); } - virtual void visit(LocationTerm &n) override { visitTerm(n); } - virtual void visit(PrefixTerm &n) override { visitTerm(n); } - virtual void visit(RangeTerm &n) override { visitTerm(n); } - virtual void visit(StringTerm &n) override { visitTerm(n); } - virtual void visit(SubstringTerm &n) override { visitTerm(n); } - virtual void visit(SuffixTerm &n) override { visitTerm(n); } - virtual void visit(PredicateQuery &n) override { visitTerm(n); } - virtual void visit(RegExpTerm &n) override { visitTerm(n); } + void visit(And &) override { } + void visit(AndNot &) override { } + void visit(Or &) override { } + void visit(WeakAnd &) override { } + void visit(Equiv &) override { } + void visit(Rank &) override { } + void visit(Near &) override { } + void visit(ONear &) override { } + void visit(SameElement &) override { } + + + void visit(WeightedSetTerm &n) override { visitTerm(n); } + void visit(DotProduct &n) override { visitTerm(n); } + void visit(WandTerm &n) override { visitTerm(n); } + void visit(Phrase &n) override { visitTerm(n); } + void visit(NumberTerm &n) override { visitTerm(n); } + void visit(LocationTerm &n) override { visitTerm(n); } + void visit(PrefixTerm &n) override { visitTerm(n); } + void visit(RangeTerm &n) override { visitTerm(n); } + void visit(StringTerm &n) override { visitTerm(n); } + void visit(SubstringTerm &n) override { visitTerm(n); } + void visit(SuffixTerm &n) override { visitTerm(n); } + void visit(PredicateQuery &n) override { visitTerm(n); } + void visit(RegExpTerm &n) override { visitTerm(n); } public: CreateBlueprintVisitor(const IIndexCollection &indexes, diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index 3b321f4a12f..6e4a3ef1e1f 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -194,6 +194,7 @@ vespa_define_module( src/tests/queryeval/multibitvectoriterator src/tests/queryeval/parallel_weak_and src/tests/queryeval/predicate + src/tests/queryeval/same_element src/tests/queryeval/simple_phrase src/tests/queryeval/sourceblender src/tests/queryeval/sparse_vector_benchmark diff --git a/searchlib/src/tests/query/customtypevisitor_test.cpp b/searchlib/src/tests/query/customtypevisitor_test.cpp index 5aabb328354..a941694d375 100644 --- a/searchlib/src/tests/query/customtypevisitor_test.cpp +++ b/searchlib/src/tests/query/customtypevisitor_test.cpp @@ -1,14 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for customtypevisitor. -#include <vespa/log/log.h> -LOG_SETUP("customtypevisitor_test"); - #include <vespa/searchlib/query/tree/customtypevisitor.h> #include <vespa/searchlib/query/tree/intermediatenodes.h> #include <vespa/searchlib/query/tree/termnodes.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/log/log.h> +LOG_SETUP("customtypevisitor_test"); + using std::string; using namespace search::query; @@ -39,6 +39,7 @@ struct MyNear : Near { MyNear() : Near(1) {} }; struct MyONear : ONear { MyONear() : ONear(1) {} }; struct MyOr : Or {}; struct MyPhrase : Phrase { MyPhrase() : Phrase("view", 0, Weight(42)) {} }; +struct MySameElement : SameElement { MySameElement() : SameElement("view") {} }; struct MyRank : Rank {}; struct MyNumberTerm : InitTerm<NumberTerm> {}; struct MyLocationTerm : InitTerm<LocationTerm> {}; @@ -64,6 +65,7 @@ struct MyQueryNodeTypes { typedef MyONear ONear; typedef MyOr Or; typedef MyPhrase Phrase; + typedef MySameElement SameElement; typedef MyPrefixTerm PrefixTerm; typedef MyRangeTerm RangeTerm; typedef MyRank Rank; @@ -89,27 +91,28 @@ public: template <typename T> void setVisited() { isVisited<T>() = true; } - virtual void visit(MyAnd &) override { setVisited<MyAnd>(); } - virtual void visit(MyAndNot &) override { setVisited<MyAndNot>(); } - virtual void visit(MyEquiv &) override { setVisited<MyEquiv>(); } - virtual void visit(MyNumberTerm &) override { setVisited<MyNumberTerm>(); } - virtual void visit(MyLocationTerm &) override { setVisited<MyLocationTerm>(); } - virtual void visit(MyNear &) override { setVisited<MyNear>(); } - virtual void visit(MyONear &) override { setVisited<MyONear>(); } - virtual void visit(MyOr &) override { setVisited<MyOr>(); } - virtual void visit(MyPhrase &) override { setVisited<MyPhrase>(); } - virtual void visit(MyPrefixTerm &) override { setVisited<MyPrefixTerm>(); } - virtual void visit(MyRangeTerm &) override { setVisited<MyRangeTerm>(); } - virtual void visit(MyRank &) override { setVisited<MyRank>(); } - virtual void visit(MyStringTerm &) override { setVisited<MyStringTerm>(); } - virtual void visit(MySubstrTerm &) override { setVisited<MySubstrTerm>(); } - virtual void visit(MySuffixTerm &) override { setVisited<MySuffixTerm>(); } - virtual void visit(MyWeakAnd &) override { setVisited<MyWeakAnd>(); } - virtual void visit(MyWeightedSetTerm &) override { setVisited<MyWeightedSetTerm>(); } - virtual void visit(MyDotProduct &) override { setVisited<MyDotProduct>(); } - virtual void visit(MyWandTerm &) override { setVisited<MyWandTerm>(); } - virtual void visit(MyPredicateQuery &) override { setVisited<MyPredicateQuery>(); } - virtual void visit(MyRegExpTerm &) override { setVisited<MyRegExpTerm>(); } + void visit(MyAnd &) override { setVisited<MyAnd>(); } + void visit(MyAndNot &) override { setVisited<MyAndNot>(); } + void visit(MyEquiv &) override { setVisited<MyEquiv>(); } + void visit(MyNumberTerm &) override { setVisited<MyNumberTerm>(); } + void visit(MyLocationTerm &) override { setVisited<MyLocationTerm>(); } + void visit(MyNear &) override { setVisited<MyNear>(); } + void visit(MyONear &) override { setVisited<MyONear>(); } + void visit(MyOr &) override { setVisited<MyOr>(); } + void visit(MyPhrase &) override { setVisited<MyPhrase>(); } + void visit(MySameElement &) override { setVisited<MySameElement>(); } + void visit(MyPrefixTerm &) override { setVisited<MyPrefixTerm>(); } + void visit(MyRangeTerm &) override { setVisited<MyRangeTerm>(); } + void visit(MyRank &) override { setVisited<MyRank>(); } + void visit(MyStringTerm &) override { setVisited<MyStringTerm>(); } + void visit(MySubstrTerm &) override { setVisited<MySubstrTerm>(); } + void visit(MySuffixTerm &) override { setVisited<MySuffixTerm>(); } + void visit(MyWeakAnd &) override { setVisited<MyWeakAnd>(); } + void visit(MyWeightedSetTerm &) override { setVisited<MyWeightedSetTerm>(); } + void visit(MyDotProduct &) override { setVisited<MyDotProduct>(); } + void visit(MyWandTerm &) override { setVisited<MyWandTerm>(); } + void visit(MyPredicateQuery &) override { setVisited<MyPredicateQuery>(); } + void visit(MyRegExpTerm &) override { setVisited<MyRegExpTerm>(); } }; template <class T> diff --git a/searchlib/src/tests/query/query_visitor_test.cpp b/searchlib/src/tests/query/query_visitor_test.cpp index 9e82c6ea5ec..f8922c54a4e 100644 --- a/searchlib/src/tests/query/query_visitor_test.cpp +++ b/searchlib/src/tests/query/query_visitor_test.cpp @@ -1,9 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for query_visitor. -#include <vespa/log/log.h> -LOG_SETUP("query_visitor_test"); - #include <vespa/searchlib/query/tree/intermediatenodes.h> #include <vespa/searchlib/query/tree/point.h> #include <vespa/searchlib/query/tree/queryvisitor.h> @@ -11,6 +8,9 @@ LOG_SETUP("query_visitor_test"); #include <vespa/searchlib/query/tree/termnodes.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/log/log.h> +LOG_SETUP("query_visitor_test"); + using namespace search::query; namespace { @@ -43,29 +43,28 @@ public: return b; } - virtual void visit(And &) override { isVisited<And>() = true; } - virtual void visit(AndNot &) override { isVisited<AndNot>() = true; } - virtual void visit(Equiv &) override { isVisited<Equiv>() = true; } - virtual void visit(NumberTerm &) override { isVisited<NumberTerm>() = true; } - virtual void visit(LocationTerm &) override { isVisited<LocationTerm>() = true; } - virtual void visit(Near &) override { isVisited<Near>() = true; } - virtual void visit(ONear &) override { isVisited<ONear>() = true; } - virtual void visit(Or &) override { isVisited<Or>() = true; } - virtual void visit(Phrase &) override { isVisited<Phrase>() = true; } - virtual void visit(PrefixTerm &) override { isVisited<PrefixTerm>() = true; } - virtual void visit(RangeTerm &) override { isVisited<RangeTerm>() = true; } - virtual void visit(Rank &) override { isVisited<Rank>() = true; } - virtual void visit(StringTerm &) override { isVisited<StringTerm>() = true; } - virtual void visit(SubstringTerm &) override { isVisited<SubstringTerm>() = true; } - virtual void visit(SuffixTerm &) override { isVisited<SuffixTerm>() = true; } - virtual void visit(WeakAnd &) override { isVisited<WeakAnd>() = true; } - virtual void visit(WeightedSetTerm &) override - { isVisited<WeightedSetTerm>() = true; } - virtual void visit(DotProduct &) override { isVisited<DotProduct>() = true; } - virtual void visit(WandTerm &) override { isVisited<WandTerm>() = true; } - virtual void visit(PredicateQuery &) override - { isVisited<PredicateQuery>() = true; } - virtual void visit(RegExpTerm &) override { isVisited<RegExpTerm>() = true; } + void visit(And &) override { isVisited<And>() = true; } + void visit(AndNot &) override { isVisited<AndNot>() = true; } + void visit(Equiv &) override { isVisited<Equiv>() = true; } + void visit(NumberTerm &) override { isVisited<NumberTerm>() = true; } + void visit(LocationTerm &) override { isVisited<LocationTerm>() = true; } + void visit(Near &) override { isVisited<Near>() = true; } + void visit(ONear &) override { isVisited<ONear>() = true; } + void visit(Or &) override { isVisited<Or>() = true; } + void visit(Phrase &) override { isVisited<Phrase>() = true; } + void visit(SameElement &) override { isVisited<SameElement>() = true; } + void visit(PrefixTerm &) override { isVisited<PrefixTerm>() = true; } + void visit(RangeTerm &) override { isVisited<RangeTerm>() = true; } + void visit(Rank &) override { isVisited<Rank>() = true; } + void visit(StringTerm &) override { isVisited<StringTerm>() = true; } + void visit(SubstringTerm &) override { isVisited<SubstringTerm>() = true; } + void visit(SuffixTerm &) override { isVisited<SuffixTerm>() = true; } + void visit(WeakAnd &) override { isVisited<WeakAnd>() = true; } + void visit(WeightedSetTerm &) override { isVisited<WeightedSetTerm>() = true; } + void visit(DotProduct &) override { isVisited<DotProduct>() = true; } + void visit(WandTerm &) override { isVisited<WandTerm>() = true; } + void visit(PredicateQuery &) override { isVisited<PredicateQuery>() = true; } + void visit(RegExpTerm &) override { isVisited<RegExpTerm>() = true; } }; template <class T> @@ -84,27 +83,20 @@ void Test::requireThatAllNodesCanBeVisited() { checkVisit<ONear>(new SimpleONear(0)); checkVisit<Or>(new SimpleOr); checkVisit<Phrase>(new SimplePhrase("field", 0, Weight(42))); - checkVisit<WeightedSetTerm>( - new SimpleWeightedSetTerm("field", 0, Weight(42))); + checkVisit<SameElement>(new SimpleSameElement("field")); + checkVisit<WeightedSetTerm>(new SimpleWeightedSetTerm("field", 0, Weight(42))); checkVisit<DotProduct>(new SimpleDotProduct("field", 0, Weight(42))); - checkVisit<WandTerm>( - new SimpleWandTerm("field", 0, Weight(42), 57, 67, 77.7)); + checkVisit<WandTerm>(new SimpleWandTerm("field", 0, Weight(42), 57, 67, 77.7)); checkVisit<Rank>(new SimpleRank); - checkVisit<NumberTerm>( - new SimpleNumberTerm("0.42", "field", 0, Weight(0))); + checkVisit<NumberTerm>(new SimpleNumberTerm("0.42", "field", 0, Weight(0))); const Location location(Point(10, 10), 20, 0); - checkVisit<LocationTerm>( - new SimpleLocationTerm(location, "field", 0, Weight(0))); + checkVisit<LocationTerm>(new SimpleLocationTerm(location, "field", 0, Weight(0))); checkVisit<PrefixTerm>(new SimplePrefixTerm("t", "field", 0, Weight(0))); - checkVisit<RangeTerm>( - new SimpleRangeTerm(Range(0, 1), "field", 0, Weight(0))); + checkVisit<RangeTerm>(new SimpleRangeTerm(Range(0, 1), "field", 0, Weight(0))); checkVisit<StringTerm>(new SimpleStringTerm("t", "field", 0, Weight(0))); - checkVisit<SubstringTerm>( - new SimpleSubstringTerm("t", "field", 0, Weight(0))); + checkVisit<SubstringTerm>(new SimpleSubstringTerm("t", "field", 0, Weight(0))); checkVisit<SuffixTerm>(new SimpleSuffixTerm("t", "field", 0, Weight(0))); - checkVisit<PredicateQuery>( - new SimplePredicateQuery(PredicateQueryTerm::UP(), - "field", 0, Weight(0))); + checkVisit<PredicateQuery>(new SimplePredicateQuery(PredicateQueryTerm::UP(), "field", 0, Weight(0))); checkVisit<RegExpTerm>(new SimpleRegExpTerm("t", "field", 0, Weight(0))); } diff --git a/searchlib/src/tests/query/querybuilder_test.cpp b/searchlib/src/tests/query/querybuilder_test.cpp index b968f794833..0f05419c4e5 100644 --- a/searchlib/src/tests/query/querybuilder_test.cpp +++ b/searchlib/src/tests/query/querybuilder_test.cpp @@ -1,22 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for querybuilder. -#include <vespa/log/log.h> -LOG_SETUP("querybuilder_test"); - #include <vespa/searchlib/parsequery/parse.h> #include <vespa/searchlib/parsequery/simplequerystack.h> #include <vespa/searchlib/query/tree/customtypevisitor.h> -#include <vespa/searchlib/query/tree/intermediatenodes.h> #include <vespa/searchlib/query/tree/point.h> #include <vespa/searchlib/query/tree/querybuilder.h> -#include <vespa/searchlib/query/tree/querytreecreator.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/query/tree/stackdumpcreator.h> -#include <vespa/searchlib/query/tree/termnodes.h> -#include <vespa/searchlib/util/rawbuf.h> #include <vespa/vespalib/testkit/test_kit.h> -#include <string> + +#include <vespa/log/log.h> +LOG_SETUP("querybuilder_test"); +#include <vespa/searchlib/query/tree/querytreecreator.h> using std::string; using search::SimpleQueryStackDumpIterator; @@ -52,7 +48,7 @@ PredicateQueryTerm::UP getPredicateQueryTerm() { template <class NodeTypes> Node::UP createQueryTree() { QueryBuilder<NodeTypes> builder; - builder.addAnd(9); + builder.addAnd(10); { builder.addRank(2); { @@ -109,6 +105,12 @@ Node::UP createQueryTree() { builder.addStringTerm(str[2], view[2], id[2], weight[2]); } builder.addRegExpTerm(str[5], view[5], id[5], weight[5]); + builder.addSameElement(3, view[4]); + { + builder.addStringTerm(str[4], view[4], id[4], weight[5]); + builder.addStringTerm(str[5], view[5], id[5], weight[6]); + builder.addStringTerm(str[6], view[6], id[6], weight[7]); + } } Node::UP node = builder.build(); ASSERT_TRUE(node.get()); @@ -146,6 +148,7 @@ void checkQueryTreeTypes(Node *node) { //typedef typename NodeTypes::NumberTerm FloatTrm; typedef typename NodeTypes::Near Near; typedef typename NodeTypes::ONear ONear; + typedef typename NodeTypes::SameElement SameElement; typedef typename NodeTypes::Or Or; typedef typename NodeTypes::Phrase Phrase; typedef typename NodeTypes::PrefixTerm PrefixTerm; @@ -165,7 +168,7 @@ void checkQueryTreeTypes(Node *node) { ASSERT_TRUE(node); And *and_node = dynamic_cast<And *>(node); ASSERT_TRUE(and_node); - EXPECT_EQUAL(9u, and_node->getChildren().size()); + EXPECT_EQUAL(10u, and_node->getChildren().size()); Rank *rank = dynamic_cast<Rank *>(and_node->getChildren()[0]); @@ -176,22 +179,18 @@ void checkQueryTreeTypes(Node *node) { ASSERT_TRUE(near); EXPECT_EQUAL(2u, near->getChildren().size()); EXPECT_EQUAL(distance, near->getDistance()); - StringTerm *string_term = - dynamic_cast<StringTerm *>(near->getChildren()[0]); + StringTerm *string_term = dynamic_cast<StringTerm *>(near->getChildren()[0]); EXPECT_TRUE(checkTerm(string_term, str[0], view[0], id[0], weight[0])); - SubstringTerm *substring_term = - dynamic_cast<SubstringTerm *>(near->getChildren()[1]); + SubstringTerm *substring_term = dynamic_cast<SubstringTerm *>(near->getChildren()[1]); EXPECT_TRUE(checkTerm(substring_term, str[1], view[1], id[1], weight[1])); ONear *onear = dynamic_cast<ONear *>(rank->getChildren()[1]); ASSERT_TRUE(onear); EXPECT_EQUAL(2u, onear->getChildren().size()); EXPECT_EQUAL(distance, onear->getDistance()); - SuffixTerm *suffix_term = - dynamic_cast<SuffixTerm *>(onear->getChildren()[0]); + SuffixTerm *suffix_term = dynamic_cast<SuffixTerm *>(onear->getChildren()[0]); EXPECT_TRUE(checkTerm(suffix_term, str[2], view[2], id[2], weight[2])); - PrefixTerm *prefix_term = - dynamic_cast<PrefixTerm *>(onear->getChildren()[1]); + PrefixTerm *prefix_term = dynamic_cast<PrefixTerm *>(onear->getChildren()[1]); EXPECT_TRUE(checkTerm(prefix_term, str[3], view[3], id[3], weight[3])); @@ -224,26 +223,20 @@ void checkQueryTreeTypes(Node *node) { AndNot *and_not = dynamic_cast<AndNot *>(or_node->getChildren()[2]); ASSERT_TRUE(and_not); EXPECT_EQUAL(2u, and_not->getChildren().size()); - NumberTerm *integer_term = - dynamic_cast<NumberTerm *>(and_not->getChildren()[0]); + NumberTerm *integer_term = dynamic_cast<NumberTerm *>(and_not->getChildren()[0]); EXPECT_TRUE(checkTerm(integer_term, int1, view[7], id[7], weight[7])); - NumberTerm *float_term = - dynamic_cast<NumberTerm *>(and_not->getChildren()[1]); - EXPECT_TRUE(checkTerm(float_term, float1, view[8], id[8], weight[8], - false)); + NumberTerm *float_term = dynamic_cast<NumberTerm *>(and_not->getChildren()[1]); + EXPECT_TRUE(checkTerm(float_term, float1, view[8], id[8], weight[8], false)); - RangeTerm *range_term = - dynamic_cast<RangeTerm *>(and_node->getChildren()[2]); + RangeTerm *range_term = dynamic_cast<RangeTerm *>(and_node->getChildren()[2]); ASSERT_TRUE(range_term); EXPECT_TRUE(checkTerm(range_term, range, view[9], id[9], weight[9])); - LocationTerm *loc_term = - dynamic_cast<LocationTerm *>(and_node->getChildren()[3]); + LocationTerm *loc_term = dynamic_cast<LocationTerm *>(and_node->getChildren()[3]); ASSERT_TRUE(loc_term); EXPECT_TRUE(checkTerm(loc_term, location, view[10], id[10], weight[10])); - WeakAnd *wand = dynamic_cast<WeakAnd *>(and_node->getChildren()[4]); ASSERT_TRUE(wand != 0); EXPECT_EQUAL(123u, wand->getMinHits()); @@ -253,15 +246,12 @@ void checkQueryTreeTypes(Node *node) { string_term = dynamic_cast<StringTerm *>(wand->getChildren()[1]); EXPECT_TRUE(checkTerm(string_term, str[5], view[5], id[5], weight[5])); - PredicateQuery *predicateQuery = - dynamic_cast<PredicateQuery *>(and_node->getChildren()[5]); + PredicateQuery *predicateQuery = dynamic_cast<PredicateQuery *>(and_node->getChildren()[5]); ASSERT_TRUE(predicateQuery); PredicateQueryTerm::UP pqt(new PredicateQueryTerm); - EXPECT_TRUE(checkTerm(predicateQuery, getPredicateQueryTerm(), - view[3], id[3], weight[3])); + EXPECT_TRUE(checkTerm(predicateQuery, getPredicateQueryTerm(), view[3], id[3], weight[3])); - DotProduct *dotProduct = - dynamic_cast<DotProduct *>(and_node->getChildren()[6]); + DotProduct *dotProduct = dynamic_cast<DotProduct *>(and_node->getChildren()[6]); ASSERT_TRUE(dotProduct); EXPECT_EQUAL(3u, dotProduct->getChildren().size()); string_term = dynamic_cast<StringTerm *>(dotProduct->getChildren()[0]); @@ -282,9 +272,20 @@ void checkQueryTreeTypes(Node *node) { string_term = dynamic_cast<StringTerm *>(wandTerm->getChildren()[1]); EXPECT_TRUE(checkTerm(string_term, str[2], view[2], id[2], weight[2])); - RegExpTerm *regexp_term = - dynamic_cast<RegExpTerm *>(and_node->getChildren()[8]); + RegExpTerm *regexp_term = dynamic_cast<RegExpTerm *>(and_node->getChildren()[8]); EXPECT_TRUE(checkTerm(regexp_term, str[5], view[5], id[5], weight[5])); + + SameElement *same = dynamic_cast<SameElement *>(and_node->getChildren()[9]); + ASSERT_TRUE(same != nullptr); + EXPECT_EQUAL(view[4], same->getView()); + EXPECT_EQUAL(3u, same->getChildren().size()); + string_term = dynamic_cast<StringTerm *>(same->getChildren()[0]); + EXPECT_TRUE(checkTerm(string_term, str[4], view[4], id[4], weight[5])); + string_term = dynamic_cast<StringTerm *>(same->getChildren()[1]); + EXPECT_TRUE(checkTerm(string_term, str[5], view[5], id[5], weight[6])); + string_term = dynamic_cast<StringTerm *>(same->getChildren()[2]); + EXPECT_TRUE(checkTerm(string_term, str[6], view[6], id[6], weight[7])); + } struct AbstractTypes { @@ -294,6 +295,7 @@ struct AbstractTypes { typedef search::query::LocationTerm LocationTerm; typedef search::query::Near Near; typedef search::query::ONear ONear; + typedef search::query::SameElement SameElement; typedef search::query::Or Or; typedef search::query::Phrase Phrase; typedef search::query::PrefixTerm PrefixTerm; @@ -333,9 +335,9 @@ struct MyNear : Near { MyNear(size_t dist) : Near(dist) {} }; struct MyONear : ONear { MyONear(size_t dist) : ONear(dist) {} }; struct MyWeakAnd : WeakAnd { MyWeakAnd(uint32_t minHits, const vespalib::string & v) : WeakAnd(minHits, v) {} }; struct MyOr : Or {}; -struct MyPhrase : Phrase { - MyPhrase(const string &f, int32_t i, Weight w) : Phrase(f, i, w) {} -}; +struct MyPhrase : Phrase { MyPhrase(const string &f, int32_t i, Weight w) : Phrase(f, i, w) {}}; +struct MySameElement : SameElement { MySameElement(const string &f) : SameElement(f) {}}; + struct MyWeightedSetTerm : WeightedSetTerm { MyWeightedSetTerm(const string &f, int32_t i, Weight w) : WeightedSetTerm(f, i, w) {} }; @@ -404,6 +406,7 @@ struct MyQueryNodeTypes { typedef MyONear ONear; typedef MyOr Or; typedef MyPhrase Phrase; + typedef MySameElement SameElement; typedef MyPrefixTerm PrefixTerm; typedef MyRangeTerm RangeTerm; typedef MyRank Rank; @@ -556,8 +559,7 @@ TEST("require that Query Tree Creator Can Create Queries From Stack") { string stackDump = StackDumpCreator::create(*node); SimpleQueryStackDumpIterator iterator(stackDump); - Node::UP new_node = - QueryTreeCreator<SimpleQueryNodeTypes>::create(iterator); + Node::UP new_node = QueryTreeCreator<SimpleQueryNodeTypes>::create(iterator); checkQueryTreeTypes<SimpleQueryNodeTypes>(new_node.get()); } diff --git a/searchlib/src/tests/queryeval/same_element/CMakeLists.txt b/searchlib/src/tests/queryeval/same_element/CMakeLists.txt new file mode 100644 index 00000000000..97034c53376 --- /dev/null +++ b/searchlib/src/tests/queryeval/same_element/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_same_element_test_app TEST + SOURCES + same_element_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_same_element_test_app COMMAND searchlib_same_element_test_app) diff --git a/searchlib/src/tests/queryeval/same_element/same_element_test.cpp b/searchlib/src/tests/queryeval/same_element/same_element_test.cpp new file mode 100644 index 00000000000..e0c20949c8e --- /dev/null +++ b/searchlib/src/tests/queryeval/same_element/same_element_test.cpp @@ -0,0 +1,99 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/searchlib/queryeval/leaf_blueprints.h> +#include <vespa/searchlib/queryeval/simpleresult.h> +#include <vespa/searchlib/queryeval/same_element_blueprint.h> + +using namespace search::fef; +using namespace search::queryeval; + +std::unique_ptr<SameElementBlueprint> make_blueprint(const std::vector<FakeResult> &children) { + auto result = std::make_unique<SameElementBlueprint>(); + for (size_t i = 0; i < children.size(); ++i) { + uint32_t field_id = i; + vespalib::string field_name = vespalib::make_string("f%u", field_id); + FieldSpec field = result->getNextChildField(field_name, field_id); + result->addTerm(std::make_unique<FakeBlueprint>(field, children[i])); + } + return result; +} + +Blueprint::UP finalize(Blueprint::UP bp, bool strict) { + Blueprint::UP result = Blueprint::optimize(std::move(bp)); + result->fetchPostings(strict); + result->freeze(); + return result; +} + +SimpleResult find_matches(const std::vector<FakeResult> &children) { + auto md = MatchData::makeTestInstance(0, 0); + auto bp = finalize(make_blueprint(children), false); + auto search = bp->createSearch(*md, false); + return SimpleResult().search(*search, 1000); +} + +FakeResult make_result(const std::vector<std::pair<uint32_t,std::vector<uint32_t> > > &match_data) { + FakeResult result; + uint32_t pos_should_be_ignored = 0; + for (const auto &doc: match_data) { + result.doc(doc.first); + for (const auto &elem: doc.second) { + result.elem(elem).pos(++pos_should_be_ignored); + } + } + return result; +} + +TEST("require that simple match can be found") { + auto a = make_result({{5, {1,3,7}}}); + auto b = make_result({{5, {3,5,10}}}); + SimpleResult result = find_matches({a, b}); + SimpleResult expect({5}); + EXPECT_EQUAL(result, expect); +} + +TEST("require that children must match within same element") { + auto a = make_result({{5, {1,3,7}}}); + auto b = make_result({{5, {2,5,10}}}); + SimpleResult result = find_matches({a, b}); + SimpleResult expect({}); + EXPECT_EQUAL(result, expect); +} + +TEST("require that strict iterator seeks to next hit") { + auto md = MatchData::makeTestInstance(0, 0); + auto a = make_result({{5, {1,2}}, {7, {1,2}}, {8, {1,2}}, {9, {1,2}}}); + auto b = make_result({{5, {3}}, {6, {1,2}}, {7, {2,4}}, {9, {1}}}); + auto bp = finalize(make_blueprint({a,b}), true); + auto search = bp->createSearch(*md, true); + search->initRange(1, 1000); + EXPECT_LESS(search->getDocId(), 1u); + EXPECT_TRUE(!search->seek(1)); + EXPECT_EQUAL(search->getDocId(), 7u); + EXPECT_TRUE(search->seek(9)); + EXPECT_EQUAL(search->getDocId(), 9u); + EXPECT_TRUE(!search->seek(10)); + EXPECT_TRUE(search->isAtEnd()); +} + +TEST("require that results are estimated appropriately") { + auto a = make_result({{5, {0}}, {5, {0}}, {5, {0}}}); + auto b = make_result({{5, {0}}, {5, {0}}}); + auto c = make_result({{5, {0}}, {5, {0}}, {5, {0}}, {5, {0}}}); + auto bp = finalize(make_blueprint({a,b,c}), true); + EXPECT_EQUAL(bp->getState().estimate().estHits, 2u); +} + +TEST("require that children are sorted") { + auto a = make_result({{5, {0}}, {5, {0}}, {5, {0}}}); + auto b = make_result({{5, {0}}, {5, {0}}}); + auto c = make_result({{5, {0}}, {5, {0}}, {5, {0}}, {5, {0}}}); + auto bp = finalize(make_blueprint({a,b,c}), true); + EXPECT_EQUAL(dynamic_cast<SameElementBlueprint&>(*bp).terms()[0]->getState().estimate().estHits, 2u); + EXPECT_EQUAL(dynamic_cast<SameElementBlueprint&>(*bp).terms()[1]->getState().estimate().estHits, 3u); + EXPECT_EQUAL(dynamic_cast<SameElementBlueprint&>(*bp).terms()[2]->getState().estimate().estHits, 4u); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp b/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp index d3a99dc439a..8ad4578b6c1 100644 --- a/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp +++ b/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp @@ -244,6 +244,7 @@ StackDumpIteratorTest::RunTest(int testno, bool verify) stack.Push(new search::ParseItem(search::ParseItem::ITEM_NUMTERM, "foo", "[0;22]")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_PREFIXTERM, "bar", "baz")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_PHRASE, 3, "bar")); + stack.Push(new search::ParseItem(search::ParseItem::ITEM_SAME_ELEMENT, 3, "bar")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_OR, 2)); stack.Push(new search::ParseItem(search::ParseItem::ITEM_AND, 3)); stack.Push(new search::ParseItem(search::ParseItem::ITEM_RANK, 5)); diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index 4dbcc85d861..1ccc9923c99 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -73,7 +73,7 @@ AttributeVector::BaseName::BaseName(const vespalib::stringref &base, append(name); } -AttributeVector::BaseName::~BaseName() { } +AttributeVector::BaseName::~BaseName() = default; AttributeVector::BaseName::string @@ -165,9 +165,7 @@ AttributeVector::AttributeVector(const vespalib::stringref &baseFileName, const _genHandler(), _genHolder(), _status(Status::createName((_baseFileName.getIndexName() + - (_baseFileName.getSnapshotName().empty() ? - "" : - ".") + + (_baseFileName.getSnapshotName().empty() ? "" : ".") + _baseFileName.getSnapshotName()), _baseFileName.getAttributeName())), _highestValueCount(1), @@ -177,29 +175,24 @@ AttributeVector::AttributeVector(const vespalib::stringref &baseFileName, const _createSerialNum(0u), _compactLidSpaceGeneration(0u), _hasEnum(false), - _hasSortedEnum(false), _loaded(false), _enableEnumeratedSave(false) { } - -AttributeVector::~AttributeVector() { } +AttributeVector::~AttributeVector() = default; void AttributeVector::updateStat(bool force) { if (force) { onUpdateStat(); } else if (_nextStatUpdateTime < fastos::ClockSystem::now()) { onUpdateStat(); - _nextStatUpdateTime = fastos::ClockSystem::now() + - fastos::TimeStamp::SEC; + _nextStatUpdateTime = fastos::ClockSystem::now() + fastos::TimeStamp::SEC; } } -size_t AttributeVector::getFixedWidth() const { return _config.basicType().fixedSize(); } bool AttributeVector::hasEnum() const { return _hasEnum; } bool AttributeVector::hasEnum2Value() const { return false; } uint32_t AttributeVector::getMaxValueCount() const { return _highestValueCount; } -uint32_t AttributeVector::getNumDocs() const { return _status.getNumDocs(); } bool AttributeVector::isEnumerated(const vespalib::GenericHeader &header) @@ -217,13 +210,11 @@ AttributeVector::commit(bool forceUpdateStat) _loaded = true; } - void AttributeVector::commit(uint64_t firstSyncToken, uint64_t lastSyncToken) { if (firstSyncToken < getStatus().getLastSyncToken()) { - LOG(error, - "Expected first token to be >= %" PRIu64 ", got %" PRIu64 ".", + LOG(error, "Expected first token to be >= %" PRIu64 ", got %" PRIu64 ".", getStatus().getLastSyncToken(), firstSyncToken); abort(); } @@ -231,7 +222,6 @@ AttributeVector::commit(uint64_t firstSyncToken, uint64_t lastSyncToken) _status.setLastSyncToken(lastSyncToken); } - bool AttributeVector::addDocs(DocId &startDoc, DocId &lastDoc, uint32_t numDocs) { @@ -271,19 +261,10 @@ AttributeVector::incGeneration() void -AttributeVector::updateStatistics(uint64_t numValues, - uint64_t numUniqueValue, - uint64_t allocated, - uint64_t used, - uint64_t dead, - uint64_t onHold) +AttributeVector::updateStatistics(uint64_t numValues, uint64_t numUniqueValue, uint64_t allocated, + uint64_t used, uint64_t dead, uint64_t onHold) { - _status.updateStatistics(numValues, - numUniqueValue, - allocated, - used, - dead, - onHold); + _status.updateStatistics(numValues, numUniqueValue, allocated, used, dead, onHold); } AddressSpace @@ -292,16 +273,6 @@ AttributeVector::getEnumStoreAddressSpaceUsage() const return AddressSpaceUsage::defaultEnumStoreUsage(); } -bool -AttributeVector::hasMultiValue() const { - return _config.collectionType().isMultiValue(); -} - -bool -AttributeVector::hasWeightedSetType() const { - return _config.collectionType().isWeightedSet(); -} - AddressSpace AttributeVector::getMultiValueAddressSpaceUsage() const { @@ -311,38 +282,7 @@ AttributeVector::getMultiValueAddressSpaceUsage() const AddressSpaceUsage AttributeVector::getAddressSpaceUsage() const { - return AddressSpaceUsage(getEnumStoreAddressSpaceUsage(), - getMultiValueAddressSpaceUsage()); -} - -const vespalib::string & -AttributeVector::getName() const { - return _baseFileName.getAttributeName(); -} - -attribute::BasicType::Type -AttributeVector::getBasicType() const { - return getInternalBasicType().type(); -} -attribute::CollectionType::Type -AttributeVector::getCollectionType() const { - return getInternalCollectionType().type(); -} - -bool -AttributeVector::getIsFilter() const { - return _config.getIsFilter(); -} - -bool -AttributeVector::getIsFastSearch() const { - return _config.fastSearch(); -} - -uint32_t -AttributeVector::getCommittedDocIdLimit() const -{ - return _committedDocIdLimit; + return AddressSpaceUsage(getEnumStoreAddressSpaceUsage(), getMultiValueAddressSpaceUsage()); } bool diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index b25c7b67299..87ef9a41432 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -213,7 +213,6 @@ protected: void setEnumMax(uint32_t e) { _enumMax = e; setEnum(); } void setEnum(bool hasEnum_=true) { _hasEnum = hasEnum_; } - void setSortedEnum(bool sorted=true) { _hasSortedEnum = sorted; } void setNumDocs(uint32_t n) { _status.setNumDocs(n); } void incNumDocs() { _status.incNumDocs(); } @@ -398,7 +397,7 @@ public: bool isLoaded() const { return _loaded; } /** Return the fixed length of the attribute. If 0 then you must inquire each document. */ - virtual size_t getFixedWidth() const override; + size_t getFixedWidth() const override { return _config.basicType().fixedSize(); } const Config &getConfig() const { return _config; } BasicType getInternalBasicType() const { return _config.basicType(); } CollectionType getInternalCollectionType() const { return _config.collectionType(); } @@ -406,17 +405,16 @@ public: void setBaseFileName(const vespalib::stringref & name) { _baseFileName = name; } // Implements IAttributeVector - virtual const vespalib::string & getName() const override; + const vespalib::string & getName() const override final { return _baseFileName.getAttributeName(); } bool hasArrayType() const { return _config.collectionType().isArray(); } - bool hasEnum() const override; - bool hasSortedEnum() const { return _hasSortedEnum; } + bool hasEnum() const override final; virtual bool hasEnum2Value() const; uint32_t getMaxValueCount() const override; uint32_t getEnumMax() const { return _enumMax; } // Implements IAttributeVector - uint32_t getNumDocs() const override; + uint32_t getNumDocs() const override final { return _status.getNumDocs(); } uint32_t & getCommittedDocIdLimitRef() { return _committedDocIdLimit; } void setCommittedDocIdLimit(uint32_t committedDocIdLimit) { _committedDocIdLimit = committedDocIdLimit; @@ -427,13 +425,12 @@ public: AddressSpaceUsage getAddressSpaceUsage() const; - // Implements IAttributeVector - virtual BasicType::Type getBasicType() const override; - virtual CollectionType::Type getCollectionType() const override; - virtual bool getIsFilter() const override; - virtual bool getIsFastSearch() const override; - virtual uint32_t getCommittedDocIdLimit() const override; - virtual bool isImported() const override; + BasicType::Type getBasicType() const override final { return getInternalBasicType().type(); } + CollectionType::Type getCollectionType() const override final { return getInternalCollectionType().type(); } + bool getIsFilter() const override final { return _config.getIsFilter(); } + bool getIsFastSearch() const override final { return _config.fastSearch(); } + uint32_t getCommittedDocIdLimit() const override final { return _committedDocIdLimit; } + bool isImported() const override; /** * Updates the base file name of this attribute vector and saves @@ -604,7 +601,6 @@ private: uint64_t _createSerialNum; uint64_t _compactLidSpaceGeneration; bool _hasEnum; - bool _hasSortedEnum; bool _loaded; bool _enableEnumeratedSave; fastos::TimeStamp _nextStatUpdateTime; @@ -634,8 +630,8 @@ private: friend class AttributeManagerTest; public: bool headerTypeOK(const vespalib::GenericHeader &header) const; - bool hasMultiValue() const override; - bool hasWeightedSetType() const override; + bool hasMultiValue() const override final { return _config.collectionType().isMultiValue(); } + bool hasWeightedSetType() const override final { return _config.collectionType().isWeightedSet(); } /** * Should be called by the writer thread. */ diff --git a/searchlib/src/vespa/searchlib/attribute/attrvector.hpp b/searchlib/src/vespa/searchlib/attribute/attrvector.hpp index abcdc0244c2..8f67d237a60 100644 --- a/searchlib/src/vespa/searchlib/attribute/attrvector.hpp +++ b/searchlib/src/vespa/searchlib/attribute/attrvector.hpp @@ -170,7 +170,6 @@ StringDirectAttrVector(const vespalib::string & baseFileName, const Config & c) _idx.push_back(0); } setEnum(); - setSortedEnum(true); } template <typename F> @@ -182,6 +181,5 @@ StringDirectAttrVector(const vespalib::string & baseFileName) : _idx.push_back(0); } setEnum(); - setSortedEnum(true); } diff --git a/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp b/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp index d4fa47eac49..ee984688594 100644 --- a/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp +++ b/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp @@ -12,7 +12,6 @@ SingleStringExtAttribute::SingleStringExtAttribute(const vespalib::string & name StringDirectAttrVector< AttrVector::Features<false> >(name, Config(BasicType::STRING, CollectionType::SINGLE)) { setEnum(false); - setSortedEnum(false); } bool SingleStringExtAttribute::addDoc(DocId & docId) @@ -45,7 +44,6 @@ MultiStringExtAttribute::MultiStringExtAttribute(const vespalib::string & name, (name, Config(BasicType::STRING, ctype)) { setEnum(false); - setSortedEnum(false); } MultiStringExtAttribute::MultiStringExtAttribute(const vespalib::string & name) : @@ -53,7 +51,6 @@ MultiStringExtAttribute::MultiStringExtAttribute(const vespalib::string & name) (name, Config(BasicType::STRING, CollectionType::ARRAY)) { setEnum(false); - setSortedEnum(false); } bool MultiStringExtAttribute::addDoc(DocId & docId) @@ -138,7 +135,6 @@ WeightedSetStringExtAttribute::WeightedSetStringExtAttribute(const vespalib::str WeightedSetExtAttributeBase<MultiStringExtAttribute>(name) { setEnum(false); - setSortedEnum(false); } WeightedSetStringExtAttribute::~WeightedSetStringExtAttribute() {} diff --git a/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp b/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp index 4f4491327e7..834f5913af9 100644 --- a/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp @@ -14,8 +14,7 @@ #include <vespa/log/log.h> LOG_SETUP(".features.distancetopathfeature"); -namespace search { -namespace features { +namespace search::features { const feature_t DistanceToPathExecutor::DEFAULT_DISTANCE(6400000000.0); @@ -96,9 +95,7 @@ DistanceToPathBlueprint::DistanceToPathBlueprint() : { } -DistanceToPathBlueprint::~DistanceToPathBlueprint() -{ -} +DistanceToPathBlueprint::~DistanceToPathBlueprint() = default; void DistanceToPathBlueprint::visitDumpFeatures(const search::fef::IIndexEnvironment &, @@ -175,5 +172,4 @@ DistanceToPathBlueprint::createExecutor(const search::fef::IQueryEnvironment &en return stash.create<DistanceToPathExecutor>(path, pos); } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/distancetopathfeature.h b/searchlib/src/vespa/searchlib/features/distancetopathfeature.h index cf268c59a2e..442f1ba3b28 100644 --- a/searchlib/src/vespa/searchlib/features/distancetopathfeature.h +++ b/searchlib/src/vespa/searchlib/features/distancetopathfeature.h @@ -6,8 +6,7 @@ #include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/searchlib/common/feature.h> -namespace search { -namespace features { +namespace search::features { /** * Define the point type that makes up the end-points in our path. @@ -52,7 +51,7 @@ private: public: DistanceToPathBlueprint(); - ~DistanceToPathBlueprint(); + ~DistanceToPathBlueprint() override; void visitDumpFeatures(const fef::IIndexEnvironment &env, fef::IDumpFeatureVisitor &visitor) const override; fef::Blueprint::UP createInstance() const override; fef::ParameterDescriptions getDescriptions() const override { @@ -64,6 +63,4 @@ public: }; -} // namespace features -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/parsequery/parse.cpp b/searchlib/src/vespa/searchlib/parsequery/parse.cpp index b189331878c..cba69ea474a 100644 --- a/searchlib/src/vespa/searchlib/parsequery/parse.cpp +++ b/searchlib/src/vespa/searchlib/parsequery/parse.cpp @@ -24,9 +24,8 @@ namespace search { ParseItem::ParseItem(ItemType type, int arity) : PARSEITEM_DEFAULT_CONSTRUCTOR_LIST { - assert(type==ITEM_OR || type==ITEM_WEAK_AND || type==ITEM_EQUIV || - type==ITEM_AND || type==ITEM_NOT || type==ITEM_RANK || - type==ITEM_PHRASE || type==ITEM_ANY || type==ITEM_NEAR || type==ITEM_ONEAR); + assert(type==ITEM_OR || type==ITEM_WEAK_AND || type==ITEM_EQUIV || type==ITEM_AND || type==ITEM_NOT + || type==ITEM_RANK || type==ITEM_ANY || type==ITEM_NEAR || type==ITEM_ONEAR); SetType(type); _arity = arity; } @@ -34,7 +33,8 @@ ParseItem::ParseItem(ItemType type, int arity) ParseItem::ParseItem(ItemType type, int arity, const char *idx) : PARSEITEM_DEFAULT_CONSTRUCTOR_LIST { - assert(type == ITEM_PHRASE || type==ITEM_WEIGHTED_SET || type==ITEM_DOT_PRODUCT || type==ITEM_WAND); + assert(type==ITEM_PHRASE || type==ITEM_SAME_ELEMENT || type==ITEM_WEIGHTED_SET + || type==ITEM_DOT_PRODUCT || type==ITEM_WAND); SetType(type); _arity = arity; SetIndex(idx); @@ -108,16 +108,24 @@ ParseItem::AppendBuffer(RawBuf *buf) const case ITEM_ANY: buf->appendCompressedPositiveNumber(_arity); break; - case ITEM_WEAK_AND: case ITEM_NEAR: case ITEM_ONEAR: buf->appendCompressedPositiveNumber(_arity); buf->appendCompressedPositiveNumber(_arg1); - if (Type() == ITEM_WEAK_AND) { - buf->appendCompressedPositiveNumber(indexLen); - if (indexLen != 0) { - buf->append(_indexName.c_str(), indexLen); - } + break; + case ITEM_SAME_ELEMENT: + buf->appendCompressedPositiveNumber(_arity); + buf->appendCompressedPositiveNumber(indexLen); + if (indexLen != 0) { + buf->append(_indexName.c_str(), indexLen); + } + break; + case ITEM_WEAK_AND: + buf->appendCompressedPositiveNumber(_arity); + buf->appendCompressedPositiveNumber(_arg1); + buf->appendCompressedPositiveNumber(indexLen); + if (indexLen != 0) { + buf->append(_indexName.c_str(), indexLen); } break; case ITEM_WEIGHTED_SET: @@ -199,6 +207,9 @@ ParseItem::GetBufferLen() const case ITEM_PHRASE: len += sizeof(uint32_t) * 2 + indexLen; break; + case ITEM_SAME_ELEMENT: + len += sizeof(uint32_t) * 2 + indexLen; + break; case ITEM_WAND: len += sizeof(uint32_t) * 4 + indexLen; break; diff --git a/searchlib/src/vespa/searchlib/parsequery/parse.h b/searchlib/src/vespa/searchlib/parsequery/parse.h index 3d5a1a5e2bb..d8cbdc696a5 100644 --- a/searchlib/src/vespa/searchlib/parsequery/parse.h +++ b/searchlib/src/vespa/searchlib/parsequery/parse.h @@ -52,7 +52,7 @@ public: ITEM_WEIGHTED_SET = 15, ITEM_WEAK_AND = 16, ITEM_EXACTSTRINGTERM = 17, - UNUSED_LEGACY_ITEM_RISE_QUERY = 18, + ITEM_SAME_ELEMENT = 18, ITEM_PURE_WEIGHTED_STRING = 19, ITEM_PURE_WEIGHTED_LONG = 20, ITEM_DOT_PRODUCT = 21, diff --git a/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp b/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp index a7b41476ec6..6f49dc62686 100644 --- a/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp +++ b/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp @@ -25,12 +25,12 @@ SimpleQueryStack::~SimpleQueryStack() } void -SimpleQueryStack::Push(search::ParseItem *item) +SimpleQueryStack::Push(ParseItem *item) { // Check if query OK for FirstPage _FP_queryOK &= - ( item->Type() != search::ParseItem::ITEM_UNDEF - && item->Type() != search::ParseItem::ITEM_PAREN + ( item->Type() != ParseItem::ITEM_UNDEF + && item->Type() != ParseItem::ITEM_PAREN ); @@ -40,10 +40,10 @@ SimpleQueryStack::Push(search::ParseItem *item) _numItems++; } -search::ParseItem * +ParseItem * SimpleQueryStack::Pop() { - search::ParseItem *item = _stack; + ParseItem *item = _stack; if (_stack != NULL) { _numItems--; _stack = _stack->_next; @@ -53,9 +53,9 @@ SimpleQueryStack::Pop() } void -SimpleQueryStack::AppendBuffer(search::RawBuf *buf) const +SimpleQueryStack::AppendBuffer(RawBuf *buf) const { - for (search::ParseItem *item = _stack; item != NULL; item = item->_next) { + for (ParseItem *item = _stack; item != NULL; item = item->_next) { item->AppendBuffer(buf); } } @@ -66,7 +66,7 @@ SimpleQueryStack::GetBufferLen() const size_t result; result = 0; - for (const search::ParseItem *item = _stack; + for (const ParseItem *item = _stack; item != NULL; item = item->_next) { result += item->GetBufferLen(); } @@ -90,34 +90,35 @@ class ItemName { public: ItemName() { memset(_name, 'X', sizeof(_name)); - _name[search::ParseItem::ITEM_OR] = '|'; - _name[search::ParseItem::ITEM_WEAK_AND] = 'w'; - _name[search::ParseItem::ITEM_EQUIV] = 'E'; - _name[search::ParseItem::ITEM_AND] = '&'; - _name[search::ParseItem::ITEM_NOT] = '-'; - _name[search::ParseItem::ITEM_ANY] = '?'; - _name[search::ParseItem::ITEM_RANK] = '%'; - _name[search::ParseItem::ITEM_NEAR] = 'N'; - _name[search::ParseItem::ITEM_ONEAR] = 'O'; - _name[search::ParseItem::ITEM_NUMTERM] = '#'; - _name[search::ParseItem::ITEM_TERM] = 't'; - _name[search::ParseItem::ITEM_PURE_WEIGHTED_STRING] = 'T'; - _name[search::ParseItem::ITEM_PURE_WEIGHTED_LONG] = 'L'; - _name[search::ParseItem::ITEM_PREFIXTERM] = '*'; - _name[search::ParseItem::ITEM_SUBSTRINGTERM] = 's'; - _name[search::ParseItem::ITEM_EXACTSTRINGTERM] = 'e'; - _name[search::ParseItem::ITEM_SUFFIXTERM] = 'S'; - _name[search::ParseItem::ITEM_PHRASE] = '"'; - _name[search::ParseItem::ITEM_WEIGHTED_SET] = 'W'; - _name[search::ParseItem::ITEM_DOT_PRODUCT] = 'D'; - _name[search::ParseItem::ITEM_WAND] = 'A'; - _name[search::ParseItem::ITEM_PREDICATE_QUERY] = 'P'; - _name[search::ParseItem::ITEM_REGEXP] = '^'; + _name[ParseItem::ITEM_OR] = '|'; + _name[ParseItem::ITEM_WEAK_AND] = 'w'; + _name[ParseItem::ITEM_EQUIV] = 'E'; + _name[ParseItem::ITEM_AND] = '&'; + _name[ParseItem::ITEM_NOT] = '-'; + _name[ParseItem::ITEM_ANY] = '?'; + _name[ParseItem::ITEM_RANK] = '%'; + _name[ParseItem::ITEM_NEAR] = 'N'; + _name[ParseItem::ITEM_ONEAR] = 'O'; + _name[ParseItem::ITEM_NUMTERM] = '#'; + _name[ParseItem::ITEM_TERM] = 't'; + _name[ParseItem::ITEM_PURE_WEIGHTED_STRING] = 'T'; + _name[ParseItem::ITEM_PURE_WEIGHTED_LONG] = 'L'; + _name[ParseItem::ITEM_PREFIXTERM] = '*'; + _name[ParseItem::ITEM_SUBSTRINGTERM] = 's'; + _name[ParseItem::ITEM_EXACTSTRINGTERM] = 'e'; + _name[ParseItem::ITEM_SUFFIXTERM] = 'S'; + _name[ParseItem::ITEM_PHRASE] = '"'; + _name[ParseItem::ITEM_SAME_ELEMENT] = 'M'; + _name[ParseItem::ITEM_WEIGHTED_SET] = 'W'; + _name[ParseItem::ITEM_DOT_PRODUCT] = 'D'; + _name[ParseItem::ITEM_WAND] = 'A'; + _name[ParseItem::ITEM_PREDICATE_QUERY] = 'P'; + _name[ParseItem::ITEM_REGEXP] = '^'; } - char operator[] (search::ParseItem::ItemType i) const { return _name[i]; } + char operator[] (ParseItem::ItemType i) const { return _name[i]; } char operator[] (size_t i) const { return _name[i]; } private: - char _name[search::ParseItem::ITEM_MAX]; + char _name[ParseItem::ITEM_MAX]; }; static ItemName _G_ItemName; @@ -162,29 +163,29 @@ SimpleQueryStack::StackbufToString(const vespalib::stringref &theBuf) while (p < ep) { vespalib::string metaStr; rawtype = *p++; - type = search::ParseItem::GetType(rawtype); - if (search::ParseItem::GetFeature_Weight(rawtype)) { + type = ParseItem::GetType(rawtype); + if (ParseItem::GetFeature_Weight(rawtype)) { int64_t tmpLong(0); p += vespalib::compress::Integer::decompress(tmpLong, p); metaStr.append("(w:"); metaStr.append(make_string("%ld", tmpLong)); metaStr.append(")"); } - if (search::ParseItem::getFeature_UniqueId(rawtype)) { + if (ParseItem::getFeature_UniqueId(rawtype)) { p += vespalib::compress::Integer::decompressPositive(tmp, p); metaStr.append("(u:"); metaStr.append(make_string("%ld", tmp)); metaStr.append(")"); } - if (search::ParseItem::getFeature_Flags(rawtype)) { + if (ParseItem::getFeature_Flags(rawtype)) { flags = *p++; metaStr.append("(f:"); metaStr.append(make_string("%d", flags)); metaStr.append(")"); } - if (search::ParseItem::GetCreator(flags) != search::ParseItem::CREA_ORIG) { + if (ParseItem::GetCreator(flags) != ParseItem::CREA_ORIG) { metaStr.append("(c:"); - metaStr.append(make_string("%d", search::ParseItem::GetCreator(flags))); + metaStr.append(make_string("%d", ParseItem::GetCreator(flags))); metaStr.append(")"); } @@ -192,41 +193,52 @@ SimpleQueryStack::StackbufToString(const vespalib::stringref &theBuf) result.append(metaStr); switch (type) { - case search::ParseItem::ITEM_OR: - case search::ParseItem::ITEM_AND: - case search::ParseItem::ITEM_EQUIV: - case search::ParseItem::ITEM_NOT: - case search::ParseItem::ITEM_RANK: - case search::ParseItem::ITEM_ANY: + case ParseItem::ITEM_OR: + case ParseItem::ITEM_AND: + case ParseItem::ITEM_EQUIV: + case ParseItem::ITEM_NOT: + case ParseItem::ITEM_RANK: + case ParseItem::ITEM_ANY: p += vespalib::compress::Integer::decompressPositive(tmp, p); arity = tmp; result.append(make_string("%c/%d~", _G_ItemName[type], arity)); break; - case search::ParseItem::ITEM_WEAK_AND: - case search::ParseItem::ITEM_NEAR: - case search::ParseItem::ITEM_ONEAR: + case ParseItem::ITEM_NEAR: + case ParseItem::ITEM_ONEAR: p += vespalib::compress::Integer::decompressPositive(tmp, p); arity = tmp; p += vespalib::compress::Integer::decompressPositive(tmp, p); arg1 = tmp; - if (type == search::ParseItem::ITEM_WEAK_AND) { - p += vespalib::compress::Integer::decompressPositive(tmp, p); - idxRefLen = tmp; - idxRef = p; - p += idxRefLen; - result.append(make_string("%c/%d/%d/%d:%.*s~", _G_ItemName[type], arity, arg1, idxRefLen, idxRefLen, idxRef)); - } else { - result.append(make_string("%c/%d/%d~", _G_ItemName[type], arity, arg1)); - } + result.append(make_string("%c/%d/%d~", _G_ItemName[type], arity, arg1)); + break; + case ParseItem::ITEM_WEAK_AND: + p += vespalib::compress::Integer::decompressPositive(tmp, p); + arity = tmp; + p += vespalib::compress::Integer::decompressPositive(tmp, p); + arg1 = tmp; + p += vespalib::compress::Integer::decompressPositive(tmp, p); + idxRefLen = tmp; + idxRef = p; + p += idxRefLen; + result.append(make_string("%c/%d/%d/%d:%.*s~", _G_ItemName[type], arity, arg1, idxRefLen, idxRefLen, idxRef)); + break; + case ParseItem::ITEM_SAME_ELEMENT: + p += vespalib::compress::Integer::decompressPositive(tmp, p); + arity = tmp; + p += vespalib::compress::Integer::decompressPositive(tmp, p); + idxRefLen = tmp; + idxRef = p; + p += idxRefLen; + result.append(make_string("%c/%d/%d:%.*s~", _G_ItemName[type], arity, idxRefLen, idxRefLen, idxRef)); break; - case search::ParseItem::ITEM_NUMTERM: - case search::ParseItem::ITEM_TERM: - case search::ParseItem::ITEM_PREFIXTERM: - case search::ParseItem::ITEM_SUBSTRINGTERM: - case search::ParseItem::ITEM_EXACTSTRINGTERM: - case search::ParseItem::ITEM_SUFFIXTERM: - case search::ParseItem::ITEM_REGEXP: + case ParseItem::ITEM_NUMTERM: + case ParseItem::ITEM_TERM: + case ParseItem::ITEM_PREFIXTERM: + case ParseItem::ITEM_SUBSTRINGTERM: + case ParseItem::ITEM_EXACTSTRINGTERM: + case ParseItem::ITEM_SUFFIXTERM: + case ParseItem::ITEM_REGEXP: p += vespalib::compress::Integer::decompressPositive(tmp, p); idxRefLen = tmp; idxRef = p; @@ -239,7 +251,7 @@ SimpleQueryStack::StackbufToString(const vespalib::stringref &theBuf) idxRefLen, idxRefLen, idxRef, termRefLen, termRefLen, termRef)); break; - case search::ParseItem::ITEM_PURE_WEIGHTED_STRING: + case ParseItem::ITEM_PURE_WEIGHTED_STRING: p += vespalib::compress::Integer::decompressPositive(tmp, p); termRefLen = tmp; termRef = p; @@ -248,23 +260,23 @@ SimpleQueryStack::StackbufToString(const vespalib::stringref &theBuf) termRefLen, termRefLen, termRef)); break; - case search::ParseItem::ITEM_PURE_WEIGHTED_LONG: + case ParseItem::ITEM_PURE_WEIGHTED_LONG: tmp = vespalib::nbo::n2h(*reinterpret_cast<const uint64_t *>(p)); p += sizeof(uint64_t); result.append(make_string("%c/%lu", _G_ItemName[type], tmp)); break; - case search::ParseItem::ITEM_PHRASE: - case search::ParseItem::ITEM_WEIGHTED_SET: - case search::ParseItem::ITEM_DOT_PRODUCT: - case search::ParseItem::ITEM_WAND: + case ParseItem::ITEM_PHRASE: + case ParseItem::ITEM_WEIGHTED_SET: + case ParseItem::ITEM_DOT_PRODUCT: + case ParseItem::ITEM_WAND: p += vespalib::compress::Integer::decompressPositive(tmp, p); arity = tmp; p += vespalib::compress::Integer::decompressPositive(tmp, p); idxRefLen = tmp; idxRef = p; p += idxRefLen; - if (type == search::ParseItem::ITEM_WAND) { + if (type == ParseItem::ITEM_WAND) { p += vespalib::compress::Integer::decompressPositive(tmp, p); uint32_t targetNumHits = tmp; double scoreThreshold = vespalib::nbo::n2h(*reinterpret_cast<const double *>(p)); @@ -279,7 +291,7 @@ SimpleQueryStack::StackbufToString(const vespalib::stringref &theBuf) } break; - case search::ParseItem::ITEM_PREDICATE_QUERY: + case ParseItem::ITEM_PREDICATE_QUERY: { idxRefLen = static_cast<uint32_t>(ReadCompressedPositiveInt(p)); idxRef = p; diff --git a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp index ec34b2d3a84..542e7990c1e 100644 --- a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp +++ b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp @@ -34,9 +34,7 @@ SimpleQueryStackDumpIterator::SimpleQueryStackDumpIterator(const vespalib::strin { } -SimpleQueryStackDumpIterator::~SimpleQueryStackDumpIterator() -{ -} +SimpleQueryStackDumpIterator::~SimpleQueryStackDumpIterator() = default; vespalib::string SimpleQueryStackDumpIterator::readString(const char *&p) { if (p >= _bufEnd) throw false; @@ -154,6 +152,20 @@ SimpleQueryStackDumpIterator::next() _currTerm = NULL; _currTermLen = 0; break; + case ParseItem::ITEM_SAME_ELEMENT: + if (p >= _bufEnd) return false; + p += vespalib::compress::Integer::decompressPositive(tmp, p); + _currArity = tmp; + _currArg1 = 0; + p += vespalib::compress::Integer::decompressPositive(tmp, p); + _currIndexNameLen = tmp; + if (p > _bufEnd) return false; + _currIndexName = p; + p += _currIndexNameLen; + if (p > _bufEnd) return false; + _currTerm = NULL; + _currTermLen = 0; + break; case ParseItem::ITEM_PURE_WEIGHTED_STRING: if (p >= _bufEnd) return false; diff --git a/searchlib/src/vespa/searchlib/query/query.cpp b/searchlib/src/vespa/searchlib/query/query.cpp index d380c273d02..984337f40ba 100644 --- a/searchlib/src/vespa/searchlib/query/query.cpp +++ b/searchlib/src/vespa/searchlib/query/query.cpp @@ -102,10 +102,11 @@ QueryConnector::create(ParseItem::ItemType type) case search::ParseItem::ITEM_WAND: return new OrQueryNode(); case search::ParseItem::ITEM_NOT: return new AndNotQueryNode(); case search::ParseItem::ITEM_PHRASE: return new PhraseQueryNode(); + case search::ParseItem::ITEM_SAME_ELEMENT: return new AndQueryNode(); // TODO: This needs a same element operation to work for streaming search too. case search::ParseItem::ITEM_NEAR: return new NearQueryNode(); case search::ParseItem::ITEM_ONEAR: return new ONearQueryNode(); default: - return NULL; + return nullptr; } } diff --git a/searchlib/src/vespa/searchlib/query/querynode.cpp b/searchlib/src/vespa/searchlib/query/querynode.cpp index 2e9d8be95f4..0d0a06de7af 100644 --- a/searchlib/src/vespa/searchlib/query/querynode.cpp +++ b/searchlib/src/vespa/searchlib/query/querynode.cpp @@ -26,6 +26,7 @@ QueryNode::UP QueryNode::Build(const QueryNode * parent, const QueryNodeResultFa case search::ParseItem::ITEM_WAND: case search::ParseItem::ITEM_NOT: case search::ParseItem::ITEM_PHRASE: + case search::ParseItem::ITEM_SAME_ELEMENT: case search::ParseItem::ITEM_NEAR: case search::ParseItem::ITEM_ONEAR: { diff --git a/searchlib/src/vespa/searchlib/query/tree/customtypetermvisitor.h b/searchlib/src/vespa/searchlib/query/tree/customtypetermvisitor.h index 72763839b95..611406e6ea3 100644 --- a/searchlib/src/vespa/searchlib/query/tree/customtypetermvisitor.h +++ b/searchlib/src/vespa/searchlib/query/tree/customtypetermvisitor.h @@ -5,8 +5,7 @@ #include "customtypevisitor.h" #include "intermediate.h" -namespace search { -namespace query { +namespace search::query { template <class NodeTypes> class CustomTypeTermVisitor : public CustomTypeVisitor<NodeTypes> @@ -27,10 +26,10 @@ private: void visit(typename NodeTypes::Or &n) override { visitChildren(n); } void visit(typename NodeTypes::Rank &n) override { visitChildren(n); } void visit(typename NodeTypes::WeakAnd &n) override { visitChildren(n); } + void visit(typename NodeTypes::SameElement &n) override { visitChildren(n); } // phrases and weighted set terms are conceptual leaf nodes and // should be handled that way. }; -} // namespace query -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/query/tree/customtypevisitor.h b/searchlib/src/vespa/searchlib/query/tree/customtypevisitor.h index 83e6343e223..cdeebcaf9e5 100644 --- a/searchlib/src/vespa/searchlib/query/tree/customtypevisitor.h +++ b/searchlib/src/vespa/searchlib/query/tree/customtypevisitor.h @@ -4,8 +4,7 @@ #include "queryvisitor.h" -namespace search { -namespace query { +namespace search::query { /** * By typedefing a (complete) set of subclasses to the query nodes in @@ -37,6 +36,7 @@ public: virtual void visit(typename NodeTypes::ONear &) = 0; virtual void visit(typename NodeTypes::Or &) = 0; virtual void visit(typename NodeTypes::Phrase &) = 0; + virtual void visit(typename NodeTypes::SameElement &) = 0; virtual void visit(typename NodeTypes::PrefixTerm &) = 0; virtual void visit(typename NodeTypes::RangeTerm &) = 0; virtual void visit(typename NodeTypes::Rank &) = 0; @@ -62,6 +62,7 @@ private: typedef typename NodeTypes::ONear TONear; typedef typename NodeTypes::Or TOr; typedef typename NodeTypes::Phrase TPhrase; + typedef typename NodeTypes::SameElement TSameElement; typedef typename NodeTypes::PrefixTerm TPrefixTerm; typedef typename NodeTypes::RangeTerm TRangeTerm; typedef typename NodeTypes::Rank TRank; @@ -84,6 +85,7 @@ private: void visit(ONear &n) override { visit(static_cast<TONear&>(n)); } void visit(Or &n) override { visit(static_cast<TOr&>(n)); } void visit(Phrase &n) override { visit(static_cast<TPhrase&>(n)); } + void visit(SameElement &n) override { visit(static_cast<TSameElement &>(n)); } void visit(PrefixTerm &n) override { visit(static_cast<TPrefixTerm&>(n)); } void visit(RangeTerm &n) override { visit(static_cast<TRangeTerm&>(n)); } void visit(Rank &n) override { visit(static_cast<TRank&>(n)); } @@ -98,5 +100,4 @@ private: void visit(RegExpTerm &n) override { visit(static_cast<TRegExpTerm&>(n)); } }; -} // namespace query -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/query/tree/intermediate.cpp b/searchlib/src/vespa/searchlib/query/tree/intermediate.cpp index 1274777dc74..f56da9c2cf9 100644 --- a/searchlib/src/vespa/searchlib/query/tree/intermediate.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/intermediate.cpp @@ -1,8 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "intermediate.h" -namespace search { -namespace query { +namespace search::query { Intermediate::~Intermediate() { for (size_t i = 0; i < _children.size(); ++i) { @@ -16,5 +15,4 @@ Intermediate &Intermediate::append(Node::UP child) return *this; } -} // namespace query -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/query/tree/intermediate.h b/searchlib/src/vespa/searchlib/query/tree/intermediate.h index c28ab8241e0..052dc1db269 100644 --- a/searchlib/src/vespa/searchlib/query/tree/intermediate.h +++ b/searchlib/src/vespa/searchlib/query/tree/intermediate.h @@ -1,11 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "node.h" #include <vector> -#include <vespa/searchlib/query/tree/node.h> -namespace search { -namespace query { +namespace search::query { class Intermediate : public Node { @@ -24,6 +23,4 @@ class Intermediate : public Node Intermediate &append(Node::UP child); }; -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.cpp b/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.cpp index 9fabbe58a19..4a4b606ef8f 100644 --- a/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.cpp @@ -1,32 +1,20 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "intermediatenodes.h" -namespace search { -namespace query { - -And::~And() {} - -AndNot::~AndNot() {} - -Or::~Or() {} - -WeakAnd::~WeakAnd() {} - -Equiv::~Equiv() {} - -Rank::~Rank() {} - -Near::~Near() {} - -ONear::~ONear() {} - -Phrase::~Phrase() {} - -WeightedSetTerm::~WeightedSetTerm() {} - -DotProduct::~DotProduct() {} - -WandTerm::~WandTerm() {} - -} // namespace query -} // namespace search +namespace search::query { + +And::~And() = default; +AndNot::~AndNot() = default; +Or::~Or() = default; +WeakAnd::~WeakAnd() = default; +Equiv::~Equiv() = default; +Rank::~Rank() = default; +Near::~Near() = default; +ONear::~ONear() = default; +Phrase::~Phrase() = default; +SameElement::~SameElement() = default; +WeightedSetTerm::~WeightedSetTerm() = default; +DotProduct::~DotProduct() = default; +WandTerm::~WandTerm() = default; + +} diff --git a/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.h b/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.h index e358c7d7bef..6d643d951f0 100644 --- a/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.h +++ b/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.h @@ -5,10 +5,8 @@ #include "intermediate.h" #include "querynodemixin.h" #include "term.h" -#include <vespa/searchlib/query/weight.h> -namespace search { -namespace query { +namespace search::query { class And : public QueryNodeMixin<And, Intermediate> { public: @@ -105,6 +103,15 @@ public: virtual ~Phrase() = 0; }; +class SameElement : public QueryNodeMixin<SameElement, Intermediate> { +public: + SameElement(const vespalib::string &view) : _view(view) {} + virtual ~SameElement() = 0; + const vespalib::string & getView() const { return _view; } +private: + vespalib::string _view; +}; + class WeightedSetTerm : public QueryNodeMixin<WeightedSetTerm, Intermediate>, public Term { public: WeightedSetTerm(const vespalib::string &view, int32_t id, Weight weight) @@ -137,6 +144,4 @@ public: double getThresholdBoostFactor() const { return _thresholdBoostFactor; } }; -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/tree/location.cpp b/searchlib/src/vespa/searchlib/query/tree/location.cpp index 5817b93a2fb..216c5ec5ad0 100644 --- a/searchlib/src/vespa/searchlib/query/tree/location.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/location.cpp @@ -7,8 +7,7 @@ using vespalib::asciistream; -namespace search { -namespace query { +namespace search::query { Location::Location(const Point &point, uint32_t max_dist, uint32_t x_aspect) { asciistream loc; @@ -61,5 +60,4 @@ vespalib::asciistream &operator<<(vespalib::asciistream &out, const Location &lo return out << loc.getLocationString(); } -} // namespace query -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/query/tree/location.h b/searchlib/src/vespa/searchlib/query/tree/location.h index 0abad19b696..5ed717c87d6 100644 --- a/searchlib/src/vespa/searchlib/query/tree/location.h +++ b/searchlib/src/vespa/searchlib/query/tree/location.h @@ -4,11 +4,8 @@ #include <vespa/vespalib/stllike/string.h> -namespace vespalib { - class asciistream; -} -namespace search { -namespace query { +namespace vespalib { class asciistream; } +namespace search::query { class Point; class Rectangle; @@ -32,6 +29,4 @@ public: vespalib::asciistream &operator<<(vespalib::asciistream &out, const Location &loc); -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/tree/point.h b/searchlib/src/vespa/searchlib/query/tree/point.h index af9b958a474..89d0bc1db44 100644 --- a/searchlib/src/vespa/searchlib/query/tree/point.h +++ b/searchlib/src/vespa/searchlib/query/tree/point.h @@ -2,10 +2,9 @@ #pragma once -#include <stdint.h> +#include <cstdint> -namespace search { -namespace query { +namespace search::query { struct Point { int64_t x; @@ -18,6 +17,4 @@ inline bool operator==(const Point &p1, const Point &p2) { return p1.x == p2.x && p1.y == p2.y; } -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/tree/predicate_query_term.h b/searchlib/src/vespa/searchlib/query/tree/predicate_query_term.h index b851563e198..0a92546e414 100644 --- a/searchlib/src/vespa/searchlib/query/tree/predicate_query_term.h +++ b/searchlib/src/vespa/searchlib/query/tree/predicate_query_term.h @@ -6,8 +6,7 @@ #include <memory> #include <vector> -namespace search { -namespace query { +namespace search::query { /** * Represents a predicate query, with features and range features. @@ -71,6 +70,4 @@ public: } }; -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/tree/querybuilder.h b/searchlib/src/vespa/searchlib/query/tree/querybuilder.h index 2732d7895a5..3c6ff93457d 100644 --- a/searchlib/src/vespa/searchlib/query/tree/querybuilder.h +++ b/searchlib/src/vespa/searchlib/query/tree/querybuilder.h @@ -20,13 +20,11 @@ #pragma once #include "predicate_query_term.h" -#include <stack> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/searchlib/query/weight.h> #include "node.h" +#include <vespa/searchlib/query/weight.h> +#include <stack> -namespace search { -namespace query { +namespace search::query { class Intermediate; class Location; @@ -124,6 +122,10 @@ typename NodeTypes::Phrase *createPhrase(const vespalib::stringref &view, int32_ return new typename NodeTypes::Phrase(view, id, weight); } template <class NodeTypes> +typename NodeTypes::SameElement *createSameElement(const vespalib::stringref &view) { + return new typename NodeTypes::SameElement(view); +} +template <class NodeTypes> typename NodeTypes::WeightedSetTerm *createWeightedSetTerm(const vespalib::stringref &view, int32_t id, Weight weight) { return new typename NodeTypes::WeightedSetTerm(view, id, weight); } @@ -243,6 +245,9 @@ public: setWeightOverride(weight); return node; } + typename NodeTypes::SameElement &addSameElement(int child_count, const stringref &view) { + return addIntermediate(createSameElement<NodeTypes>(view), child_count); + } typename NodeTypes::WeightedSetTerm &addWeightedSetTerm( int child_count, const stringref &view, int32_t id, Weight weight) { adjustWeight(weight); typename NodeTypes::WeightedSetTerm &node = addIntermediate(createWeightedSetTerm<NodeTypes>(view, id, weight), child_count); @@ -306,6 +311,4 @@ public: } }; -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/tree/querynodemixin.h b/searchlib/src/vespa/searchlib/query/tree/querynodemixin.h index d6e0950dea2..9e8c97cff94 100644 --- a/searchlib/src/vespa/searchlib/query/tree/querynodemixin.h +++ b/searchlib/src/vespa/searchlib/query/tree/querynodemixin.h @@ -4,8 +4,7 @@ #include "queryvisitor.h" -namespace search { -namespace query { +namespace search::query { template <typename T, typename Base> struct QueryNodeMixin : Base { @@ -21,8 +20,6 @@ protected: }; template <typename T, typename Base> -QueryNodeMixin<T, Base>::~QueryNodeMixin() {} - -} // namespace query -} // namespace search +QueryNodeMixin<T, Base>::~QueryNodeMixin() = default; +} diff --git a/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h b/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h index aad85d9b28c..7bf6c17f136 100644 --- a/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h +++ b/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h @@ -7,8 +7,7 @@ #include "queryvisitor.h" #include "termnodes.h" -namespace search { -namespace query { +namespace search::query { /** * Creates a new query tree based on an existing one. The traits class @@ -70,22 +69,24 @@ private: } void visit(Phrase &node) override { - replicate(node, _builder.addPhrase(node.getChildren().size(), - node.getView(), + replicate(node, _builder.addPhrase(node.getChildren().size(), node.getView(), node.getId(), node.getWeight())); visitNodes(node.getChildren()); } + void visit(SameElement &node) override { + _builder.addSameElement(node.getChildren().size(), node.getView()); + visitNodes(node.getChildren()); + } + void visit(WeightedSetTerm &node) override { - replicate(node, _builder.addWeightedSetTerm(node.getChildren().size(), - node.getView(), + replicate(node, _builder.addWeightedSetTerm(node.getChildren().size(), node.getView(), node.getId(), node.getWeight())); visitNodes(node.getChildren()); } void visit(DotProduct &node) override { - replicate(node, _builder.addDotProduct(node.getChildren().size(), - node.getView(), + replicate(node, _builder.addDotProduct(node.getChildren().size(), node.getView(), node.getId(), node.getWeight())); visitNodes(node.getChildren()); } @@ -165,5 +166,4 @@ private: } }; -} // namespace query -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/query/tree/querytreecreator.h b/searchlib/src/vespa/searchlib/query/tree/querytreecreator.h index a3e6ac523ae..c42a16d8ab3 100644 --- a/searchlib/src/vespa/searchlib/query/tree/querytreecreator.h +++ b/searchlib/src/vespa/searchlib/query/tree/querytreecreator.h @@ -5,8 +5,7 @@ #include "queryreplicator.h" #include "stackdumpquerycreator.h" -namespace search { -namespace query { +namespace search::query { /** * Holds functions for creating query trees, either from a stack dump @@ -27,6 +26,4 @@ private: QueryTreeCreator(); }; -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/tree/queryvisitor.h b/searchlib/src/vespa/searchlib/query/tree/queryvisitor.h index 4596359201b..0cb56f9127a 100644 --- a/searchlib/src/vespa/searchlib/query/tree/queryvisitor.h +++ b/searchlib/src/vespa/searchlib/query/tree/queryvisitor.h @@ -2,8 +2,7 @@ #pragma once -namespace search { -namespace query { +namespace search::query { class And; class AndNot; @@ -26,6 +25,7 @@ class DotProduct; class WandTerm; class PredicateQuery; class RegExpTerm; +class SameElement; struct QueryVisitor { virtual ~QueryVisitor() {} @@ -39,6 +39,7 @@ struct QueryVisitor { virtual void visit(ONear &) = 0; virtual void visit(Or &) = 0; virtual void visit(Phrase &) = 0; + virtual void visit(SameElement &node) = 0; virtual void visit(PrefixTerm &) = 0; virtual void visit(RangeTerm &) = 0; virtual void visit(Rank &) = 0; @@ -53,6 +54,5 @@ struct QueryVisitor { virtual void visit(RegExpTerm &) = 0; }; -} // namespace query -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/query/tree/range.cpp b/searchlib/src/vespa/searchlib/query/tree/range.cpp index 583d57bb74d..202b80d98a2 100644 --- a/searchlib/src/vespa/searchlib/query/tree/range.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/range.cpp @@ -3,8 +3,7 @@ #include "range.h" #include <vespa/vespalib/stllike/asciistream.h> -namespace search { -namespace query { +namespace search::query { Range::Range(int64_t f, int64_t t) { @@ -18,5 +17,4 @@ vespalib::asciistream &operator<<(vespalib::asciistream &out, const Range &range return out << range.getRangeString(); } -} // namespace query -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/query/tree/range.h b/searchlib/src/vespa/searchlib/query/tree/range.h index 862258873e6..e55ddf7f14b 100644 --- a/searchlib/src/vespa/searchlib/query/tree/range.h +++ b/searchlib/src/vespa/searchlib/query/tree/range.h @@ -3,12 +3,9 @@ #pragma once #include <vespa/vespalib/stllike/string.h> -namespace vespalib { - class asciistream; -} +namespace vespalib { class asciistream; } -namespace search { -namespace query { +namespace search::query { class Range { vespalib::string _range; @@ -27,6 +24,4 @@ inline bool operator==(const Range &r1, const Range &r2) { vespalib::asciistream &operator<<(vespalib::asciistream &out, const Range &range); -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/tree/rectangle.h b/searchlib/src/vespa/searchlib/query/tree/rectangle.h index 29b144ac5dc..97be9ddeb32 100644 --- a/searchlib/src/vespa/searchlib/query/tree/rectangle.h +++ b/searchlib/src/vespa/searchlib/query/tree/rectangle.h @@ -2,8 +2,7 @@ #pragma once -namespace search { -namespace query { +namespace search::query { struct Rectangle { int64_t left; @@ -21,6 +20,5 @@ inline bool operator==(const Rectangle &r1, const Rectangle &r2) { && r1.top == r2.top && r1.bottom == r2.bottom; } -} // namespace query -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/query/tree/simplequery.h b/searchlib/src/vespa/searchlib/query/tree/simplequery.h index 28791dafe53..557d0964bcb 100644 --- a/searchlib/src/vespa/searchlib/query/tree/simplequery.h +++ b/searchlib/src/vespa/searchlib/query/tree/simplequery.h @@ -10,8 +10,7 @@ #include "intermediatenodes.h" #include "termnodes.h" -namespace search { -namespace query { +namespace search::query { struct SimpleAnd : And {}; struct SimpleAndNot : AndNot {}; @@ -31,6 +30,10 @@ struct SimplePhrase : Phrase { SimplePhrase(const vespalib::stringref &view, int32_t id, Weight weight) : Phrase(view, id, weight) {} }; + +struct SimpleSameElement : SameElement { + SimpleSameElement(const vespalib::stringref &view) : SameElement(view) {} +}; struct SimpleWeightedSetTerm : WeightedSetTerm { SimpleWeightedSetTerm(const vespalib::stringref &view, int32_t id, Weight weight) : WeightedSetTerm(view, id, weight) {} @@ -112,6 +115,7 @@ struct SimpleQueryNodeTypes { typedef SimpleONear ONear; typedef SimpleOr Or; typedef SimplePhrase Phrase; + typedef SimpleSameElement SameElement; typedef SimplePrefixTerm PrefixTerm; typedef SimpleRangeTerm RangeTerm; typedef SimpleRank Rank; @@ -126,6 +130,4 @@ struct SimpleQueryNodeTypes { typedef SimpleRegExpTerm RegExpTerm; }; -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp index 771830c0b03..645750b8576 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp @@ -64,23 +64,48 @@ class QueryNodeConverter : public QueryVisitor { template <typename V> void appendPredicateQueryTermVector(const V& v); + void createComplexIntermediate(const Term &node, const std::vector<Node *> & children, size_t type) { + uint8_t flags = 0; + if (!node.isRanked()) { + flags |= ParseItem::IFLAG_NORANK; + } + if (!node.usePositionData()) { + flags |= ParseItem::IFLAG_NOPOSITIONDATA; + } + if (flags != 0) { + type |= ParseItem::IF_FLAGS; + } + appendByte(type); + appendCompressedNumber(node.getWeight().percent()); + if (type & ParseItem::IF_FLAGS) { + appendByte(flags); + } + appendCompressedPositiveNumber(children.size()); + appendString(node.getView()); + visitNodes(children); + } + void createIntermediate(const Intermediate &node, size_t type) { appendByte(type); appendCompressedPositiveNumber(node.getChildren().size()); visitNodes(node.getChildren()); } - void createIntermediate(const Intermediate &node, size_t type, - size_t distance) { + void createIntermediate(const Intermediate &node, size_t type, size_t distance) { appendByte(type); appendCompressedPositiveNumber(node.getChildren().size()); appendCompressedPositiveNumber(distance); visitNodes(node.getChildren()); } - void createIntermediate(const Intermediate &node, size_t type, - size_t distance, - const vespalib::string & view) { + void createIntermediate(const Intermediate &node, size_t type, const vespalib::string & view) { + appendByte(type); + appendCompressedPositiveNumber(node.getChildren().size()); + appendString(view); + visitNodes(node.getChildren()); + } + + void createIntermediate(const Intermediate &node, size_t type, size_t distance, const vespalib::string & view) { appendByte(type); appendCompressedPositiveNumber(node.getChildren().size()); appendCompressedPositiveNumber(distance); @@ -116,26 +141,12 @@ class QueryNodeConverter : public QueryVisitor { createIntermediate(node, ParseItem::ITEM_EQUIV); } + void visit(SameElement &node) override { + createIntermediate(node, ParseItem::ITEM_SAME_ELEMENT, node.getView()); + } + void visit(Phrase &node) override { - uint8_t typefield = (ParseItem::ITEM_PHRASE | ParseItem::IF_WEIGHT); - uint8_t flags = 0; - if (!node.isRanked()) { - flags |= ParseItem::IFLAG_NORANK; - } - if (!node.usePositionData()) { - flags |= ParseItem::IFLAG_NOPOSITIONDATA; - } - if (flags != 0) { - typefield |= ParseItem::IF_FLAGS; - } - appendByte(typefield); - appendCompressedNumber(node.getWeight().percent()); - if (typefield & ParseItem::IF_FLAGS) { - appendByte(flags); - } - appendCompressedPositiveNumber(node.getChildren().size()); - appendString(node.getView()); - visitNodes(node.getChildren()); + createComplexIntermediate(node, node.getChildren(), (ParseItem::ITEM_PHRASE | ParseItem::IF_WEIGHT)); } template <typename NODE> @@ -187,9 +198,7 @@ class QueryNodeConverter : public QueryVisitor { template <class Term> void createTerm(const Term &node, size_t type) { - uint8_t typefield = type | - ParseItem::IF_WEIGHT | - ParseItem::IF_UNIQUEID; + uint8_t typefield = type | ParseItem::IF_WEIGHT | ParseItem::IF_UNIQUEID; uint8_t flags = 0; if (!node.isRanked()) { flags |= ParseItem::IFLAG_NORANK; diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.h b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.h index 448f1c9bd08..4e1556d05e6 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.h +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.h @@ -4,8 +4,7 @@ #include <vespa/vespalib/stllike/string.h> -namespace search { -namespace query { +namespace search::query { class Node; @@ -14,6 +13,4 @@ struct StackDumpCreator { static vespalib::string create(const Node &node); }; -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h index f9c66965cc4..fa42cdac1c0 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h @@ -9,8 +9,7 @@ #include <vespa/searchlib/parsequery/simplequerystack.h> #include <vespa/vespalib/objects/hexdump.h> -namespace search { -namespace query { +namespace search::query { /** * Creates a query tree from a stack dump. @@ -90,6 +89,10 @@ private: Weight weight = queryStack.GetWeight(); t = &builder.addPhrase(arity, view, id, weight); pureTermView = view; + } else if (type == ParseItem::ITEM_SAME_ELEMENT) { + vespalib::stringref view = queryStack.getIndexName(); + builder.addSameElement(arity, view); + pureTermView = view; } else if (type == ParseItem::ITEM_WEIGHTED_SET) { vespalib::stringref view = queryStack.getIndexName(); int32_t id = queryStack.getUniqueId(); @@ -152,6 +155,4 @@ private: } }; -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/tree/templatetermvisitor.h b/searchlib/src/vespa/searchlib/query/tree/templatetermvisitor.h index e4ce4ccf807..0cdaca82572 100644 --- a/searchlib/src/vespa/searchlib/query/tree/templatetermvisitor.h +++ b/searchlib/src/vespa/searchlib/query/tree/templatetermvisitor.h @@ -4,8 +4,7 @@ #include "customtypetermvisitor.h" -namespace search { -namespace query { +namespace search::query { /** * Use this class to visit all term nodes by deriving from this class @@ -54,5 +53,4 @@ class TemplateTermVisitor : public CustomTypeTermVisitor<NodeTypes> { void visit(typename NodeTypes::WandTerm &n) override { myVisit(n); } }; -} // namespace query -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/query/tree/term.cpp b/searchlib/src/vespa/searchlib/query/tree/term.cpp index 54def08afe8..de59752aa10 100644 --- a/searchlib/src/vespa/searchlib/query/tree/term.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/term.cpp @@ -1,11 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "term.h" +#include <cassert> -namespace search { -namespace query { +namespace search::query { -Term::~Term() { } +Term::~Term() = default; Term::Term(const vespalib::stringref &view, int32_t id, Weight weight) : _view(view), @@ -16,5 +16,14 @@ Term::Term(const vespalib::stringref &view, int32_t id, Weight weight) : _position_data(true) { } -} // namespace query -} // namespace search +void Term::setStateFrom(const Term& other) { + setTermIndex(other.getTermIndex()); + setRanked(other.isRanked()); + setPositionData(other.usePositionData()); + // too late to copy this state: + assert(_view == other.getView()); + assert(_id == other.getId()); + assert(_weight == other.getWeight()); +} + +} diff --git a/searchlib/src/vespa/searchlib/query/tree/term.h b/searchlib/src/vespa/searchlib/query/tree/term.h index f8c0d98ac22..8e9b9897e9d 100644 --- a/searchlib/src/vespa/searchlib/query/tree/term.h +++ b/searchlib/src/vespa/searchlib/query/tree/term.h @@ -1,13 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/stllike/string.h> -#include <vespa/searchlib/query/tree/node.h> +#include "node.h" #include <vespa/searchlib/query/weight.h> -#include <cassert> +#include <vespa/vespalib/stllike/string.h> -namespace search { -namespace query { +namespace search::query { /** * This is a leaf in the Query tree. Sort of. Phrases are both terms @@ -16,11 +14,11 @@ namespace query { class Term { vespalib::string _view; - int32_t _id; - Weight _weight; - int32_t _term_index; - bool _ranked; - bool _position_data; + int32_t _id; + Weight _weight; + int32_t _term_index; + bool _ranked; + bool _position_data; public: virtual ~Term() = 0; @@ -29,15 +27,7 @@ public: void setRanked(bool ranked) { _ranked = ranked; } void setPositionData(bool position_data) { _position_data = position_data; } - void setStateFrom(const Term& other) { - setTermIndex(other.getTermIndex()); - setRanked(other.isRanked()); - setPositionData(other.usePositionData()); - // too late to copy this state: - assert(_view == other.getView()); - assert(_id == other.getId()); - assert(_weight == other.getWeight()); - } + void setStateFrom(const Term& other); const vespalib::string & getView() const { return _view; } Weight getWeight() const { return _weight; } @@ -60,7 +50,7 @@ class TermBase : public Node, public Term { public: typedef T Type; - virtual ~TermBase() = 0; + ~TermBase() override = 0; const T &getTerm() const { return _term; } protected: @@ -71,8 +61,6 @@ protected: }; template <typename T> -TermBase<T>::~TermBase() {} - -} // namespace query -} // namespace search +TermBase<T>::~TermBase() = default; +} diff --git a/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp b/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp index 8e8ae16827b..0a6a6af62b5 100644 --- a/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp @@ -2,27 +2,25 @@ #include "termnodes.h" -namespace search { -namespace query { +namespace search::query { -NumberTerm::~NumberTerm() {} +NumberTerm::~NumberTerm() = default; -PrefixTerm::~PrefixTerm() {} +PrefixTerm::~PrefixTerm() = default; -RangeTerm::~RangeTerm() {} +RangeTerm::~RangeTerm() = default; StringTerm::StringTerm(const Type &term, const vespalib::stringref &view, int32_t id, Weight weight) : QueryNodeMixinType(term, view, id, weight) {} -StringTerm::~StringTerm() {} +StringTerm::~StringTerm() = default; -SubstringTerm::~SubstringTerm() {} +SubstringTerm::~SubstringTerm() = default; -SuffixTerm::~SuffixTerm() {} +SuffixTerm::~SuffixTerm() = default; -LocationTerm::~LocationTerm() {} +LocationTerm::~LocationTerm() = default; -RegExpTerm::~RegExpTerm() {} +RegExpTerm::~RegExpTerm() = default; -} // namespace query -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/query/tree/termnodes.h b/searchlib/src/vespa/searchlib/query/tree/termnodes.h index 4c98ba92ff5..8d4882fb393 100644 --- a/searchlib/src/vespa/searchlib/query/tree/termnodes.h +++ b/searchlib/src/vespa/searchlib/query/tree/termnodes.h @@ -8,8 +8,7 @@ #include "range.h" #include "term.h" -namespace search { -namespace query { +namespace search::query { typedef TermBase<vespalib::string> StringBase; @@ -117,6 +116,4 @@ public: }; -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/weight.h b/searchlib/src/vespa/searchlib/query/weight.h index 1b4168231c5..51596642cd2 100644 --- a/searchlib/src/vespa/searchlib/query/weight.h +++ b/searchlib/src/vespa/searchlib/query/weight.h @@ -3,8 +3,7 @@ #include <cstdint> -namespace search { -namespace query { +namespace search::query { /** * Represents the weight given on a query item such as a term, phrase, or equiv. @@ -44,8 +43,7 @@ public: bool operator== (const Weight& other) const { return _weight == other._weight; } }; -} // namespace query -} // namespace search +} inline search::query::Weight operator+(const search::query::Weight& a, const search::query::Weight& b) { diff --git a/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt b/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt index 683de780107..ecda6d6d6ef 100644 --- a/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt @@ -33,6 +33,8 @@ vespa_add_library(searchlib_queryeval OBJECT predicate_blueprint.cpp predicate_search.cpp ranksearch.cpp + same_element_blueprint.cpp + same_element_search.cpp searchable.cpp searchiterator.cpp simple_phrase_blueprint.cpp diff --git a/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp b/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp index 8e6429aaa90..e3dac98e588 100644 --- a/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp @@ -19,7 +19,7 @@ CreateBlueprintVisitorHelper::CreateBlueprintVisitorHelper(Searchable &searchabl _result() {} -CreateBlueprintVisitorHelper::~CreateBlueprintVisitorHelper() {} +CreateBlueprintVisitorHelper::~CreateBlueprintVisitorHelper() = default; Blueprint::UP CreateBlueprintVisitorHelper::getResult() @@ -30,7 +30,7 @@ CreateBlueprintVisitorHelper::getResult() } void -CreateBlueprintVisitorHelper::visitPhrase(search::query::Phrase &n) { +CreateBlueprintVisitorHelper::visitPhrase(query::Phrase &n) { SimplePhraseBlueprint *phrase = new SimplePhraseBlueprint(_field, _requestContext); Blueprint::UP result(phrase); for (size_t i = 0; i < n.getChildren().size(); ++i) { @@ -42,7 +42,7 @@ CreateBlueprintVisitorHelper::visitPhrase(search::query::Phrase &n) { } void -CreateBlueprintVisitorHelper::handleNumberTermAsText(search::query::NumberTerm &n) +CreateBlueprintVisitorHelper::handleNumberTermAsText(query::NumberTerm &n) { vespalib::string termStr = termAsString(n); queryeval::SplitFloat splitter(termStr); @@ -73,24 +73,24 @@ CreateBlueprintVisitorHelper::createWeightedSet(WS *bp, NODE &n) { for (size_t i = 0; i < n.getChildren().size(); ++i) { fields.clear(); fields.add(bp->getNextChildField(_field)); - const search::query::Node &node = *n.getChildren()[i]; + const query::Node &node = *n.getChildren()[i]; uint32_t weight = getWeightFromNode(node).percent(); bp->addTerm(_searchable.createBlueprint(_requestContext, fields, node), weight); } setResult(std::move(result)); } void -CreateBlueprintVisitorHelper::visitWeightedSetTerm(search::query::WeightedSetTerm &n) { +CreateBlueprintVisitorHelper::visitWeightedSetTerm(query::WeightedSetTerm &n) { WeightedSetTermBlueprint *bp = new WeightedSetTermBlueprint(_field); createWeightedSet(bp, n); } void -CreateBlueprintVisitorHelper::visitDotProduct(search::query::DotProduct &n) { +CreateBlueprintVisitorHelper::visitDotProduct(query::DotProduct &n) { DotProductBlueprint *bp = new DotProductBlueprint(_field); createWeightedSet(bp, n); } void -CreateBlueprintVisitorHelper::visitWandTerm(search::query::WandTerm &n) { +CreateBlueprintVisitorHelper::visitWandTerm(query::WandTerm &n) { ParallelWeakAndBlueprint *bp = new ParallelWeakAndBlueprint(_field, n.getTargetNumHits(), n.getScoreThreshold(), diff --git a/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.h b/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.h index cded9c103dc..5bcc4f4c4c5 100644 --- a/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.h +++ b/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.h @@ -12,7 +12,7 @@ namespace search::queryeval { -class CreateBlueprintVisitorHelper : public search::query::QueryVisitor +class CreateBlueprintVisitorHelper : public query::QueryVisitor { private: const IRequestContext & _requestContext; @@ -37,42 +37,43 @@ public: const FieldSpec &getField() const { return _field; } - void visitPhrase(search::query::Phrase &n); + void visitPhrase(query::Phrase &n); template <typename WS, typename NODE> void createWeightedSet(WS *bp, NODE &n); - void visitWeightedSetTerm(search::query::WeightedSetTerm &n); - void visitDotProduct(search::query::DotProduct &n); - void visitWandTerm(search::query::WandTerm &n); + void visitWeightedSetTerm(query::WeightedSetTerm &n); + void visitDotProduct(query::DotProduct &n); + void visitWandTerm(query::WandTerm &n); - void handleNumberTermAsText(search::query::NumberTerm &n); + void handleNumberTermAsText(query::NumberTerm &n); void illegalVisit() {} - void visit(search::query::And &) override { illegalVisit(); } - void visit(search::query::AndNot &) override { illegalVisit(); } - void visit(search::query::Equiv &) override { illegalVisit(); } - void visit(search::query::Near &) override { illegalVisit(); } - void visit(search::query::ONear &) override { illegalVisit(); } - void visit(search::query::Or &) override { illegalVisit(); } - void visit(search::query::Rank &) override { illegalVisit(); } - void visit(search::query::WeakAnd &) override { illegalVisit(); } - - void visit(search::query::Phrase &n) override { + void visit(query::And &) override { illegalVisit(); } + void visit(query::AndNot &) override { illegalVisit(); } + void visit(query::Equiv &) override { illegalVisit(); } + void visit(query::Near &) override { illegalVisit(); } + void visit(query::ONear &) override { illegalVisit(); } + void visit(query::Or &) override { illegalVisit(); } + void visit(query::Rank &) override { illegalVisit(); } + void visit(query::WeakAnd &) override { illegalVisit(); } + void visit(query::SameElement &) override { illegalVisit(); } + + void visit(query::Phrase &n) override { visitPhrase(n); } - void visit(search::query::WeightedSetTerm &n) override { visitWeightedSetTerm(n); } - void visit(search::query::DotProduct &n) override { visitDotProduct(n); } - void visit(search::query::WandTerm &n) override { visitWandTerm(n); } - - void visit(search::query::NumberTerm &n) override = 0; - void visit(search::query::LocationTerm &n) override = 0; - void visit(search::query::PrefixTerm &n) override = 0; - void visit(search::query::RangeTerm &n) override = 0; - void visit(search::query::StringTerm &n) override = 0; - void visit(search::query::SubstringTerm &n) override = 0; - void visit(search::query::SuffixTerm &n) override = 0; - void visit(search::query::RegExpTerm &n) override = 0; + void visit(query::WeightedSetTerm &n) override { visitWeightedSetTerm(n); } + void visit(query::DotProduct &n) override { visitDotProduct(n); } + void visit(query::WandTerm &n) override { visitWandTerm(n); } + + void visit(query::NumberTerm &n) override = 0; + void visit(query::LocationTerm &n) override = 0; + void visit(query::PrefixTerm &n) override = 0; + void visit(query::RangeTerm &n) override = 0; + void visit(query::StringTerm &n) override = 0; + void visit(query::SubstringTerm &n) override = 0; + void visit(query::SuffixTerm &n) override = 0; + void visit(query::RegExpTerm &n) override = 0; }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/get_weight_from_node.cpp b/searchlib/src/vespa/searchlib/queryeval/get_weight_from_node.cpp index 8fa6af74ae2..bd9de0a1762 100644 --- a/searchlib/src/vespa/searchlib/queryeval/get_weight_from_node.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/get_weight_from_node.cpp @@ -13,8 +13,7 @@ using search::query::Weight; namespace search::queryeval { namespace { -struct WeightExtractor : public TemplateTermVisitor<WeightExtractor, - SimpleQueryNodeTypes> { +struct WeightExtractor : public TemplateTermVisitor<WeightExtractor, SimpleQueryNodeTypes> { Weight weight; WeightExtractor() : weight(0) {} diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp new file mode 100644 index 00000000000..a563a288396 --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp @@ -0,0 +1,82 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "same_element_blueprint.h" +#include "same_element_search.h" +#include <vespa/searchlib/fef/termfieldmatchdata.h> +#include <vespa/vespalib/objects/visit.hpp> +#include <algorithm> +#include <map> + +namespace search::queryeval { + +SameElementBlueprint::SameElementBlueprint() + : ComplexLeafBlueprint(FieldSpecBaseList()), + _estimate(), + _layout(), + _terms() +{ +} + +FieldSpec +SameElementBlueprint::getNextChildField(const vespalib::string &field_name, uint32_t field_id) +{ + return FieldSpec(field_name, field_id, _layout.allocTermField(field_id), false); +} + +void +SameElementBlueprint::addTerm(Blueprint::UP term) +{ + const State &childState = term->getState(); + assert(childState.numFields() == 1); + HitEstimate childEst = childState.estimate(); + if (_terms.empty() || (childEst < _estimate)) { + _estimate = childEst; + setEstimate(_estimate); + } + _terms.push_back(std::move(term)); +} + +void +SameElementBlueprint::optimize_self() +{ + std::sort(_terms.begin(), _terms.end(), + [](const auto &a, const auto &b) + { + return (a->getState().estimate() < b->getState().estimate()); + }); +} + +void +SameElementBlueprint::fetchPostings(bool strict) +{ + for (size_t i = 0; i < _terms.size(); ++i) { + _terms[i]->fetchPostings(strict && (i == 0)); + } +} + +SearchIterator::UP +SameElementBlueprint::createLeafSearch(const search::fef::TermFieldMatchDataArray &tfmda, + bool strict) const +{ + (void) tfmda; + assert(!tfmda.valid()); + fef::MatchData::UP md = _layout.createMatchData(); + search::fef::TermFieldMatchDataArray childMatch; + std::vector<SearchIterator::UP> children(_terms.size()); + for (size_t i = 0; i < _terms.size(); ++i) { + const State &childState = _terms[i]->getState(); + assert(childState.numFields() == 1); + childMatch.add(childState.field(0).resolve(*md)); + children[i] = _terms[i]->createSearch(*md, (strict && (i == 0))); + } + return std::make_unique<SameElementSearch>(std::move(md), std::move(children), childMatch, strict); +} + +void +SameElementBlueprint::visitMembers(vespalib::ObjectVisitor &visitor) const +{ + ComplexLeafBlueprint::visitMembers(visitor); + visit(visitor, "terms", _terms); +} + +} diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.h new file mode 100644 index 00000000000..050a2bc31d4 --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.h @@ -0,0 +1,43 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "searchable.h" +#include <vespa/searchlib/fef/matchdatalayout.h> + +namespace search::fef { class TermFieldMatchData; } + +namespace search::queryeval { + +class SameElementBlueprint : public ComplexLeafBlueprint +{ +private: + HitEstimate _estimate; + fef::MatchDataLayout _layout; + std::vector<Blueprint::UP> _terms; + +public: + SameElementBlueprint(); + SameElementBlueprint(const SameElementBlueprint &) = delete; + SameElementBlueprint &operator=(const SameElementBlueprint &) = delete; + ~SameElementBlueprint() = default; + + // no match data + bool isWhiteList() const override { return true; } + + // used by create visitor + FieldSpec getNextChildField(const vespalib::string &field_name, uint32_t field_id); + + // used by create visitor + void addTerm(Blueprint::UP term); + + void optimize_self() override; + void fetchPostings(bool strict) override; + + SearchIteratorUP createLeafSearch(const search::fef::TermFieldMatchDataArray &tfmda, + bool strict) const override; + void visitMembers(vespalib::ObjectVisitor &visitor) const override; + const std::vector<Blueprint::UP> &terms() const { return _terms; } +}; + +} diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_search.cpp b/searchlib/src/vespa/searchlib/queryeval/same_element_search.cpp new file mode 100644 index 00000000000..8f3fc9c350d --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_search.cpp @@ -0,0 +1,117 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "same_element_search.h" +#include <vespa/searchlib/fef/termfieldmatchdata.h> +#include <vespa/vespalib/objects/visit.h> +#include <vespa/vespalib/objects/visit.hpp> +#include <algorithm> +#include <functional> + +using TFMD = search::fef::TermFieldMatchData; + +namespace search::queryeval { + +namespace { + +template <typename It> +int32_t try_match(const fef::TermFieldMatchDataArray &match, std::vector<It> &iterators, uint32_t cand) { + for (size_t i = 0; i < iterators.size(); ++i) { + while ((iterators[i] != match[i]->end()) && (iterators[i]->getElementId() < cand)) { + ++iterators[i]; + } + if (iterators[i] == match[i]->end()) { + return -1; + } + if (iterators[i]->getElementId() != cand) { + return iterators[i]->getElementId(); + } + } + return cand; +} + +} + +bool +SameElementSearch::check_docid_match(uint32_t docid) +{ + for (const auto &child: _children) { + if (!child->seek(docid)) { + return false; + } + } + return true; +} + +void +SameElementSearch::unpack_children(uint32_t docid) +{ + for (const auto &child: _children) { + child->doUnpack(docid); + } + for (size_t i = 0; i < _childMatch.size(); ++i) { + _iterators[i] = _childMatch[i]->begin(); + } +} + +bool +SameElementSearch::check_element_match(uint32_t docid) +{ + unpack_children(docid); + int32_t cand = 0; + int32_t next = try_match(_childMatch, _iterators, cand); + while (next > cand) { + cand = next; + next = try_match(_childMatch, _iterators, cand); + } + return (cand == next); +} + +SameElementSearch::SameElementSearch(fef::MatchData::UP md, + std::vector<SearchIterator::UP> children, + const fef::TermFieldMatchDataArray &childMatch, + bool strict) + : _md(std::move(md)), + _children(std::move(children)), + _childMatch(childMatch), + _iterators(childMatch.size()), + _strict(strict) +{ + assert(!_children.empty()); + assert(_childMatch.valid()); +} + +void +SameElementSearch::initRange(uint32_t begin_id, uint32_t end_id) +{ + SearchIterator::initRange(begin_id, end_id); + for (const auto &child: _children) { + child->initRange(begin_id, end_id); + } +} + +void +SameElementSearch::doSeek(uint32_t docid) { + if (check_docid_match(docid) && check_element_match(docid)) { + setDocId(docid); + } else if (_strict) { + docid = std::max(docid + 1, _children[0]->getDocId()); + while (!isAtEnd(docid)) { + if (check_docid_match(docid) && check_element_match(docid)) { + setDocId(docid); + return; + } + docid = std::max(docid + 1, _children[0]->getDocId()); + } + setAtEnd(); + } +} + +void +SameElementSearch::visitMembers(vespalib::ObjectVisitor &visitor) const +{ + SearchIterator::visitMembers(visitor); + visit(visitor, "children", _children); + visit(visitor, "strict", _strict); +} + +} diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_search.h b/searchlib/src/vespa/searchlib/queryeval/same_element_search.h new file mode 100644 index 00000000000..6a116c76e73 --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_search.h @@ -0,0 +1,44 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "searchiterator.h" +#include <vespa/searchlib/fef/matchdata.h> +#include <vespa/searchlib/fef/termfieldmatchdataarray.h> +#include <vespa/searchlib/fef/termfieldmatchdata.h> +#include <memory> +#include <vector> + +namespace search::queryeval { + +/** + * Search iterator for a collection of terms that need to match within + * the same element (array index). + */ +class SameElementSearch : public SearchIterator +{ +private: + using It = fef::TermFieldMatchData::PositionsIterator; + + fef::MatchData::UP _md; + std::vector<SearchIterator::UP> _children; + fef::TermFieldMatchDataArray _childMatch; + std::vector<It> _iterators; + bool _strict; + + void unpack_children(uint32_t docid); + bool check_docid_match(uint32_t docid); + bool check_element_match(uint32_t docid); + +public: + SameElementSearch(fef::MatchData::UP md, + std::vector<SearchIterator::UP> children, + const fef::TermFieldMatchDataArray &childMatch, + bool strict); + void initRange(uint32_t begin_id, uint32_t end_id) override; + void doSeek(uint32_t docid) override; + void doUnpack(uint32_t) override {} + void visitMembers(vespalib::ObjectVisitor &visitor) const override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/queryeval/termasstring.cpp b/searchlib/src/vespa/searchlib/queryeval/termasstring.cpp index 14a6cefaf1b..3829ea45e2b 100644 --- a/searchlib/src/vespa/searchlib/queryeval/termasstring.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/termasstring.cpp @@ -22,6 +22,7 @@ using search::query::Node; using search::query::ONear; using search::query::Or; using search::query::Phrase; +using search::query::SameElement; using search::query::PredicateQuery; using search::query::PrefixTerm; using search::query::QueryVisitor; @@ -84,6 +85,7 @@ struct TermAsStringVisitor : public QueryVisitor { void visit(ONear &) override {illegalVisit(); } void visit(Or &) override {illegalVisit(); } void visit(Phrase &) override {illegalVisit(); } + void visit(SameElement &) override {illegalVisit(); } void visit(Rank &) override {illegalVisit(); } void visit(WeakAnd &) override {illegalVisit(); } void visit(WeightedSetTerm &) override {illegalVisit(); } diff --git a/searchsummary/src/tests/extractkeywords/extractkeywordstest.cpp b/searchsummary/src/tests/extractkeywords/extractkeywordstest.cpp index eac6f65a48e..b2920b39eaf 100644 --- a/searchsummary/src/tests/extractkeywords/extractkeywordstest.cpp +++ b/searchsummary/src/tests/extractkeywords/extractkeywordstest.cpp @@ -200,7 +200,7 @@ ExtractKeywordsTest::RunTest(int testno, bool verify) stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "foobar")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "foo")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "bar")); - stack.Push(new search::ParseItem(search::ParseItem::ITEM_PHRASE, 3)); + stack.Push(new search::ParseItem(search::ParseItem::ITEM_PHRASE, 3, "index")); stack.AppendBuffer(&buf); keywords = _extractor->ExtractKeywords(vespalib::stringref(buf.GetDrainPos(), buf.GetUsedLen())); @@ -216,11 +216,11 @@ ExtractKeywordsTest::RunTest(int testno, bool verify) // multiple phrase and term query stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "xyzzy")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "xyz")); - stack.Push(new search::ParseItem(search::ParseItem::ITEM_PHRASE, 2)); + stack.Push(new search::ParseItem(search::ParseItem::ITEM_PHRASE, 2, "index")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "foobar")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "foo")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "bar")); - stack.Push(new search::ParseItem(search::ParseItem::ITEM_PHRASE, 3)); + stack.Push(new search::ParseItem(search::ParseItem::ITEM_PHRASE, 3, "index")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "baz")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "zog")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_AND, 3)); @@ -241,7 +241,7 @@ ExtractKeywordsTest::RunTest(int testno, bool verify) stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "foo")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_AND, 2)); stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "bar")); - stack.Push(new search::ParseItem(search::ParseItem::ITEM_PHRASE, 2)); + stack.Push(new search::ParseItem(search::ParseItem::ITEM_PHRASE, 2, "index")); stack.AppendBuffer(&buf); keywords = _extractor->ExtractKeywords(vespalib::stringref(buf.GetDrainPos(), buf.GetUsedLen())); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp index e535eef660c..c7eb63a4480 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp @@ -280,6 +280,7 @@ JuniperQueryAdapter::Traverse(juniper::IQueryVisitor *v) const case search::ParseItem::ITEM_SUFFIXTERM: case search::ParseItem::ITEM_REGEXP: case search::ParseItem::ITEM_PREDICATE_QUERY: + case search::ParseItem::ITEM_SAME_ELEMENT: if (!v->VisitOther(&item, iterator.getArity())) { rc = SkipItem(&iterator); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp index e153a898f6a..3a60db52cf3 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp @@ -165,7 +165,7 @@ KeywordExtractor::ExtractKeywords(vespalib::stringref buf) const break; case search::ParseItem::ITEM_PHRASE: - { + { // Must take the next arity TERMS and put together bool phraseterms_was_added = false; int phraseterms = si.getArity(); diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index ad98d64b173..6bb2ca31ec1 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -206,10 +206,8 @@ StorageNode::initialize() _chain.reset(createChain().release()); - if (_component->enableMultipleBucketSpaces()) { - assert(_communicationManager != nullptr); - _communicationManager->updateBucketSpacesConfig(*_bucketSpacesConfig); - } + assert(_communicationManager != nullptr); + _communicationManager->updateBucketSpacesConfig(*_bucketSpacesConfig); // Start the metric manager, such that it starts generating snapshots // and the like. Note that at this time, all metrics should hopefully @@ -359,9 +357,7 @@ StorageNode::handleLiveConfigUpdate(const InitialGuard & initGuard) if (_newBucketSpacesConfig) { _bucketSpacesConfig = std::move(_newBucketSpacesConfig); _context.getComponentRegister().setBucketSpacesConfig(*_bucketSpacesConfig); - if (_component->enableMultipleBucketSpaces()) { - _communicationManager->updateBucketSpacesConfig(*_bucketSpacesConfig); - } + _communicationManager->updateBucketSpacesConfig(*_bucketSpacesConfig); } } diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp index 33fca0f161a..f83188f7dd8 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp @@ -17,12 +17,12 @@ mbus::string StorageProtocol::NAME = "StorageProtocol"; StorageProtocol::StorageProtocol(const std::shared_ptr<const document::DocumentTypeRepo> repo, const documentapi::LoadTypeSet& loadTypes, - bool activateBucketSpaceSerialization) + bool configForcedBucketSpaceSerialization) : _serializer5_0(repo, loadTypes), _serializer5_1(repo, loadTypes), _serializer5_2(repo, loadTypes), _serializer6_0(repo, loadTypes), - _activateBucketSpaceSerialization(activateBucketSpaceSerialization) + _configForcedBucketSpaceSerialization(configForcedBucketSpaceSerialization) { } @@ -106,7 +106,7 @@ StorageProtocol::encode(const vespalib::Version& version, } else if (version < version5_2) { return encodeMessage(_serializer5_1, routable, message, version5_1, version); } else { - if (_activateBucketSpaceSerialization) { + if (_configForcedBucketSpaceSerialization) { return encodeMessage(_serializer6_0, routable, message, version6_0, version); } else { if (version < version6_0) { @@ -184,7 +184,7 @@ StorageProtocol::decode(const vespalib::Version & version, } else if (version < version5_2) { return decodeMessage(_serializer5_1, data, type, version5_1, version); } else { - if (_activateBucketSpaceSerialization) { + if (_configForcedBucketSpaceSerialization) { return decodeMessage(_serializer6_0, data, type, version6_0, version); } else { if (version < version6_0) { diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h index d85e9d55d1a..56f271db1d0 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h +++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h @@ -29,7 +29,7 @@ private: ProtocolSerialization5_1 _serializer5_1; ProtocolSerialization5_2 _serializer5_2; ProtocolSerialization6_0 _serializer6_0; - bool _activateBucketSpaceSerialization; + bool _configForcedBucketSpaceSerialization; }; } diff --git a/storageserver/src/tests/testhelper.cpp b/storageserver/src/tests/testhelper.cpp index b245a6500bd..5e6a71b078f 100644 --- a/storageserver/src/tests/testhelper.cpp +++ b/storageserver/src/tests/testhelper.cpp @@ -96,7 +96,11 @@ vdstestlib::DirConfig getStandardConfig(bool storagenode) { // By default, need "old" behaviour of maxconcurrent config->set("maxconcurrentvisitors_fixed", "4"); config->set("maxconcurrentvisitors_variable", "0"); - config = &dc.addConfig("stor-visitordispatcher"); + dc.addConfig("stor-visitordispatcher"); + config = &dc.addConfig("bucketspaces"); + config->set("documenttype[1]"); + config->set("documenttype[0].name", "testdoctype1"); + config->set("documenttype[0].bucketspace", "default"); addFileConfig(dc, "documenttypes", "config-doctypes.cfg"); addStorageDistributionConfig(dc); return dc; diff --git a/valgrind-suppressions.txt b/valgrind-suppressions.txt index baef981a3f9..2df6c9c5691 100644 --- a/valgrind-suppressions.txt +++ b/valgrind-suppressions.txt @@ -6,6 +6,14 @@ fun:pthread_create@@GLIBC_2.2.5 } { + NPTL keeps a cache of thread stacks, and metadata for thread local storage is not freed for threads in that cache + Memcheck:Leak + fun:calloc + fun:allocate_dtv + fun:_dl_allocate_tls + fun:pthread_create@@GLIBC_2.2.5 +} +{ This is a bug in glibc. We can not suffer for that. Memcheck:Free fun:free |