diff options
34 files changed, 160 insertions, 166 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/document/Attribute.java b/config-model/src/main/java/com/yahoo/schema/document/Attribute.java index 999d040b48c..ac936ee989e 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/schema/document/Attribute.java @@ -5,7 +5,9 @@ import com.yahoo.document.ArrayDataType; import com.yahoo.document.CollectionDataType; import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; +import com.yahoo.document.MapDataType; import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.StructDataType; import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.document.StructuredDataType; import com.yahoo.document.TensorDataType; @@ -143,8 +145,8 @@ public final class Attribute implements Cloneable, Serializable { } /** Creates an attribute with default settings */ - public Attribute(String name, DataType fieldType) { - this(name, convertDataType(fieldType), convertCollectionType(fieldType), convertTensorType(fieldType), convertTargetType(fieldType)); + public Attribute(String schemaName, String fieldName, String name, DataType fieldType) { + this(name, convertDataType(schemaName, fieldName, fieldType), convertCollectionType(fieldType), convertTensorType(fieldType), convertTargetType(fieldType)); setRemoveIfZero(fieldType instanceof WeightedSetDataType wsdt && wsdt.removeIfZero()); setCreateIfNonExistent(fieldType instanceof WeightedSetDataType wsdt && wsdt.createIfNonExistent()); } @@ -266,12 +268,26 @@ public final class Attribute implements Cloneable, Serializable { private void setType(Type type) { this.type=type; } public void setCollectionType(CollectionType type) { this.collectionType=type; } + private static void failDataType(String schemaName, String fieldName, String dataType) throws IllegalArgumentException { + throw new IllegalArgumentException("For schema '" + schemaName + "': Field '" + fieldName + "' of type '" + dataType + "' cannot be an attribute. " + + "Instead specify the struct fields to be searchable as attribute"); + } + public static void validateDataType(String schemaName, String fieldName, DataType fieldType) throws IllegalArgumentException { + if (fieldType instanceof MapDataType mapType) { + failDataType(schemaName, fieldName, "map<" + mapType.getKeyType().getName() + "," + mapType.getValueType().getName() + ">"); + } + if (fieldType instanceof ArrayDataType arrayType && arrayType.getNestedType() instanceof StructDataType nestedType) { + failDataType(schemaName, fieldName, "array<" + nestedType.getName() + ">"); + } + } + /** Converts to the right attribute type from a field datatype */ - public static Type convertDataType(DataType fieldType) { + public static Type convertDataType(String schemaName, String fieldName, DataType fieldType) { + validateDataType(schemaName, fieldName, fieldType); if (fieldType instanceof NewDocumentReferenceDataType) { return Type.REFERENCE; } else if (fieldType instanceof CollectionDataType) { - return convertDataType(((CollectionDataType) fieldType).getNestedType()); + return convertDataType(schemaName, fieldName, ((CollectionDataType) fieldType).getNestedType()); } FieldValue fval = fieldType.createFieldValue(); if (fval instanceof StringFieldValue) { diff --git a/config-model/src/main/java/com/yahoo/schema/document/SDField.java b/config-model/src/main/java/com/yahoo/schema/document/SDField.java index 538cb56d210..f165141b16e 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/SDField.java @@ -398,22 +398,23 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { } /** Parse an indexing expression which will use the simple linguistics implementation suitable for testing */ - public void parseIndexingScript(String script) { - parseIndexingScript(script, new SimpleLinguistics(), Embedder.throwsOnUse.asMap()); + public void parseIndexingScript(String schemaName, String script) { + parseIndexingScript(schemaName, script, new SimpleLinguistics(), Embedder.throwsOnUse.asMap()); } - public void parseIndexingScript(String script, Linguistics linguistics, Map<String, Embedder> embedders) { + public void parseIndexingScript(String schemaName, String script, Linguistics linguistics, Map<String, Embedder> embedders) { try { ScriptParserContext config = new ScriptParserContext(linguistics, embedders); config.setInputStream(new IndexingInput(script)); - setIndexingScript(ScriptExpression.newInstance(config)); + setIndexingScript(schemaName, ScriptExpression.newInstance(config)); } catch (ParseException e) { throw new IllegalArgumentException("Failed to parse script '" + script + "'", e); } } /** Sets the indexing script of this, or null to not use a script */ - public void setIndexingScript(ScriptExpression exp) { + + public void setIndexingScript(String schemaName, ScriptExpression exp) { if (exp == null) { exp = new ScriptExpression(); } @@ -441,13 +442,13 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { } Attribute attribute = attributes.get(fieldName); if (attribute == null) { - addAttribute(new Attribute(fieldName, getDataType())); + addAttribute(new Attribute(schemaName, fieldName, fieldName, getDataType())); } } }.visit(indexingScript); } for (SDField structField : getStructFields()) { - structField.setIndexingScript(exp); + structField.setIndexingScript(schemaName, exp); } } diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperation.java index 22a9eed2914..5a0c2c3c915 100644 --- a/config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperation.java @@ -12,7 +12,7 @@ import com.yahoo.schema.document.SDField; public interface FieldOperation extends Comparable<FieldOperation> { /** Apply this operation on the given field */ - void apply(SDField field); + void apply(String schemaName, SDField field); @Override default int compareTo(FieldOperation other) { diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexingOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexingOperation.java index f5366c4b07a..11065f040ea 100644 --- a/config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexingOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexingOperation.java @@ -28,8 +28,8 @@ public class IndexingOperation implements FieldOperation { public ScriptExpression getScript() { return script; } - public void apply(SDField field) { - field.setIndexingScript(script); + public void apply(String schemaName, SDField field) { + field.setIndexingScript(schemaName, script); } /** Creates an indexing operation which will use the simple linguistics implementation suitable for testing */ diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java index e3ca0090408..053a5ac777b 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java @@ -2,6 +2,7 @@ package com.yahoo.schema.parser; import com.yahoo.document.DataType; +import com.yahoo.schema.document.GeoPos; import com.yahoo.schema.parser.ConvertParsedTypes.TypeResolver; import com.yahoo.schema.Index; import com.yahoo.schema.Schema; @@ -49,10 +50,10 @@ public class ConvertParsedFields { (exactMatchTerminator -> field.getMatching().setExactMatchTerminator(exactMatchTerminator)); } - void convertSorting(SDField field, ParsedSorting parsed, String name) { + void convertSorting(Schema schema, SDField field, ParsedSorting parsed, String name) { Attribute attribute = field.getAttributes().get(name); if (attribute == null) { - attribute = new Attribute(name, field.getDataType()); + attribute = new Attribute(schema.getName(), field.getName(), name, field.getDataType()); field.addAttribute(attribute); } Sorting sorting = attribute.getSorting(); @@ -66,7 +67,7 @@ public class ConvertParsedFields { parsed.getLocale().ifPresent(locale -> sorting.setLocale(locale)); } - void convertAttribute(SDField field, ParsedAttribute parsed) { + void convertAttribute(Schema schema, SDField field, ParsedAttribute parsed) { String name = parsed.name(); String fieldName = field.getName(); Attribute attribute = null; @@ -76,7 +77,7 @@ public class ConvertParsedFields { if (attribute == null) { attribute = field.getAttributes().get(name); if (attribute == null) { - attribute = new Attribute(name, field.getDataType()); + attribute = new Attribute(schema.getName(), field.getName(), name, field.getDataType()); field.addAttribute(attribute); } } @@ -102,7 +103,7 @@ public class ConvertParsedFields { } var sorting = parsed.getSorting(); if (sorting.isPresent()) { - convertSorting(field, sorting.get(), name); + convertSorting(schema, field, sorting.get(), name); } } @@ -143,13 +144,16 @@ public class ConvertParsedFields { convertMatchSettings(field, parsed.matchSettings()); var indexing = parsed.getIndexing(); if (indexing.isPresent()) { - field.setIndexingScript(indexing.get().script()); + field.setIndexingScript(schema.getName(), indexing.get().script()); + } + if (field.doesAttributing() && !GeoPos.isAnyPos(field.getDataType())) { + Attribute.validateDataType(schema.getName(), field.getName(), field.getDataType()); } parsed.getWeight().ifPresent(value -> field.setWeight(value)); parsed.getStemming().ifPresent(value -> field.setStemming(value)); parsed.getNormalizing().ifPresent(value -> convertNormalizing(field, value)); for (var attribute : parsed.getAttributes()) { - convertAttribute(field, attribute); + convertAttribute(schema, field, attribute); } for (var summaryField : parsed.getSummaryFields()) { var dataType = field.getDataType(); @@ -190,7 +194,7 @@ public class ConvertParsedFields { convertCommonFieldSettings(schema, structField, parsed); } - private void convertExtraFieldSettings(SDField field, ParsedField parsed) { + private void convertExtraFieldSettings(Schema schema, SDField field, ParsedField parsed) { String name = parsed.name(); for (var dictOp : parsed.getDictionaryOptions()) { var dictionary = field.getOrSetDictionary(); @@ -208,7 +212,7 @@ public class ConvertParsedFields { field.getAliasToName().put(alias, parsed.lookupAliasedFrom(alias)); } parsed.getRankTypes().forEach((indexName, rankType) -> convertRankType(field, indexName, rankType)); - parsed.getSorting().ifPresent(sortInfo -> convertSorting(field, sortInfo, name)); + parsed.getSorting().ifPresent(sortInfo -> convertSorting(schema, field, sortInfo, name)); if (parsed.hasBolding()) { // TODO must it be so ugly: SummaryField summaryField = field.getSummaryField(name, true); @@ -288,7 +292,7 @@ public class ConvertParsedFields { DataType dataType = context.resolveType(parsed.getType()); var field = new SDField(document, name, dataType); convertCommonFieldSettings(schema, field, parsed); - convertExtraFieldSettings(field, parsed); + convertExtraFieldSettings(schema, field, parsed); document.addField(field); return field; } @@ -298,7 +302,7 @@ public class ConvertParsedFields { DataType dataType = context.resolveType(parsed.getType()); var field = new SDField(schema.getDocument(), name, dataType); convertCommonFieldSettings(schema, field, parsed); - convertExtraFieldSettings(field, parsed); + convertExtraFieldSettings(schema, field, parsed); schema.addExtraField(field); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java index 7bd66ad8f0b..deb57e157f3 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java @@ -86,14 +86,14 @@ public class CreatePositionZCurve extends Processor { SummaryTransform.DISTANCE, summaryTo, validate); } // clear indexing script - field.setIndexingScript(null); + field.setIndexingScript(schema.getName(), null); SDField posX = field.getStructField(PositionDataType.FIELD_X); if (posX != null) { - posX.setIndexingScript(null); + posX.setIndexingScript(schema.getName(), null); } SDField posY = field.getStructField(PositionDataType.FIELD_Y); if (posY != null) { - posY.setIndexingScript(null); + posY.setIndexingScript(schema.getName(), null); } if (doesSummary) ensureCompatibleSummary(field, zName, field.getName(), @@ -118,7 +118,7 @@ public class CreatePositionZCurve extends Processor { ScriptExpression script = inputField.getIndexingScript(); script = (ScriptExpression)new RemoveSummary(inputField.getName()).convert(script); script = (ScriptExpression)new PerformZCurve(field, fieldName).convert(script); - field.setIndexingScript(script); + field.setIndexingScript(schema.getName(), script); return field; } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java index a12183262c4..056c37a9830 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java @@ -75,7 +75,7 @@ public class ExactMatch extends Processor { } ScriptExpression script = field.getIndexingScript(); if (new ExpressionSearcher<>(IndexExpression.class).containedIn(script)) { - field.setIndexingScript((ScriptExpression)new MyProvider(schema).convert(field.getIndexingScript())); + field.setIndexingScript(schema.getName(), (ScriptExpression)new MyProvider(schema).convert(field.getIndexingScript())); } } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingInputs.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingInputs.java index 53ebd136e08..a5ba67d6976 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingInputs.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingInputs.java @@ -36,7 +36,7 @@ public class IndexingInputs extends Processor { if (validate) new VerifyInputExpression(schema, field).visit(script); - field.setIndexingScript(script); + field.setIndexingScript(schema.getName(), script); } } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java index 071c2878ae8..be129b53c5e 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java @@ -37,7 +37,7 @@ public class IndexingOutputs extends Processor { Set<String> summaryFields = new TreeSet<>(); findSummaryTo(schema, field, summaryFields, summaryFields); MyConverter converter = new MyConverter(schema, field, summaryFields, validate); - field.setIndexingScript((ScriptExpression)converter.convert(script)); + field.setIndexingScript(schema.getName(), (ScriptExpression)converter.convert(script)); } } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IntegerIndex2Attribute.java b/config-model/src/main/java/com/yahoo/schema/processing/IntegerIndex2Attribute.java index 0d296783cfb..37815ef5396 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IntegerIndex2Attribute.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IntegerIndex2Attribute.java @@ -38,7 +38,7 @@ public class IntegerIndex2Attribute extends Processor { ScriptExpression script = field.getIndexingScript(); Set<String> attributeNames = new HashSet<>(); new MyVisitor(attributeNames).visit(script); - field.setIndexingScript((ScriptExpression)new MyConverter(attributeNames).convert(script)); + field.setIndexingScript(schema.getName(), (ScriptExpression)new MyConverter(attributeNames).convert(script)); warn(schema, field, "Changed to attribute because numerical indexes (field has type " + field.getDataType().getName() + ") is not currently supported." + " Index-only settings may fail. Ignore this warning for streaming search."); diff --git a/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java index 2ec5c03e04c..c06e0565109 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java @@ -46,7 +46,7 @@ public class NGramMatch extends Processor { field.getNormalizing().inferCodepoint(); field.setStemming(Stemming.NONE); // not compatible with stemming and normalizing field.addQueryCommand("ngram " + n); - field.setIndexingScript((ScriptExpression)new MyProvider(schema, n).convert(field.getIndexingScript())); + field.setIndexingScript(schema.getName(), (ScriptExpression)new MyProvider(schema, n).convert(field.getIndexingScript())); } private static class MyProvider extends TypedTransformProvider { diff --git a/config-model/src/main/java/com/yahoo/schema/processing/OptimizeIlscript.java b/config-model/src/main/java/com/yahoo/schema/processing/OptimizeIlscript.java index b268a7a9c03..fbb49497837 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/OptimizeIlscript.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/OptimizeIlscript.java @@ -27,7 +27,7 @@ public class OptimizeIlscript extends Processor { ScriptExpression script = field.getIndexingScript(); if (script == null) continue; - field.setIndexingScript((ScriptExpression)new ExpressionOptimizer().convert(script)); + field.setIndexingScript(schema.getName(), (ScriptExpression)new ExpressionOptimizer().convert(script)); if ( ! field.getIndexingScript().toString().equals(script.toString())) { info(schema, field, "Rewrote ilscript from:\n" + script.toString() + "\nto\n" + field.getIndexingScript().toString()); diff --git a/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java index 2a654991835..c451df6370d 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java @@ -107,7 +107,7 @@ public class PredicateProcessor extends Processor { script = new StatementExpression(makeSetPredicateVariablesScript(booleanIndexDefiniton), script); ExpressionConverter converter = new PredicateOutputTransformer(schema); - field.setIndexingScript(new ScriptExpression((StatementExpression)converter.convert(script))); + field.setIndexingScript(schema.getName(), new ScriptExpression((StatementExpression)converter.convert(script))); } private Expression makeSetPredicateVariablesScript(BooleanIndexDefinition options) { diff --git a/config-model/src/main/java/com/yahoo/schema/processing/Processor.java b/config-model/src/main/java/com/yahoo/schema/processing/Processor.java index beaff13c613..dd36bbb3b61 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/Processor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/Processor.java @@ -87,7 +87,7 @@ public abstract class Processor { implementationField.setRankType(RankType.EMPTY); implementationField.setStemming(Stemming.NONE); implementationField.getNormalizing().inferCodepoint(); - implementationField.parseIndexingScript(indexing); + implementationField.parseIndexingScript(schema.getName(), indexing); String indexName = field.getName(); String implementationIndexName = indexName + "_" + suffix; Index implementationIndex = new Index(implementationIndexName); diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java index e6fed35b821..e29f683761f 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java @@ -48,7 +48,7 @@ public class TextMatch extends Processor { if ( ! visitor.requiresTokenize) continue; ExpressionConverter converter = new MyStringTokenizer(schema, findAnnotatorConfig(schema, field)); - field.setIndexingScript((ScriptExpression)converter.convert(script)); + field.setIndexingScript(schema.getName(), (ScriptExpression)converter.convert(script)); } } diff --git a/config-model/src/test/java/com/yahoo/schema/AttributeUtils.java b/config-model/src/test/java/com/yahoo/schema/AttributeUtils.java index 69e7e14b3be..9ae24d6bfd4 100644 --- a/config-model/src/test/java/com/yahoo/schema/AttributeUtils.java +++ b/config-model/src/test/java/com/yahoo/schema/AttributeUtils.java @@ -8,8 +8,8 @@ import com.yahoo.schema.document.SDField; */ public class AttributeUtils { - public static void addAttributeAspect(SDField field) { - field.parseIndexingScript("{ attribute }"); + public static void addAttributeAspect(String schemaName, SDField field) { + field.parseIndexingScript(schemaName, "{ attribute }"); } } diff --git a/config-model/src/test/java/com/yahoo/schema/DocumentReferenceResolverTest.java b/config-model/src/test/java/com/yahoo/schema/DocumentReferenceResolverTest.java index 76006ad28d7..e2029df70d6 100644 --- a/config-model/src/test/java/com/yahoo/schema/DocumentReferenceResolverTest.java +++ b/config-model/src/test/java/com/yahoo/schema/DocumentReferenceResolverTest.java @@ -34,7 +34,7 @@ public class DocumentReferenceResolverTest { SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); SDField fooRefToBarField = new SDField (fooDocument, "bar_ref", new NewDocumentReferenceDataType(barDocument.getDocumentType())); - AttributeUtils.addAttributeAspect(fooRefToBarField); + AttributeUtils.addAttributeAspect(fooSchema.getName(), fooRefToBarField); SDField irrelevantField = new SDField(fooDocument, "irrelevant_stuff", DataType.INT); fooDocument.addField(fooRefToBarField); fooDocument.addField(irrelevantField); @@ -60,7 +60,7 @@ public class DocumentReferenceResolverTest { SDField fooRefToBarField = new SDField( fooDocument, "bar_ref", NewDocumentReferenceDataType.forDocumentName("bar")); - AttributeUtils.addAttributeAspect(fooRefToBarField); + AttributeUtils.addAttributeAspect(fooSchema.getName(), fooRefToBarField); fooDocument.addField(fooRefToBarField); fooSchema.addDocument(fooDocument); diff --git a/config-model/src/test/java/com/yahoo/schema/ImportedFieldsEnumeratorTest.java b/config-model/src/test/java/com/yahoo/schema/ImportedFieldsEnumeratorTest.java index 8c0b8c32d81..b96a43e9b53 100644 --- a/config-model/src/test/java/com/yahoo/schema/ImportedFieldsEnumeratorTest.java +++ b/config-model/src/test/java/com/yahoo/schema/ImportedFieldsEnumeratorTest.java @@ -22,7 +22,7 @@ public class ImportedFieldsEnumeratorTest { Schema parentSchema = new Schema(PARENT, MockApplicationPackage.createEmpty()); SDDocumentType parentDocument = new SDDocumentType(PARENT, parentSchema); var parentField = new SDField(parentDocument, "their_field", DataType.INT); - AttributeUtils.addAttributeAspect(parentField); + AttributeUtils.addAttributeAspect(parentSchema.getName(), parentField); parentDocument.addField(parentField); parentSchema.addDocument(parentDocument); diff --git a/config-model/src/test/java/com/yahoo/schema/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/AttributeListTestCase.java index 0cfb9474365..f21999df94c 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/AttributeListTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/AttributeListTestCase.java @@ -14,6 +14,7 @@ import java.util.Iterator; import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; /** * Tests attribute deriving @@ -126,4 +127,38 @@ public class AttributeListTestCase extends AbstractSchemaTestCase { assertFalse(attributes.hasNext()); } + @Test + void bad_map_attribute() throws ParseException { + run_bad_struct_or_map_attribute("map<string,string>"); + } + + @Test + void bad_array_of_struct_attribute() throws ParseException { + run_bad_struct_or_map_attribute("array<s>"); + } + + private void run_bad_struct_or_map_attribute(String type) throws ParseException { + run_bad_struct_or_map_attribute(type, false, true); + run_bad_struct_or_map_attribute(type, true, false); + run_bad_struct_or_map_attribute(type, true, true); + } + + private void run_bad_struct_or_map_attribute(String type, boolean fs, boolean ilscript) throws ParseException { + var exception = assertThrows(IllegalArgumentException.class, () -> ApplicationBuilder.createFromString( + joinLines("search test {", + " document test {", + " struct s {", + " field a type string { }", + " }", + " field metadata type " + type + " {", + " indexing: summary" + (ilscript ? "| attribute" : ""), + " " + (fs ? "attribute: fast-search" : ""), + " }", + " }", + "}")).getSchema()); + assertEquals("For schema 'test': Field 'metadata' of type '" + type + "' cannot be an attribute." + + " Instead specify the struct fields to be searchable as attribute", + exception.getMessage()); + } + } diff --git a/config-model/src/test/java/com/yahoo/schema/derived/IdTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/IdTestCase.java index 188017e0af1..60adf7cbaf0 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/IdTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/IdTestCase.java @@ -30,7 +30,7 @@ public class IdTestCase extends AbstractExportingTestCase { SDDocumentType document = new SDDocumentType("test"); schema.addDocument(document); SDField uri = new SDField(document, "URI", DataType.URI); - uri.parseIndexingScript("{ summary | index }"); + uri.parseIndexingScript(schema.getName(), "{ summary | index }"); document.addField(uri); new Processing().process(schema, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), diff --git a/config-model/src/test/java/com/yahoo/schema/derived/InheritanceTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/InheritanceTestCase.java index 472fa58230e..628f5042140 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/InheritanceTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/InheritanceTestCase.java @@ -159,7 +159,7 @@ public class InheritanceTestCase extends AbstractExportingTestCase { Schema parentSchema = new Schema("parent", MockApplicationPackage.createEmpty()); parentSchema.addDocument(parent); SDField prefixed = parent.addField("prefixed", DataType.STRING); - prefixed.parseIndexingScript("{ index }"); + prefixed.parseIndexingScript(parentSchema.getName(), "{ index }"); prefixed.addIndex(new Index("prefixed", true)); SDDocumentType child = new SDDocumentType("child"); diff --git a/config-model/src/test/java/com/yahoo/schema/derived/LiteralBoostTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/LiteralBoostTestCase.java index 97a3f06ac64..8677a14e66b 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/LiteralBoostTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/LiteralBoostTestCase.java @@ -36,7 +36,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { SDDocumentType document = new SDDocumentType("literalboost"); schema.addDocument(document); SDField field1 = document.addField("a", DataType.STRING); - field1.parseIndexingScript("{ index }"); + field1.parseIndexingScript(schema.getName(), "{ index }"); field1.setLiteralBoost(20); RankProfile other = new RankProfile("other", schema, rankProfileRegistry); rankProfileRegistry.add(other); @@ -69,7 +69,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { SDDocumentType document = new SDDocumentType("literalboost"); schema.addDocument(document); SDField field1 = document.addField("a", DataType.STRING); - field1.parseIndexingScript("{ index }"); + field1.parseIndexingScript(schema.getName(), "{ index }"); RankProfile other = new RankProfile("other", schema, rankProfileRegistry); rankProfileRegistry.add(other); other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); @@ -95,10 +95,10 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { SDDocumentType document = new SDDocumentType("msb"); schema.addDocument(document); SDField field1 = document.addField("title", DataType.STRING); - field1.parseIndexingScript("{ summary | index }"); + field1.parseIndexingScript(schema.getName(), "{ summary | index }"); field1.setLiteralBoost(20); SDField field2 = document.addField("body", DataType.STRING); - field2.parseIndexingScript("{ summary | index }"); + field2.parseIndexingScript(schema.getName(), "{ summary | index }"); field2.setLiteralBoost(20); schema = ApplicationBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry()); diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java index a27bc824b45..63510785ca5 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java @@ -172,7 +172,7 @@ public class SummaryTestCase extends AbstractSchemaTestCase { schema.addDocument(document); String fieldName = "location"; SDField field = document.addField(fieldName, PositionDataType.INSTANCE); - field.parseIndexingScript("{ attribute | summary }"); + field.parseIndexingScript(schema.getName(), "{ attribute | summary }"); new Processing().process(schema, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true, false, Set.of()); diff --git a/config-model/src/test/java/com/yahoo/schema/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/TypeConversionTestCase.java index d2b3acc9a1e..cdfe376416b 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/TypeConversionTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/TypeConversionTestCase.java @@ -32,7 +32,7 @@ public class TypeConversionTestCase extends AbstractSchemaTestCase { SDDocumentType document = new SDDocumentType("test"); schema.addDocument(document); SDField a = new SDField(document, "a", DataType.STRING); - a.parseIndexingScript("{ index }"); + a.parseIndexingScript(schema.getName(), "{ index }"); document.addField(a); new Processing().process(schema, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), diff --git a/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java index 852f567ccfa..a90b4fa8d9f 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java @@ -46,7 +46,7 @@ public class VsmFieldsTestCase { void reference_type_field_is_unsearchable() { Schema schema = createSchema(); SDField field = new TemporarySDField(schema.getDocument(), "ref_field", NewDocumentReferenceDataType.forDocumentName("parent_type")); - field.parseIndexingScript("{ summary }"); + field.parseIndexingScript(schema.getName(), "{ summary }"); schema.getDocument().addField(field); VsmfieldsConfig cfg = vsmfieldsConfig(schema); @@ -59,7 +59,7 @@ public class VsmFieldsTestCase { private void testIndexMatching(Matching matching, VsmfieldsConfig.Fieldspec.Normalize.Enum normalize, String arg1) { Schema schema = createSchema(); SDField field = new TemporarySDField(schema.getDocument(), "f", DataType.STRING); - field.parseIndexingScript("{ index }"); + field.parseIndexingScript(schema.getName(), "{ index }"); field.setMatching(matching); schema.getDocument().addField(field); VsmfieldsConfig cfg = vsmfieldsConfig(schema); diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java index c2cc28ea6b3..b487622a928 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java @@ -151,7 +151,7 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { void requireThatMaxTermOccurrencesIsPropagated() { var field = new SDField("test", DataType.STRING); field.getMatching().maxTermOccurrences(10); - field.parseIndexingScript("{ summary | index }"); + field.parseIndexingScript("test", "{ summary | index }"); assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" max-occurrences:10 | summary test | index test; }", field); } @@ -173,14 +173,14 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { private static SDField createField(String name, DataType type, String script) { SDField field = new SDField(null, name, type); - field.parseIndexingScript(script); + field.parseIndexingScript("test", script); return field; } private static SDField createPredicateField( String name, DataType type, String script, int arity, OptionalLong lower_bound, OptionalLong upper_bound) { SDField field = new SDField(null, name, type); - field.parseIndexingScript(script); + field.parseIndexingScript("test", script); Index index = new Index("foo"); index.setBooleanIndexDefiniton(new BooleanIndexDefinition( OptionalInt.of(arity), lower_bound, upper_bound, OptionalDouble.empty())); diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ParentChildSearchModel.java b/config-model/src/test/java/com/yahoo/schema/processing/ParentChildSearchModel.java index af275feffed..129d65584ba 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/ParentChildSearchModel.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/ParentChildSearchModel.java @@ -36,8 +36,11 @@ public class ParentChildSearchModel { } protected static TemporarySDField createField(SDDocumentType repo, String name, DataType dataType, String indexingScript) { + return createField(repo, repo.getName(), name, dataType, indexingScript); + } + protected static TemporarySDField createField(SDDocumentType repo, String schemaName, String name, DataType dataType, String indexingScript) { TemporarySDField result = new TemporarySDField(repo, name, dataType); - result.parseIndexingScript(indexingScript); + result.parseIndexingScript(schemaName, indexingScript); return result; } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java index b2291099b44..ae1db366c9f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java @@ -18,6 +18,7 @@ import java.util.List; import java.util.logging.Level; import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -67,7 +68,9 @@ public class ComplexFieldsValidatorTestCase { "}", "}")); }); - assertTrue(exception.getMessage().contains(getExpectedMessage("docTopics (docTopics.topics, docTopics.topics.id, docTopics.topics.label)"))); + assertEquals("For schema 'test': Field 'docTopics.topics' of type 'array<topic>' cannot be an attribute." + + " Instead specify the struct fields to be searchable as attribute", + exception.getMessage()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java index f9999a30869..7809a97f85c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java @@ -41,8 +41,8 @@ public class SchemaClusterTest { SDDocumentType sdt1 = new SDDocumentType("s1"); Schema schema1 = new Schema("s1", MockApplicationPackage.createEmpty()); SDField f1 = new SDField(sdt1, "f1", DataType.STRING); - f1.addAttribute(new Attribute("f1", DataType.STRING)); - f1.setIndexingScript(new ScriptExpression(new StatementExpression(new AttributeExpression("f1")))); + f1.addAttribute(new Attribute(schema1.getName(), f1.getName(), "f1", DataType.STRING)); + f1.setIndexingScript("s1", new ScriptExpression(new StatementExpression(new AttributeExpression("f1")))); sdt1.addField(f1); schema1.addDocument(sdt1); @@ -50,8 +50,8 @@ public class SchemaClusterTest { SDDocumentType sdt2 = new SDDocumentType("s2"); Schema schema2 = new Schema("s2", MockApplicationPackage.createEmpty()); SDField f2 = new SDField(sdt2, "f2", DataType.STRING); - f2.addAttribute(new Attribute("f2", DataType.STRING)); - f2.setIndexingScript(new ScriptExpression(new StatementExpression(new AttributeExpression("f2")))); + f2.addAttribute(new Attribute(schema2.getName(), f2.getName(), "f2", DataType.STRING)); + f2.setIndexingScript("s2", new ScriptExpression(new StatementExpression(new AttributeExpression("f2")))); sdt2.addField(f2); schema2.addDocument(sdt2); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java index 8502bfa92f4..672bba83e87 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java @@ -102,7 +102,7 @@ public class SchemaInfoTestCase { var schemaInfoTester = new SchemaInfoTester(); var schema = schemaInfoTester.createSchema("test"); var field = (SDField)schema.getDocument().addField(new SDField("f1", DataType.STRING)); - var attribute = field.addAttribute(new Attribute("f1Attribute", field.getDataType())); + var attribute = field.addAttribute(new Attribute(schema.getName(), field.getName(), "f1Attribute", field.getDataType())); attribute.getAliases().add("a1"); attribute.getAliases().add("a2"); assertEquals(""" diff --git a/persistence/src/vespa/persistence/spi/context.cpp b/persistence/src/vespa/persistence/spi/context.cpp index deed8d451bd..4cd016215d2 100644 --- a/persistence/src/vespa/persistence/spi/context.cpp +++ b/persistence/src/vespa/persistence/spi/context.cpp @@ -4,7 +4,7 @@ namespace storage::spi { -Context::Context(Priority pri, int maxTraceLevel) noexcept +Context::Context(Priority pri, uint32_t maxTraceLevel) noexcept : _priority(pri), _trace(maxTraceLevel), _readConsistency(ReadConsistency::STRONG) diff --git a/persistence/src/vespa/persistence/spi/context.h b/persistence/src/vespa/persistence/spi/context.h index 590acd1c549..2b35d5f3a8d 100644 --- a/persistence/src/vespa/persistence/spi/context.h +++ b/persistence/src/vespa/persistence/spi/context.h @@ -38,7 +38,7 @@ using Priority = uint16_t; // 0 - max pri, 255 - min pri // Define this type just because a ton of tests currently use it. struct Trace { - using TraceLevel = int; + using TraceLevel = uint32_t; }; class Context { @@ -48,10 +48,10 @@ class Context { public: Context(Context &&) noexcept = default; Context & operator = (Context &&) noexcept = default; - Context(Priority pri, int maxTraceLevel) noexcept; + Context(Priority pri, uint32_t maxTraceLevel) noexcept; ~Context(); - Priority getPriority() const noexcept { return _priority; } + [[nodiscard]] Priority getPriority() const noexcept { return _priority; } /** * A read operation might choose to relax its consistency requirements, @@ -65,16 +65,16 @@ public: void setReadConsistency(ReadConsistency consistency) noexcept { _readConsistency = consistency; } - ReadConsistency getReadConsistency() const noexcept { + [[nodiscard]] ReadConsistency getReadConsistency() const noexcept { return _readConsistency; } - vespalib::Trace && steal_trace() noexcept { return std::move(_trace); } - vespalib::Trace& getTrace() noexcept { return _trace; } - const vespalib::Trace& getTrace() const noexcept { return _trace; } + [[nodiscard]] vespalib::Trace && steal_trace() noexcept { return std::move(_trace); } + [[nodiscard]] vespalib::Trace& getTrace() noexcept { return _trace; } + [[nodiscard]] const vespalib::Trace& getTrace() const noexcept { return _trace; } - bool shouldTrace(int level) noexcept { return _trace.shouldTrace(level); } - void trace(int level, vespalib::stringref msg, bool addTime = true) { + [[nodiscard]] bool shouldTrace(uint32_t level) const noexcept { return _trace.shouldTrace(level); } + void trace(uint32_t level, vespalib::stringref msg, bool addTime = true) { _trace.trace(level, msg, addTime); } }; diff --git a/searchlib/src/tests/queryeval/flow/queryeval_flow_test.cpp b/searchlib/src/tests/queryeval/flow/queryeval_flow_test.cpp index 181c591ab20..5eb3a2ebf47 100644 --- a/searchlib/src/tests/queryeval/flow/queryeval_flow_test.cpp +++ b/searchlib/src/tests/queryeval/flow/queryeval_flow_test.cpp @@ -7,6 +7,7 @@ constexpr size_t loop_cnt = 64; constexpr size_t max_work = 1; // 500'000'000; +constexpr bool dump_unexpected = false; constexpr bool verbose = false; using namespace search::queryeval; @@ -40,7 +41,7 @@ double dual_ordered_cost_of(const std::vector<FlowStats> &data, InFlow in_flow, any_flow.update_cost(total_cost, child_cost); any_flow.add(item.estimate); } - EXPECT_EQ(total_cost, result); + EXPECT_DOUBLE_EQ(total_cost, result); return result; } @@ -174,7 +175,7 @@ void verify_flow(auto flow, const std::vector<double> &est_list, const std::vect AnyFlow any_flow = AnyFlow::create<decltype(flow)>(InFlow(flow.strict(), flow.flow())); ASSERT_EQ(est_list.size() + 1, expect.size()); for (size_t i = 0; i < est_list.size(); ++i) { - EXPECT_EQ(any_flow.flow(), flow.flow()); + EXPECT_DOUBLE_EQ(any_flow.flow(), flow.flow()); EXPECT_EQ(any_flow.strict(), flow.strict()); EXPECT_DOUBLE_EQ(flow.flow(), expect[i].flow); EXPECT_EQ(flow.strict(), expect[i].strict); @@ -182,7 +183,7 @@ void verify_flow(auto flow, const std::vector<double> &est_list, const std::vect any_flow.add(est_list[i]); flow.add(est_list[i]); } - EXPECT_EQ(any_flow.flow(), flow.flow()); + EXPECT_DOUBLE_EQ(any_flow.flow(), flow.flow()); EXPECT_EQ(any_flow.strict(), flow.strict()); EXPECT_DOUBLE_EQ(flow.flow(), expect.back().flow); EXPECT_EQ(flow.strict(), expect.back().strict); @@ -356,10 +357,10 @@ TEST(FlowTest, optimal_and_flow) { double min_cost = AndFlow::cost_of(data, strict); double max_cost = 0.0; AndFlow::sort(data, strict); - EXPECT_EQ(ordered_cost_of<AndFlow>(data, strict, false), min_cost); + EXPECT_DOUBLE_EQ(ordered_cost_of<AndFlow>(data, strict, false), min_cost); auto check = [&](const std::vector<FlowStats> &my_data) noexcept { double my_cost = ordered_cost_of<AndFlow>(my_data, strict, false); - EXPECT_LE(min_cost, my_cost); + EXPECT_LE(min_cost, my_cost + 1e-9); max_cost = std::max(max_cost, my_cost); }; each_perm(data, check); @@ -379,7 +380,7 @@ TEST(FlowTest, optimal_or_flow) { double min_cost = OrFlow::cost_of(data, strict); double max_cost = 0.0; OrFlow::sort(data, strict); - EXPECT_EQ(ordered_cost_of<OrFlow>(data, strict, false), min_cost); + EXPECT_DOUBLE_EQ(ordered_cost_of<OrFlow>(data, strict, false), min_cost); auto check = [&](const std::vector<FlowStats> &my_data) noexcept { double my_cost = ordered_cost_of<OrFlow>(my_data, strict, false); EXPECT_LE(min_cost, my_cost + 1e-9); @@ -451,7 +452,7 @@ void test_strict_AND_sort_strategy(auto my_sort) { flow.add(item.estimate); total_cost += child_cost; } - EXPECT_EQ(total_cost, ordered_cost_of<AndFlow>(list, true, true)); + EXPECT_DOUBLE_EQ(total_cost, ordered_cost_of<AndFlow>(list, true, true)); fprintf(stderr, " total cost: %10f\n", total_cost); }; auto verify_order = [&](const std::vector<FlowStats> &list){ @@ -515,10 +516,7 @@ void test_strict_AND_sort_strategy(auto my_sort) { }; each_perm(data, check); double rel_err = 0.0; - double cost_range = (max_cost - min_cost); - if (cost_range > 1e-9) { - rel_err = (est_cost - min_cost) / cost_range; - } + rel_err = (est_cost - min_cost) / min_cost; if (rel_err > max_rel_err) { max_rel_err = rel_err; my_worst_order = my_order; @@ -526,7 +524,7 @@ void test_strict_AND_sort_strategy(auto my_sort) { } sum_rel_err += rel_err; errs.push_back(rel_err); - if (verbose && !verify_order(best_order)) { + if (dump_unexpected && !verify_order(best_order)) { fprintf(stderr, " BEST ORDER IS UNEXPECTED:\n"); dump_flow(best_order, best_order); fprintf(stderr, " UNEXPECTED case, my_order:\n"); @@ -551,60 +549,12 @@ TEST(FlowTest, strict_and_with_allow_force_strict_basic_order) { test_strict_AND_sort_strategy(my_sort); } -TEST(FlowTest, strict_and_with_allow_force_strict_incremental_strict_selection) { - auto my_sort = [](auto &data) { - AndFlow::sort(data, true); - for (size_t next = 1; (next + 1) < data.size(); ++next) { - auto [idx, score] = flow::select_forced_strict_and_child(flow::DirectAdapter(), data, next); - if (score >= 0.0) { - break; - } - auto pos = data.begin() + idx; - std::rotate(data.begin() + next, pos, pos + 1); - } - }; - test_strict_AND_sort_strategy(my_sort); -} - -TEST(FlowTest, strict_and_with_allow_force_strict_incremental_strict_selection_with_strict_re_sorting) { - auto my_sort = [](auto &data) { - AndFlow::sort(data, true); - size_t strict_cnt = 1; - for (; strict_cnt < data.size(); ++strict_cnt) { - auto [idx, score] = flow::select_forced_strict_and_child(flow::DirectAdapter(), data, strict_cnt); - if (score >= 0.0) { - break; - } - auto pos = data.begin() + idx; - std::rotate(data.begin() + strict_cnt, pos, pos + 1); - } - std::sort(data.begin(), data.begin() + strict_cnt, - [](const auto &a, const auto &b){ return (a.estimate < b.estimate); }); - }; - test_strict_AND_sort_strategy(my_sort); -} - -TEST(FlowTest, strict_and_with_allow_force_strict_incremental_strict_selection_with_order) { - auto my_sort = [](auto &data) { - AndFlow::sort(data, true); - for (size_t next = 1; next < data.size(); ++next) { - auto [idx, target, score] = flow::select_forced_strict_and_child_with_order(flow::DirectAdapter(), data, next); - if (score >= 0.0) { - break; - } - auto pos = data.begin() + idx; - std::rotate(data.begin() + target, pos, pos + 1); - } - }; - test_strict_AND_sort_strategy(my_sort); -} - -TEST(FlowTest, strict_and_with_allow_force_strict_incremental_strict_selection_with_destructive_order) { +TEST(FlowTest, strict_and_with_allow_force_strict_incremental_strict_selection_destructive_order_max_3_extra_strict) { auto my_sort = [](auto &data) { AndFlow::sort(data, true); - for (size_t next = 1; next < data.size(); ++next) { - auto [idx, target, score] = flow::select_forced_strict_and_child_with_destructive_order(flow::DirectAdapter(), data, next); - if (score >= 0.0) { + for (size_t next = 1; next <= 3 && next < data.size(); ++next) { + auto [idx, target, diff] = flow::select_forced_strict_and_child(flow::DirectAdapter(), data, next); + if (diff >= 0.0) { break; } auto pos = data.begin() + idx; diff --git a/searchlib/src/vespa/searchlib/queryeval/flow.h b/searchlib/src/vespa/searchlib/queryeval/flow.h index b57e823f4d7..9ac79244755 100644 --- a/searchlib/src/vespa/searchlib/queryeval/flow.h +++ b/searchlib/src/vespa/searchlib/queryeval/flow.h @@ -214,44 +214,39 @@ size_t select_strict_and_child(auto adapter, const auto &children) { return best_idx; } -auto select_forced_strict_and_child_impl(auto adapter, const auto &children, size_t first, auto ordered, auto destructive) { - auto est_until = [&](size_t limit)noexcept{ - double my_est = 1.0; - for (size_t i = 0; i < limit; ++i) { - my_est *= adapter.estimate(children[i]); - } - return my_est; - }; - double est = est_until(first); +auto select_forced_strict_and_child(auto adapter, const auto &children, size_t first) { + double est = 1.0; double cost = 0.0; size_t best_idx = first; size_t best_target = first; double best_diff = std::numeric_limits<double>::max(); + for (size_t i = 0; i < first; ++i) { + est *= adapter.estimate(children[i]); + } for (size_t idx = first; idx < children.size(); ++idx) { auto child = FlowStats::from(adapter, children[idx]); double child_abs_cost = est * child.cost; double forced_cost = forced_strict_cost(child, est); double my_diff = (forced_cost + child.estimate * cost) - (cost + child_abs_cost); size_t target = first; - while (ordered && target > 0) { + while (target > 0) { size_t candidate = target - 1; auto other = FlowStats::from(adapter, children[candidate]); if (other.estimate < child.estimate) { // do not move past someone with lower estimate break; } - if (!destructive && !should_force_strict(other, (est_until(candidate) * child.estimate))) { - // no not move past someone who will lose strictness, - // unless it is explicitly allowed by enabling - // destructive (re-)ordering - break; - } target = candidate; my_diff += (0.2 * child.estimate - 0.2 * other.estimate); if (candidate == 0) { // the first iterator produces its own in-flow my_diff += (0.2 * child.estimate - 0.2 * other.estimate); } + // note that 'my_diff' might overestimate the cost + // (underestimate the benefit) of inserting 'child' before + // 'other' if it leads to 'other' becoming + // non-strict. This will also leave 'other' in a + // potentially unoptimal location. } if (my_diff < best_diff) { best_diff = my_diff; @@ -264,19 +259,6 @@ auto select_forced_strict_and_child_impl(auto adapter, const auto &children, siz return std::make_tuple(best_idx, best_target, best_diff); } -auto select_forced_strict_and_child(auto adapter, const auto &children, size_t first) { - auto [idx, target, diff] = select_forced_strict_and_child_impl(adapter, children, first, std::false_type{}, std::false_type{}); - return std::make_pair(idx, diff); -} - -auto select_forced_strict_and_child_with_order(auto adapter, const auto &children, size_t first) { - return select_forced_strict_and_child_impl(adapter, children, first, std::true_type{}, std::false_type{}); -} - -auto select_forced_strict_and_child_with_destructive_order(auto adapter, const auto &children, size_t first) { - return select_forced_strict_and_child_impl(adapter, children, first, std::true_type{}, std::true_type{}); -} - } // flow template <typename FLOW> |