diff options
Diffstat (limited to 'config-model/src/main/java/com/yahoo/searchdefinition')
7 files changed, 282 insertions, 8 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 702eba8c8b5..6960a0a8afd 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -775,7 +775,7 @@ public class RankProfile implements Cloneable { final Phase phase; final String attribute; final String operation; - MutateOperation(Phase phase, String attribute, String operation) { + public MutateOperation(Phase phase, String attribute, String operation) { this.phase = phase; this.attribute = attribute; this.operation = operation; @@ -785,7 +785,7 @@ public class RankProfile implements Cloneable { public void addMutateOperation(MutateOperation.Phase phase, String attribute, String operation) { mutateOperations.add(new MutateOperation(phase, attribute, operation)); - String prefix = "vespa.mutate." + phase.toString().replace('-', '_'); + String prefix = "vespa.mutate." + phase.toString(); addRankProperty(prefix + ".attribute", attribute); addRankProperty(prefix + ".operation", operation); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/InheritanceResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/InheritanceResolver.java new file mode 100644 index 00000000000..488464ccd1f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/InheritanceResolver.java @@ -0,0 +1,105 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.parser; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Class resolving some inheritance relationships. + * + * @author arnej27959 + **/ +public class InheritanceResolver { + + private final Map<String, ParsedSchema> parsedSchemas; + private final Map<String, ParsedDocument> parsedDocs = new HashMap<>(); + + public InheritanceResolver(Map<String, ParsedSchema> parsedSchemas) { + this.parsedSchemas = parsedSchemas; + } + + private void inheritanceCycleCheck(ParsedSchema schema, List<String> seen) { + String name = schema.name(); + if (seen.contains(name)) { + seen.add(name); + throw new IllegalArgumentException("Inheritance cycle for schemas: " + + String.join(" -> ", seen)); + } + seen.add(name); + for (ParsedSchema parent : schema.getResolvedInherits()) { + inheritanceCycleCheck(parent, seen); + } + seen.remove(name); + } + + private void resolveSchemaInheritance() { + for (ParsedSchema schema : parsedSchemas.values()) { + for (String inherit : schema.getInherited()) { + var parent = parsedSchemas.get(inherit); + if (parent == null) { + throw new IllegalArgumentException("schema " + schema.name() + " inherits from unavailable schema " + inherit); + } + schema.resolveInherit(inherit, parent); + } + } + } + + private void checkSchemaCycles() { + List<String> seen = new ArrayList<>(); + for (ParsedSchema schema : parsedSchemas.values()) { + inheritanceCycleCheck(schema, seen); + } + } + + private void resolveDocumentInheritance() { + for (ParsedSchema schema : parsedSchemas.values()) { + if (! schema.hasDocument()) { + // TODO: is schema without a document even valid? + continue; + } + ParsedDocument doc = schema.getDocument(); + var old = parsedDocs.put(doc.name(), doc); + assert(old == null); + } + for (ParsedDocument doc : parsedDocs.values()) { + for (String inherit : doc.getInherited()) { + var parentDoc = parsedDocs.get(inherit); + if (parentDoc == null) { + throw new IllegalArgumentException("document " + doc.name() + " inherits from unavailable document " + inherit); + } + doc.resolveInherit(inherit, parentDoc); + } + } + } + + private void inheritanceCycleCheck(ParsedDocument document, List<String> seen) { + String name = document.name(); + if (seen.contains(name)) { + seen.add(name); + throw new IllegalArgumentException("Inheritance cycle for documents: " + + String.join(" -> ", seen)); + } + seen.add(name); + for (ParsedDocument parent : document.getResolvedInherits()) { + inheritanceCycleCheck(parent, seen); + } + seen.remove(name); + } + + private void checkDocumentCycles() { + List<String> seen = new ArrayList<>(); + for (ParsedDocument doc : parsedDocs.values()) { + inheritanceCycleCheck(doc, seen); + } + } + + public void resolveInheritance() { + resolveSchemaInheritance(); + resolveDocumentInheritance(); + checkSchemaCycles(); + checkDocumentCycles(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java new file mode 100644 index 00000000000..a93fb441563 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java @@ -0,0 +1,146 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.parser; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.io.IOUtils; +import com.yahoo.io.reader.NamedReader; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Class wrapping parsing of schema files and holding a collection of + * schemas in the intermediate format. + * + * @author arnej27959 + **/ +public class IntermediateCollection { + + private final DeployLogger deployLogger; + private final ModelContext.Properties modelProperties; + + private Map<String, ParsedSchema> parsedSchemas = new HashMap<>(); + + IntermediateCollection() { + this.deployLogger = new BaseDeployLogger(); + this.modelProperties = new TestProperties(); + } + + public IntermediateCollection(DeployLogger logger, ModelContext.Properties properties) { + this.deployLogger = logger; + this.modelProperties = properties; + } + + public Map<String, ParsedSchema> getParsedSchemas() { return Map.copyOf(parsedSchemas); } + + public ParsedSchema getParsedSchema(String name) { return parsedSchemas.get(name); } + + ParsedSchema addSchemaFromString(String input) throws ParseException { + var stream = new SimpleCharStream(input); + var parser = new IntermediateParser(stream, deployLogger, modelProperties); + var schema = parser.schema(); + if (parsedSchemas.containsKey(schema.name())) { + throw new IllegalArgumentException("Duplicate schemas named: " + schema.name()); + } + parsedSchemas.put(schema.name(), schema); + return schema; + } + + private void addSchemaFromStringWithFileName(String input, String fileName) throws ParseException { + var parsed = addSchemaFromString(input); + String nameFromFile = baseName(fileName); + if (! parsed.name().equals(nameFromFile)) { + throw new IllegalArgumentException("The file containing schema '" + + parsed.name() + "' must be named '" + + parsed.name() + ApplicationPackage.SD_NAME_SUFFIX + + "', was '" + stripDirs(fileName) + "'"); + } + } + + private String baseName(String filename) { + int pos = filename.lastIndexOf('/'); + if (pos != -1) { + filename = filename.substring(pos + 1); + } + pos = filename.lastIndexOf('.'); + if (pos != -1) { + filename = filename.substring(0, pos); + } + return filename; + } + + private String stripDirs(String filename) { + int pos = filename.lastIndexOf('/'); + if (pos != -1) { + return filename.substring(pos + 1); + } + return filename; + } + + /** + * parse a schema from the given reader and add result to collection + **/ + public void addSchemaFromReader(NamedReader reader) { + try { + addSchemaFromStringWithFileName(IOUtils.readAll(reader.getReader()), reader.getName()); + } catch (java.io.IOException ex) { + throw new IllegalArgumentException("Failed reading from " + reader.getName() + ": " + ex.getMessage()); + } catch (ParseException ex) { + throw new IllegalArgumentException("Failed parsing schema from " + reader.getName() + ": " + ex.getMessage()); + } + + } + + /** for unit tests */ + public void addSchemaFromFile(String fileName) { + try { + addSchemaFromStringWithFileName(IOUtils.readFile(new File(fileName)), fileName); + } catch (java.io.IOException ex) { + throw new IllegalArgumentException("Could not read file " + fileName + ": " + ex.getMessage()); + } catch (ParseException ex) { + throw new IllegalArgumentException("Failed parsing schema file " + fileName + ": " + ex.getMessage()); + } + } + + /** + * parse a rank profile from the given reader and add to the schema identified by name. + * note: the named schema must have been parsed already. + **/ + public void addRankProfileFile(String schemaName, NamedReader reader) { + try { + ParsedSchema schema = parsedSchemas.get(schemaName); + if (schema == null) { + throw new IllegalArgumentException("No schema named: " + schemaName); + } + var stream = new SimpleCharStream(IOUtils.readAll(reader.getReader())); + var parser = new IntermediateParser(stream, deployLogger, modelProperties); + parser.rankProfile(schema); + } catch (java.io.IOException ex) { + throw new IllegalArgumentException("Failed reading from " + reader.getName() + ": " + ex.getMessage()); + } catch (ParseException ex) { + throw new IllegalArgumentException("Failed parsing rank-profile from " + reader.getName() + ": " + ex.getMessage()); + } + } + + // for unit test + void addRankProfileFile(String schemaName, String fileName) { + try { + var reader = IOUtils.createReader(fileName, "UTF-8"); + addRankProfileFile(schemaName, new NamedReader(fileName, reader)); + } catch (java.io.IOException ex) { + throw new IllegalArgumentException("Could not read file " + fileName + ": " + ex.getMessage()); + } + } + + void resolveInternalConnections() { + var resolver = new InheritanceResolver(parsedSchemas); + resolver.resolveInheritance(); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java index 0b7d0507161..8cd64ef16f7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java @@ -17,6 +17,7 @@ import java.util.Optional; **/ public class ParsedDocument extends ParsedBlock { private final List<String> inherited = new ArrayList<>(); + private final Map<String, ParsedDocument> resolvedInherits = new HashMap(); private final Map<String, ParsedField> docFields = new HashMap<>(); private final Map<String, ParsedStruct> docStructs = new HashMap<>(); private final Map<String, ParsedAnnotation> docAnnotations = new HashMap<>(); @@ -27,6 +28,7 @@ public class ParsedDocument extends ParsedBlock { List<String> getInherited() { return List.copyOf(inherited); } List<ParsedAnnotation> getAnnotations() { return List.copyOf(docAnnotations.values()); } + List<ParsedDocument> getResolvedInherits() { return List.copyOf(resolvedInherits.values()); } List<ParsedField> getFields() { return List.copyOf(docFields.values()); } List<ParsedStruct> getStructs() { return List.copyOf(docStructs.values()); } @@ -50,6 +52,13 @@ public class ParsedDocument extends ParsedBlock { docAnnotations.put(annName, annotation); } - public String toString() { return "document "+name(); } + public String toString() { return "document " + name(); } + + void resolveInherit(String name, ParsedDocument parsed) { + verifyThat(inherited.contains(name), "resolveInherit for non-inherited name", name); + verifyThat(name.equals(parsed.name()), "resolveInherit name mismatch for", name); + verifyThat(! resolvedInherits.containsKey(name), "double resolveInherit for", name); + resolvedInherits.put(name, parsed); + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java index 6fce026948d..2011ac24148 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java @@ -39,6 +39,7 @@ class ParsedRankProfile extends ParsedBlock { private String inheritedMatchFeatures = null; private String secondPhaseExpression = null; private Boolean strict = null; + private final List<MutateOperation> mutateOperations = new ArrayList<>(); private final List<String> inherited = new ArrayList<>(); private final Map<String, Boolean> fieldsRankFilter = new HashMap<>(); private final Map<String, Integer> fieldsRankWeight = new HashMap<>(); @@ -66,6 +67,7 @@ class ParsedRankProfile extends ParsedBlock { Optional<String> getFirstPhaseExpression() { return Optional.ofNullable(this.firstPhaseExpression); } Optional<String> getInheritedMatchFeatures() { return Optional.ofNullable(this.inheritedMatchFeatures); } List<ParsedRankFunction> getFunctions() { return List.copyOf(functions.values()); } + List<MutateOperation> getMutateOperations() { return List.copyOf(mutateOperations); } List<String> getInherited() { return List.copyOf(inherited); } Map<String, Boolean> getFieldsWithRankFilter() { return Map.copyOf(fieldsRankFilter); } Map<String, Integer> getFieldsWithRankWeight() { return Map.copyOf(fieldsRankWeight); } @@ -127,7 +129,9 @@ class ParsedRankProfile extends ParsedBlock { functions.put(func.name(), func); } - void addMutateOperation(MutateOperation.Phase phase, String attrName, String operation) {} + void addMutateOperation(MutateOperation.Phase phase, String attrName, String operation) { + mutateOperations.add(new MutateOperation(phase, attrName, operation)); + } void addRankProperty(String key, String value) { verifyThat(! rankProperties.containsKey(key), "already has value for rank property", key); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java index 9bdd6a0409d..bf448b31dd2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java @@ -36,6 +36,7 @@ public class ParsedSchema extends ParsedBlock { private final List<OnnxModel> onnxModels = new ArrayList<>(); private final List<RankingConstant> rankingConstants = new ArrayList<>(); private final List<String> inherited = new ArrayList<>(); + private final Map<String, ParsedSchema> resolvedInherits = new HashMap(); private final Map<String, ParsedAnnotation> extraAnnotations = new HashMap<>(); private final Map<String, ParsedDocumentSummary> docSums = new HashMap<>(); private final Map<String, ParsedField> extraFields = new HashMap<>(); @@ -60,10 +61,11 @@ public class ParsedSchema extends ParsedBlock { List<ParsedField> getFields() { return List.copyOf(extraFields.values()); } List<ParsedFieldSet> getFieldSets() { return List.copyOf(fieldSets.values()); } List<ParsedIndex> getIndexes() { return List.copyOf(extraIndexes.values()); } - List<ParsedRankProfile> getRankProfiles() { return List.copyOf(rankProfiles.values()); } List<ParsedStruct> getStructs() { return List.copyOf(extraStructs.values()); } List<RankingConstant> getRankingConstants() { return List.copyOf(rankingConstants); } List<String> getInherited() { return List.copyOf(inherited); } + Map<String, ParsedRankProfile> getRankProfiles() { return Map.copyOf(rankProfiles); } + List<ParsedSchema> getResolvedInherits() { return List.copyOf(resolvedInherits.values()); } void addAnnotation(ParsedAnnotation annotation) { String annName = annotation.name(); @@ -136,4 +138,12 @@ public class ParsedSchema extends ParsedBlock { "already has stemming", defaultStemming, "cannot also set", value); defaultStemming = value; } + + void resolveInherit(String name, ParsedSchema parsed) { + verifyThat(inherited.contains(name), "resolveInherit for non-inherited name", name); + verifyThat(name.equals(parsed.name()), "resolveInherit name mismatch for", name); + verifyThat(! resolvedInherits.containsKey(name), "double resolveInherit for", name); + resolvedInherits.put(name, parsed); + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java index 4d5280d29eb..9f02c5247ef 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java @@ -104,14 +104,14 @@ class ParsedType { void setCreateIfNonExistent(boolean value) { if (variant != Variant.WSET) { - throw new IllegalArgumentException("CreateIfNonExistent only valid for weightedset, not "+variant); + throw new IllegalArgumentException("CreateIfNonExistent only valid for weightedset, not " + variant); } this.createIfNonExistent = value; } void setRemoveIfZero(boolean value) { if (variant != Variant.WSET) { - throw new IllegalArgumentException("RemoveIfZero only valid for weightedset, not "+variant); + throw new IllegalArgumentException("RemoveIfZero only valid for weightedset, not " + variant); } this.removeIfZero = value; } @@ -119,7 +119,7 @@ class ParsedType { void setVariant(Variant value) { if (variant == value) return; // already OK if (variant != Variant.UNKNOWN) { - throw new IllegalArgumentException("setVariant("+value+") only valid for UNKNOWN, not: "+variant); + throw new IllegalArgumentException("setVariant(" + value + ") only valid for UNKNOWN, not: " + variant); } // maybe even more checking would be useful this.variant = value; |