diff options
Diffstat (limited to 'config-model/src/main/java/com/yahoo/schema')
21 files changed, 177 insertions, 163 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java index e747235dc3c..92f7e87209a 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java @@ -12,6 +12,7 @@ import com.yahoo.schema.document.Sorting; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.indexinglanguage.expressions.ToPositionExpression; +import java.io.IOException; import java.util.Collection; import java.util.Collections; import java.util.Iterator; @@ -27,7 +28,7 @@ import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isSupportedCo * * @author bratseth */ -public class AttributeFields extends Derived implements AttributesConfig.Producer { +public class AttributeFields extends Derived { public enum FieldSet {ALL, FAST_ACCESS} @@ -178,12 +179,16 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce return "attributes"; } - @Override public void getConfig(AttributesConfig.Builder builder) { - //TODO This is just to get some exporting tests to work, Should be undone and removed getConfig(builder, FieldSet.ALL, 77777); } + public void export(String toDirectory) throws IOException { + var builder = new AttributesConfig.Builder(); + getConfig(builder); + export(toDirectory, builder.build()); + } + private boolean isAttributeInFieldSet(Attribute attribute, FieldSet fs) { return (fs == FieldSet.ALL) || ((fs == FieldSet.FAST_ACCESS) && attribute.isFastAccess()); } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/Derived.java b/config-model/src/main/java/com/yahoo/schema/derived/Derived.java index 3ed9807b8f1..f4311912584 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/Derived.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/Derived.java @@ -2,7 +2,6 @@ package com.yahoo.schema.derived; import com.yahoo.config.ConfigInstance; -import com.yahoo.config.ConfigInstance.Builder; import com.yahoo.document.Field; import com.yahoo.io.IOUtils; import com.yahoo.schema.Index; @@ -14,7 +13,6 @@ import com.yahoo.text.StringUtilities; import java.io.IOException; import java.io.Writer; -import java.lang.reflect.Method; import java.util.List; /** @@ -22,7 +20,7 @@ import java.util.List; * * @author bratseth */ -public abstract class Derived implements Exportable { +public abstract class Derived { private String name; @@ -88,23 +86,13 @@ public abstract class Derived implements Exportable { return labels ? getName() : String.valueOf(number); } - /** - * Exports this derived configuration to its .cfg file - * in toDirectory - * - * @param toDirectory the directory to export to, or null - * - */ - public void export(String toDirectory) throws IOException { + protected void export(String toDirectory, ConfigInstance cfg) throws IOException { Writer writer = null; try { String fileName = getDerivedName() + ".cfg"; - if (toDirectory != null) - writer = IOUtils.createWriter(toDirectory + "/" + fileName,false); - try { - exportBuilderConfig(writer); - } catch (ReflectiveOperationException | SecurityException | IllegalArgumentException e) { - throw new RuntimeException(e); + if (toDirectory != null) { + writer = IOUtils.createWriter(toDirectory + "/" + fileName, false); + exportConfig(writer, cfg); } } finally { @@ -112,30 +100,10 @@ public abstract class Derived implements Exportable { } } - /** - * Checks what this is a producer of, instantiate that and export to writer - */ - // TODO move to ReflectionUtil, and move that to unexported pkg - private void exportBuilderConfig(Writer writer) throws ReflectiveOperationException, SecurityException, IllegalArgumentException, IOException { - for (Class<?> intf : getClass().getInterfaces()) { - if (ConfigInstance.Producer.class.isAssignableFrom(intf)) { - Class<?> configClass = intf.getEnclosingClass(); - String builderClassName = configClass.getCanonicalName()+"$Builder"; - Class<?> builderClass = Class.forName(builderClassName); - ConfigInstance.Builder builder = (Builder) builderClass.getDeclaredConstructor().newInstance(); - Method getConfig = getClass().getMethod("getConfig", builderClass); - getConfig.invoke(this, builder); - ConfigInstance inst = (ConfigInstance) configClass.getConstructor(builderClass).newInstance(builder); - List<String> payloadL = ConfigInstance.serialize(inst); - String payload = StringUtilities.implodeMultiline(payloadL); - writer.write(payload); - } - } - } - - @Override - public String getFileName() { - return getDerivedName() + ".cfg"; + private void exportConfig(Writer writer, ConfigInstance cfg) throws IOException { + List<String> payloadL = ConfigInstance.serialize(cfg); + String payload = StringUtilities.implodeMultiline(payloadL); + writer.write(payload); } } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java index bc1c097ea4b..b35918b3460 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java @@ -12,6 +12,7 @@ import com.yahoo.schema.RankProfileRegistry; import com.yahoo.schema.Schema; import com.yahoo.schema.derived.validation.Validation; import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -27,6 +28,7 @@ import java.io.Writer; public class DerivedConfiguration { private final Schema schema; + private final boolean isStreaming; private Summaries summaries; private Juniperrc juniperrc; private AttributeFields attributeFields; @@ -66,6 +68,7 @@ public class DerivedConfiguration { * schema is later modified. */ public DerivedConfiguration(Schema schema, DeployState deployState, boolean isStreaming) { + this.isStreaming = isStreaming; try { Validator.ensureNotNull("Schema", schema); this.schema = schema; @@ -126,10 +129,15 @@ public class DerivedConfiguration { } public void exportConstants(String toDirectory) throws IOException { - RankingConstantsConfig.Builder b = new RankingConstantsConfig.Builder(); - rankProfileList.getConfig(b); + var b = new RankingConstantsConfig.Builder() + .constant(rankProfileList.getConstantsConfig()); exportCfg(b.build(), toDirectory + "/" + "ranking-constants.cfg"); } + public void exportOnnxModels(String toDirectory) throws IOException { + var b = new OnnxModelsConfig.Builder() + .model(rankProfileList.getOnnxConfig()); + exportCfg(b.build(), toDirectory + "/" + "onnx-models.cfg"); + } private static void exportCfg(ConfigInstance instance, String fileName) throws IOException { Writer writer = null; @@ -144,6 +152,10 @@ public class DerivedConfiguration { } } + public boolean isStreaming() { + return isStreaming; + } + public Summaries getSummaries() { return summaries; } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/Exportable.java b/config-model/src/main/java/com/yahoo/schema/derived/Exportable.java deleted file mode 100644 index 983be8bf13a..00000000000 --- a/config-model/src/main/java/com/yahoo/schema/derived/Exportable.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.schema.derived; - -/** - * Classes exportable to configurations - * - * @author bratseth - */ -public interface Exportable { - - /** - * Exports the configuration of this object - * - * - * @param toDirectory the directory to export to, does not write to disk if null - * @throws java.io.IOException if exporting fails, some files may still be created - */ - public void export(String toDirectory) throws java.io.IOException; - - /** - * The (short) name of the exported file - * @return a String with the (short) name of the exported file - */ - public String getFileName(); - -} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedConstants.java b/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedConstants.java index 91ab1ad9172..275fbe82ad4 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedConstants.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedConstants.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; /** @@ -25,7 +26,7 @@ public class FileDistributedConstants { public FileDistributedConstants(FileRegistry fileRegistry, Collection<RankProfile.Constant> constants) { Map<String, DistributableConstant> distributableConstants = new LinkedHashMap<>(); for (var constant : constants) { - if ( ! constant.valuePath().isPresent()) continue; + if (constant.valuePath().isEmpty()) continue; var distributableConstant = new DistributableConstant(constant.name().simpleArgument().get(), constant.type(), @@ -41,23 +42,21 @@ public class FileDistributedConstants { /** Returns a read-only map of the constants in this indexed by name. */ public Map<String, DistributableConstant> asMap() { return constants; } - public void getConfig(RankingConstantsConfig.Builder builder) { - for (var constant : constants.values()) { - builder.constant(new RankingConstantsConfig.Constant.Builder() - .name(constant.getName()) - .fileref(constant.getFileReference()) - .type(constant.getType())); - } + private static RankingConstantsConfig.Constant.Builder toConfig(DistributableConstant constant) { + return new RankingConstantsConfig.Constant.Builder() + .name(constant.getName()) + .fileref(constant.getFileReference()) + .type(constant.getType()); + } + + public List<RankingConstantsConfig.Constant.Builder> getConfig() { + return constants.values().stream().map(FileDistributedConstants::toConfig).toList(); } public static class DistributableConstant extends DistributableResource { private final TensorType tensorType; - public DistributableConstant(String name, TensorType type, String fileName) { - this(name, type, fileName, PathType.FILE); - } - public DistributableConstant(String name, TensorType type, String fileName, PathType pathType) { super(name, fileName, pathType); this.tensorType = type; diff --git a/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedOnnxModels.java b/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedOnnxModels.java index 1cc33cc4180..6e50e3c094c 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedOnnxModels.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedOnnxModels.java @@ -5,9 +5,11 @@ import com.yahoo.config.application.api.FileRegistry; import com.yahoo.schema.OnnxModel; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.logging.Logger; @@ -16,13 +18,10 @@ import java.util.logging.Logger; * * @author bratseth */ -public class FileDistributedOnnxModels extends Derived implements OnnxModelsConfig.Producer { +public class FileDistributedOnnxModels { private static final Logger log = Logger.getLogger(FileDistributedOnnxModels.class.getName()); - @Override - public String getDerivedName() { return "onnx-models"; } - private final Map<String, OnnxModel> models; public FileDistributedOnnxModels(FileRegistry fileRegistry, Collection<OnnxModel> models) { @@ -47,30 +46,36 @@ public class FileDistributedOnnxModels extends Derived implements OnnxModelsConf public Map<String, OnnxModel> asMap() { return models; } - public void getConfig(OnnxModelsConfig.Builder builder) { + private static OnnxModelsConfig.Model.Builder toConfig(OnnxModel model) { + OnnxModelsConfig.Model.Builder builder = new OnnxModelsConfig.Model.Builder(); + builder.dry_run_on_setup(true); + builder.name(model.getName()); + builder.fileref(model.getFileReference()); + model.getInputMap().forEach((name, source) -> builder.input(new OnnxModelsConfig.Model.Input.Builder().name(name).source(source))); + model.getOutputMap().forEach((name, as) -> builder.output(new OnnxModelsConfig.Model.Output.Builder().name(name).as(as))); + if (model.getStatelessExecutionMode().isPresent()) + builder.stateless_execution_mode(model.getStatelessExecutionMode().get()); + if (model.getStatelessInterOpThreads().isPresent()) + builder.stateless_interop_threads(model.getStatelessInterOpThreads().get()); + if (model.getStatelessIntraOpThreads().isPresent()) + builder.stateless_intraop_threads(model.getStatelessIntraOpThreads().get()); + if (model.getGpuDevice().isPresent()) { + builder.gpu_device(model.getGpuDevice().get().deviceNumber()); + builder.gpu_device_required(model.getGpuDevice().get().required()); + } + return builder; + } + + public List<OnnxModelsConfig.Model.Builder> getConfig() { + List<OnnxModelsConfig.Model.Builder> cfgList = new ArrayList<>(); for (OnnxModel model : models.values()) { if ("".equals(model.getFileReference())) log.warning("Illegal file reference " + model); // Let tests pass ... we should find a better way else { - OnnxModelsConfig.Model.Builder modelBuilder = new OnnxModelsConfig.Model.Builder(); - modelBuilder.dry_run_on_setup(true); - modelBuilder.name(model.getName()); - modelBuilder.fileref(model.getFileReference()); - model.getInputMap().forEach((name, source) -> modelBuilder.input(new OnnxModelsConfig.Model.Input.Builder().name(name).source(source))); - model.getOutputMap().forEach((name, as) -> modelBuilder.output(new OnnxModelsConfig.Model.Output.Builder().name(name).as(as))); - if (model.getStatelessExecutionMode().isPresent()) - modelBuilder.stateless_execution_mode(model.getStatelessExecutionMode().get()); - if (model.getStatelessInterOpThreads().isPresent()) - modelBuilder.stateless_interop_threads(model.getStatelessInterOpThreads().get()); - if (model.getStatelessIntraOpThreads().isPresent()) - modelBuilder.stateless_intraop_threads(model.getStatelessIntraOpThreads().get()); - if (model.getGpuDevice().isPresent()) { - modelBuilder.gpu_device(model.getGpuDevice().get().deviceNumber()); - modelBuilder.gpu_device_required(model.getGpuDevice().get().required()); - } - builder.model(modelBuilder); + cfgList.add(toConfig(model)); } } + return cfgList; } } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java b/config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java index a781adcefb7..e5e28ccea27 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java @@ -9,6 +9,7 @@ import com.yahoo.schema.document.ImportedComplexField; import com.yahoo.schema.document.ImportedField; import com.yahoo.vespa.config.search.ImportedFieldsConfig; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -22,7 +23,7 @@ import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isMapOfSimple * * @author geirst */ -public class ImportedFields extends Derived implements ImportedFieldsConfig.Producer { +public class ImportedFields extends Derived { private Optional<com.yahoo.schema.document.ImportedFields> importedFields = Optional.empty(); @@ -40,7 +41,6 @@ public class ImportedFields extends Derived implements ImportedFieldsConfig.Prod return "imported-fields"; } - @Override public void getConfig(ImportedFieldsConfig.Builder builder) { // Replace if (importedFields.isPresent()) { @@ -50,6 +50,12 @@ public class ImportedFields extends Derived implements ImportedFieldsConfig.Prod } } + public void export(String toDirectory) throws IOException { + var builder = new ImportedFieldsConfig.Builder(); + getConfig(builder); + export(toDirectory, builder.build()); + } + private static boolean isNestedFieldName(String fieldName) { return fieldName.indexOf('.') != -1; } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java index b0cf4a0013c..c1b698df55f 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java @@ -24,6 +24,7 @@ import com.yahoo.schema.processing.NGramMatch; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.search.config.IndexInfoConfig; +import java.io.IOException; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -34,7 +35,7 @@ import java.util.Set; * * @author bratseth */ -public class IndexInfo extends Derived implements IndexInfoConfig.Producer { +public class IndexInfo extends Derived { private static final String CMD_ATTRIBUTE = "attribute"; private static final String CMD_DEFAULT_POSITION = "default-position"; @@ -314,7 +315,6 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { return true; } - @Override public void getConfig(IndexInfoConfig.Builder builder) { // Append IndexInfoConfig.Indexinfo.Builder iiB = new IndexInfoConfig.Indexinfo.Builder(); @@ -335,6 +335,12 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { builder.indexinfo(iiB); } + public void export(String toDirectory) throws IOException { + var builder = new IndexInfoConfig.Builder(); + getConfig(builder); + export(toDirectory, builder.build()); + } + // TODO: Move this to the FieldSetSettings processor (and rename it) as that already has to look at this. private void addFieldSetCommands(IndexInfoConfig.Indexinfo.Builder iiB, FieldSet fieldSet) { for (String qc : fieldSet.queryCommands()) { diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java index 1c22e80e579..34ed9f2f60c 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java @@ -12,6 +12,7 @@ import com.yahoo.schema.document.FieldSet; import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.vespa.config.search.IndexschemaConfig; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; @@ -24,7 +25,7 @@ import java.util.Map; * * @author geirst */ -public class IndexSchema extends Derived implements IndexschemaConfig.Producer { +public class IndexSchema extends Derived { private final List<IndexField> fields = new ArrayList<>(); private final Map<String, FieldCollection> collections = new LinkedHashMap<>(); @@ -131,13 +132,18 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { return fsB; } - @Override public void getConfig(IndexschemaConfig.Builder icB) { // Replace icB.indexfield(fields.stream().map(IndexSchema::createIndexFieldConfig).toList()); icB.fieldset(fieldSets.values().stream().map(IndexSchema::createFieldSetConfig).toList()); } + public void export(String toDirectory) throws IOException { + var builder = new IndexschemaConfig.Builder(); + getConfig(builder); + export(toDirectory, builder.build()); + } + static List<Field> flattenField(Field field) { DataType fieldType = field.getDataType(); if (fieldType.getPrimitiveType() != null){ diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java index 7245e3f266d..60d27c617f1 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; import com.yahoo.vespa.indexinglanguage.expressions.ClearStateExpression; import com.yahoo.vespa.indexinglanguage.expressions.Expression; -import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; import com.yahoo.vespa.indexinglanguage.expressions.GuardExpression; import com.yahoo.vespa.indexinglanguage.expressions.InputExpression; import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; @@ -22,6 +21,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; import com.yahoo.vespa.indexinglanguage.expressions.TokenizeExpression; import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -35,7 +35,7 @@ import java.util.Set; * * @author bratseth */ -public final class IndexingScript extends Derived implements IlscriptsConfig.Producer { +public final class IndexingScript extends Derived { private final List<String> docFields = new ArrayList<>(); private final List<Expression> expressions = new ArrayList<>(); @@ -93,7 +93,6 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro return "ilscripts"; } - @Override public void getConfig(IlscriptsConfig.Builder configBuilder) { // Append IlscriptsConfig.Ilscript.Builder ilscriptBuilder = new IlscriptsConfig.Ilscript.Builder(); @@ -103,6 +102,12 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro configBuilder.ilscript(ilscriptBuilder); } + public void export(String toDirectory) throws IOException { + var builder = new IlscriptsConfig.Builder(); + getConfig(builder); + export(toDirectory, builder.build()); + } + private static class DropTokenize extends ExpressionConverter { @Override protected boolean shouldConvert(Expression exp) { diff --git a/config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java b/config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java index a39fa642f34..58e77cedaf1 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.config.search.summary.JuniperrcConfig; +import java.io.IOException; import java.util.ArrayList; import java.util.Set; import java.util.stream.Collectors; @@ -15,7 +16,7 @@ import java.util.stream.Collectors; * * @author Simon Thoresen Hult */ -public class Juniperrc extends Derived implements JuniperrcConfig.Producer { +public class Juniperrc extends Derived { private static final int Mb = 1024 * 1024; @@ -43,8 +44,13 @@ public class Juniperrc extends Derived implements JuniperrcConfig.Producer { } } - @Override - protected String getDerivedName() { return "juniperrc"; } + public void export(String toDirectory) throws IOException { + var builder = new JuniperrcConfig.Builder(); + getConfig(builder); + export(toDirectory, builder.build()); + } + + @Override protected String getDerivedName() { return "juniperrc"; } private static JuniperrcConfig.Override.Builder createOverride(String name) { return new JuniperrcConfig.Override.Builder() @@ -55,7 +61,6 @@ public class Juniperrc extends Derived implements JuniperrcConfig.Producer { .surround_max(64*Mb); } - @Override public void getConfig(JuniperrcConfig.Builder builder) { // Replace if (!boldingFields.isEmpty()) { diff --git a/config-model/src/main/java/com/yahoo/schema/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/schema/derived/RankProfileList.java index 0a419768aaf..b08629d68a3 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/RankProfileList.java @@ -4,6 +4,7 @@ package com.yahoo.schema.derived; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.schema.RankingExpressionBody; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.schema.LargeRankingExpressions; import com.yahoo.schema.OnnxModel; @@ -16,6 +17,7 @@ import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -32,7 +34,7 @@ import java.util.concurrent.Future; * * @author bratseth */ -public class RankProfileList extends Derived implements RankProfilesConfig.Producer { +public class RankProfileList extends Derived { private final Map<String, RawRankProfile> rankProfiles; private final FileDistributedConstants constants; @@ -190,32 +192,32 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ public FileDistributedConstants constants() { return constants; } public FileDistributedOnnxModels getOnnxModels() { return onnxModels; } - @Override - public String getDerivedName() { return "rank-profiles"; } + @Override public String getDerivedName() { return "rank-profiles"; } - @Override - public void export(String toDirectory) throws java.io.IOException { - super.export(toDirectory); - onnxModels.export(toDirectory); + public void export(String toDirectory) throws IOException { + export(toDirectory, new RankProfilesConfig.Builder().rankprofile(getRankProfilesConfig()).build()); } - @Override - public void getConfig(RankProfilesConfig.Builder builder) { - for (RawRankProfile rank : rankProfiles.values() ) { - rank.getConfig(builder); - } + public List<RankProfilesConfig.Rankprofile.Builder> getRankProfilesConfig() { + return rankProfiles.values().stream().map(RawRankProfile::getConfig).toList(); + } + + private static RankingExpressionsConfig.Expression.Builder toConfig(RankingExpressionBody expr) { + return new RankingExpressionsConfig.Expression.Builder() + .name(expr.getName()) + .fileref(expr.getFileReference()); } - public void getConfig(RankingExpressionsConfig.Builder builder) { - largeRankingExpressions.expressions().forEach((expr) -> builder.expression.add(new RankingExpressionsConfig.Expression.Builder().name(expr.getName()).fileref(expr.getFileReference()))); + public List<RankingExpressionsConfig.Expression.Builder> getExpressionsConfig() { + return largeRankingExpressions.expressions().stream().map(RankProfileList::toConfig).toList(); } - public void getConfig(RankingConstantsConfig.Builder builder) { - constants.getConfig(builder); + public List<RankingConstantsConfig.Constant.Builder> getConstantsConfig() { + return constants.getConfig(); } - public void getConfig(OnnxModelsConfig.Builder builder) { - onnxModels.getConfig(builder); + public List<OnnxModelsConfig.Model.Builder> getOnnxConfig() { + return onnxModels.getConfig(); } } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java index db76d6397fc..b057624f055 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java @@ -43,7 +43,7 @@ import java.util.Set; * * @author bratseth */ -public class RawRankProfile implements RankProfilesConfig.Producer { +public class RawRankProfile { /** A reusable compressor with default settings */ private static final Compressor compressor = new Compressor(); @@ -130,12 +130,11 @@ public class RawRankProfile implements RankProfilesConfig.Producer { */ public List<Pair<String, String>> configProperties() { return decompress(compressedProperties); } - @Override - public void getConfig(RankProfilesConfig.Builder builder) { + public RankProfilesConfig.Rankprofile.Builder getConfig() { RankProfilesConfig.Rankprofile.Builder b = new RankProfilesConfig.Rankprofile.Builder().name(getName()); getRankProperties(b); buildNormalizers(b); - builder.rankprofile(b); + return b; } @Override diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java index fb6d7fb70ec..19a045ac444 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java @@ -21,6 +21,7 @@ import com.yahoo.schema.RankProfileRegistry; import com.yahoo.schema.Schema; import com.yahoo.searchlib.rankingexpression.Reference; +import java.io.IOException; import java.util.Collection; import java.util.Collections; import java.util.Iterator; @@ -32,7 +33,7 @@ import java.util.Map; * * @author bratseth */ -public final class SchemaInfo extends Derived implements SchemaInfoConfig.Producer { +public final class SchemaInfo extends Derived { private final Schema schema; @@ -63,7 +64,6 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc return rankProfileInfos; } - @Override public void getConfig(SchemaInfoConfig.Builder builder) { // Append var schemaBuilder = new SchemaInfoConfig.Schema.Builder(); @@ -75,6 +75,12 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc builder.schema(schemaBuilder); } + public void export(String toDirectory) throws IOException { + var builder = new SchemaInfoConfig.Builder(); + getConfig(builder); + export(toDirectory, builder.build()); + } + private void addFieldsConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) { for (var field : schema.allFieldsList()) { addFieldConfig(field, schemaBuilder); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/Summaries.java b/config-model/src/main/java/com/yahoo/schema/derived/Summaries.java index 51c648aa1be..1f0c6f7e761 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/Summaries.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/Summaries.java @@ -7,6 +7,7 @@ import com.yahoo.schema.Schema; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.config.search.SummaryConfig; +import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -15,7 +16,7 @@ import java.util.List; * * @author bratseth */ -public class Summaries extends Derived implements SummaryConfig.Producer { +public class Summaries extends Derived { private final boolean useV8GeoPositions; private final List<SummaryClass> summaries; @@ -36,15 +37,18 @@ public class Summaries extends Derived implements SummaryConfig.Producer { public List<SummaryClass> asList() { return summaries; } - @Override - protected String getDerivedName() { return "summary"; } + @Override protected String getDerivedName() { return "summary"; } - @Override public void getConfig(SummaryConfig.Builder builder) { // Replace builder.defaultsummaryid(summaries.isEmpty() ? -1 : summaries.get(0).hashCode()); builder.usev8geopositions(useV8GeoPositions); builder.classes(summaries.stream().map(SummaryClass::getSummaryClassConfig).toList()); - } + } + public void export(String toDirectory) throws IOException { + var builder = new SummaryConfig.Builder(); + getConfig(builder); + export(toDirectory, builder.build()); + } } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java index 799324d26e9..398897c6b78 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; +import java.io.IOException; import java.util.Collections; import java.util.Map; import java.util.logging.Level; @@ -119,11 +120,9 @@ public class SummaryClass extends Derived { return classBuilder; } - @Override - public int hashCode() { return id; } + @Override public int hashCode() { return id; } - @Override - protected String getDerivedName() { return "summary"; } + @Override protected String getDerivedName() { return "summary"; } @Override public String toString() { diff --git a/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java b/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java index b2b2e9d3a14..6992ac70a1a 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java @@ -25,13 +25,14 @@ import com.yahoo.schema.document.SDField; import com.yahoo.schema.processing.TensorFieldProcessor; import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; +import java.io.IOException; import java.util.LinkedHashMap; import java.util.Map; /** * Vertical streaming matcher field specification */ -public class VsmFields extends Derived implements VsmfieldsConfig.Producer { +public class VsmFields extends Derived { private final Map<String, StreamingField> fields=new LinkedHashMap<>(); private final Map<String, StreamingDocumentType> doctypes=new LinkedHashMap<>(); @@ -106,13 +107,18 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { return "vsmfields"; } - @Override public void getConfig(VsmfieldsConfig.Builder vsB) { // Replace vsB.fieldspec(fields.values().stream().map(StreamingField::getFieldSpecConfig).toList()); vsB.documenttype(doctypes.values().stream().map(StreamingDocumentType::getDocTypeConfig).toList()); } + public void export(String toDirectory) throws IOException { + var builder = new VsmfieldsConfig.Builder(); + getConfig(builder); + export(toDirectory, builder.build()); + } + private static boolean isAttributeField(ImmutableSDField field, boolean isStructField, boolean ignoreAttributeAspect) { if (field.doesAttributing()) { return true; diff --git a/config-model/src/main/java/com/yahoo/schema/derived/VsmSummary.java b/config-model/src/main/java/com/yahoo/schema/derived/VsmSummary.java index 248cca33bc6..7507ef95482 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/VsmSummary.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/VsmSummary.java @@ -9,14 +9,18 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; -import java.util.*; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; /** * Vertical streaming matcher summary specification * * @author bratseth */ -public class VsmSummary extends Derived implements VsmsummaryConfig.Producer { +public class VsmSummary extends Derived { private final Map<SummaryField, List<String>> summaryMap = new java.util.LinkedHashMap<>(1); @@ -89,12 +93,10 @@ public class VsmSummary extends Derived implements VsmsummaryConfig.Producer { return ret; } - @Override - public String getDerivedName() { + @Override public String getDerivedName() { return "vsmsummary"; } - @Override public void getConfig(VsmsummaryConfig.Builder vB) { // Replace vB.fieldmap( @@ -105,5 +107,10 @@ public class VsmSummary extends Derived implements VsmsummaryConfig.Producer { ).toList() ); } - + + public void export(String toDirectory) throws IOException { + var builder = new VsmsummaryConfig.Builder(); + getConfig(builder); + export(toDirectory, builder.build()); + } } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaries.java index a05450aa3a0..816e5a074f6 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaries.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaries.java @@ -12,7 +12,7 @@ import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.logging.Level; -import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; +import static com.yahoo.prelude.fastsearch.VespaBackend.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; /** diff --git a/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java index 7e9d79fc858..26107ad2dca 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java @@ -24,7 +24,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.SetVarExpression; import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; import com.yahoo.vespa.model.container.search.QueryProfiles; -import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; +import static com.yahoo.prelude.fastsearch.VespaBackend.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; import java.util.ArrayList; import java.util.List; diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ReferenceFieldsProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/ReferenceFieldsProcessor.java index e425f81c0b4..42457295895 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/ReferenceFieldsProcessor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ReferenceFieldsProcessor.java @@ -10,7 +10,7 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; -import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; +import static com.yahoo.prelude.fastsearch.VespaBackend.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; /** * Class that processes reference fields and removes attribute aspect of such fields from summary. |