diff options
57 files changed, 777 insertions, 615 deletions
diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index d6f0e6d1a1e..933d0ba8010 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -1315,10 +1315,10 @@ dependencies: "@babel/runtime" "^7.13.10" -"@remix-run/router@1.15.2": - version "1.15.2" - resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.15.2.tgz#35726510d332ba5349c6398d13259d5da184553d" - integrity sha512-+Rnav+CaoTE5QJc4Jcwh5toUpnVLKYbpU6Ys0zqbakqbaLQHeglLVHPfxOiQqdNmUy5C2lXz5dwC6tQNX2JW2Q== +"@remix-run/router@1.15.3": + version "1.15.3" + resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.15.3.tgz#d2509048d69dbb72d5389a14945339f1430b2d3c" + integrity sha512-Oy8rmScVrVxWZVOpEF57ovlnhpZ8CCPlnIIumVcV9nFdiSIrus99+Lw78ekXyGvVDlIsFJbSfmSovJUhCWYV3w== "@rollup/rollup-android-arm-eabi@4.12.0": version "4.12.0" @@ -4645,19 +4645,19 @@ react-refresh@^0.14.0: integrity sha512-wViHqhAd8OHeLS/IRMJjTSDHF3U9eWi62F/MledQGPdJGDhodXJ9PBLNGr6WWL7qlH12Mt3TyTpbS+hGXMjCzQ== react-router-dom@^6: - version "6.22.2" - resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.22.2.tgz#8233968a8a576f3006e5549c80f3527d2598fc9c" - integrity sha512-WgqxD2qySEIBPZ3w0sHH+PUAiamDeszls9tzqMPBDA1YYVucTBXLU7+gtRfcSnhe92A3glPnvSxK2dhNoAVOIQ== + version "6.22.3" + resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.22.3.tgz#9781415667fd1361a475146c5826d9f16752a691" + integrity sha512-7ZILI7HjcE+p31oQvwbokjk6OA/bnFxrhJ19n82Ex9Ph8fNAq+Hm/7KchpMGlTgWhUxRHMMCut+vEtNpWpowKw== dependencies: - "@remix-run/router" "1.15.2" - react-router "6.22.2" + "@remix-run/router" "1.15.3" + react-router "6.22.3" -react-router@6.22.2: - version "6.22.2" - resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.22.2.tgz#27e77e4c635a5697693b922d131d773451c98a5b" - integrity sha512-YD3Dzprzpcq+tBMHBS822tCjnWD3iIZbTeSXMY9LPSG541EfoBGyZ3bS25KEnaZjLcmQpw2AVLkFyfgXY8uvcw== +react-router@6.22.3: + version "6.22.3" + resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.22.3.tgz#9d9142f35e08be08c736a2082db5f0c9540a885e" + integrity sha512-dr2eb3Mj5zK2YISHK++foM9w4eBnO23eKnZEDs7c880P6oKbrjz/Svg9+nxqtHQK+oMW4OtjZca0RqPglXxguQ== dependencies: - "@remix-run/router" "1.15.2" + "@remix-run/router" "1.15.3" react-textarea-autosize@8.3.4: version "8.3.4" diff --git a/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java index e747235dc3c..92f7e87209a 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java @@ -12,6 +12,7 @@ import com.yahoo.schema.document.Sorting; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.indexinglanguage.expressions.ToPositionExpression; +import java.io.IOException; import java.util.Collection; import java.util.Collections; import java.util.Iterator; @@ -27,7 +28,7 @@ import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isSupportedCo * * @author bratseth */ -public class AttributeFields extends Derived implements AttributesConfig.Producer { +public class AttributeFields extends Derived { public enum FieldSet {ALL, FAST_ACCESS} @@ -178,12 +179,16 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce return "attributes"; } - @Override public void getConfig(AttributesConfig.Builder builder) { - //TODO This is just to get some exporting tests to work, Should be undone and removed getConfig(builder, FieldSet.ALL, 77777); } + public void export(String toDirectory) throws IOException { + var builder = new AttributesConfig.Builder(); + getConfig(builder); + export(toDirectory, builder.build()); + } + private boolean isAttributeInFieldSet(Attribute attribute, FieldSet fs) { return (fs == FieldSet.ALL) || ((fs == FieldSet.FAST_ACCESS) && attribute.isFastAccess()); } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/Derived.java b/config-model/src/main/java/com/yahoo/schema/derived/Derived.java index 3ed9807b8f1..f4311912584 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/Derived.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/Derived.java @@ -2,7 +2,6 @@ package com.yahoo.schema.derived; import com.yahoo.config.ConfigInstance; -import com.yahoo.config.ConfigInstance.Builder; import com.yahoo.document.Field; import com.yahoo.io.IOUtils; import com.yahoo.schema.Index; @@ -14,7 +13,6 @@ import com.yahoo.text.StringUtilities; import java.io.IOException; import java.io.Writer; -import java.lang.reflect.Method; import java.util.List; /** @@ -22,7 +20,7 @@ import java.util.List; * * @author bratseth */ -public abstract class Derived implements Exportable { +public abstract class Derived { private String name; @@ -88,23 +86,13 @@ public abstract class Derived implements Exportable { return labels ? getName() : String.valueOf(number); } - /** - * Exports this derived configuration to its .cfg file - * in toDirectory - * - * @param toDirectory the directory to export to, or null - * - */ - public void export(String toDirectory) throws IOException { + protected void export(String toDirectory, ConfigInstance cfg) throws IOException { Writer writer = null; try { String fileName = getDerivedName() + ".cfg"; - if (toDirectory != null) - writer = IOUtils.createWriter(toDirectory + "/" + fileName,false); - try { - exportBuilderConfig(writer); - } catch (ReflectiveOperationException | SecurityException | IllegalArgumentException e) { - throw new RuntimeException(e); + if (toDirectory != null) { + writer = IOUtils.createWriter(toDirectory + "/" + fileName, false); + exportConfig(writer, cfg); } } finally { @@ -112,30 +100,10 @@ public abstract class Derived implements Exportable { } } - /** - * Checks what this is a producer of, instantiate that and export to writer - */ - // TODO move to ReflectionUtil, and move that to unexported pkg - private void exportBuilderConfig(Writer writer) throws ReflectiveOperationException, SecurityException, IllegalArgumentException, IOException { - for (Class<?> intf : getClass().getInterfaces()) { - if (ConfigInstance.Producer.class.isAssignableFrom(intf)) { - Class<?> configClass = intf.getEnclosingClass(); - String builderClassName = configClass.getCanonicalName()+"$Builder"; - Class<?> builderClass = Class.forName(builderClassName); - ConfigInstance.Builder builder = (Builder) builderClass.getDeclaredConstructor().newInstance(); - Method getConfig = getClass().getMethod("getConfig", builderClass); - getConfig.invoke(this, builder); - ConfigInstance inst = (ConfigInstance) configClass.getConstructor(builderClass).newInstance(builder); - List<String> payloadL = ConfigInstance.serialize(inst); - String payload = StringUtilities.implodeMultiline(payloadL); - writer.write(payload); - } - } - } - - @Override - public String getFileName() { - return getDerivedName() + ".cfg"; + private void exportConfig(Writer writer, ConfigInstance cfg) throws IOException { + List<String> payloadL = ConfigInstance.serialize(cfg); + String payload = StringUtilities.implodeMultiline(payloadL); + writer.write(payload); } } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java index bc1c097ea4b..e303e878550 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java @@ -12,6 +12,7 @@ import com.yahoo.schema.RankProfileRegistry; import com.yahoo.schema.Schema; import com.yahoo.schema.derived.validation.Validation; import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -126,10 +127,15 @@ public class DerivedConfiguration { } public void exportConstants(String toDirectory) throws IOException { - RankingConstantsConfig.Builder b = new RankingConstantsConfig.Builder(); - rankProfileList.getConfig(b); + var b = new RankingConstantsConfig.Builder() + .constant(rankProfileList.getConstantsConfig()); exportCfg(b.build(), toDirectory + "/" + "ranking-constants.cfg"); } + public void exportOnnxModels(String toDirectory) throws IOException { + var b = new OnnxModelsConfig.Builder() + .model(rankProfileList.getOnnxConfig()); + exportCfg(b.build(), toDirectory + "/" + "onnx-models.cfg"); + } private static void exportCfg(ConfigInstance instance, String fileName) throws IOException { Writer writer = null; diff --git a/config-model/src/main/java/com/yahoo/schema/derived/Exportable.java b/config-model/src/main/java/com/yahoo/schema/derived/Exportable.java deleted file mode 100644 index 983be8bf13a..00000000000 --- a/config-model/src/main/java/com/yahoo/schema/derived/Exportable.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.schema.derived; - -/** - * Classes exportable to configurations - * - * @author bratseth - */ -public interface Exportable { - - /** - * Exports the configuration of this object - * - * - * @param toDirectory the directory to export to, does not write to disk if null - * @throws java.io.IOException if exporting fails, some files may still be created - */ - public void export(String toDirectory) throws java.io.IOException; - - /** - * The (short) name of the exported file - * @return a String with the (short) name of the exported file - */ - public String getFileName(); - -} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedConstants.java b/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedConstants.java index 91ab1ad9172..275fbe82ad4 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedConstants.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedConstants.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; /** @@ -25,7 +26,7 @@ public class FileDistributedConstants { public FileDistributedConstants(FileRegistry fileRegistry, Collection<RankProfile.Constant> constants) { Map<String, DistributableConstant> distributableConstants = new LinkedHashMap<>(); for (var constant : constants) { - if ( ! constant.valuePath().isPresent()) continue; + if (constant.valuePath().isEmpty()) continue; var distributableConstant = new DistributableConstant(constant.name().simpleArgument().get(), constant.type(), @@ -41,23 +42,21 @@ public class FileDistributedConstants { /** Returns a read-only map of the constants in this indexed by name. */ public Map<String, DistributableConstant> asMap() { return constants; } - public void getConfig(RankingConstantsConfig.Builder builder) { - for (var constant : constants.values()) { - builder.constant(new RankingConstantsConfig.Constant.Builder() - .name(constant.getName()) - .fileref(constant.getFileReference()) - .type(constant.getType())); - } + private static RankingConstantsConfig.Constant.Builder toConfig(DistributableConstant constant) { + return new RankingConstantsConfig.Constant.Builder() + .name(constant.getName()) + .fileref(constant.getFileReference()) + .type(constant.getType()); + } + + public List<RankingConstantsConfig.Constant.Builder> getConfig() { + return constants.values().stream().map(FileDistributedConstants::toConfig).toList(); } public static class DistributableConstant extends DistributableResource { private final TensorType tensorType; - public DistributableConstant(String name, TensorType type, String fileName) { - this(name, type, fileName, PathType.FILE); - } - public DistributableConstant(String name, TensorType type, String fileName, PathType pathType) { super(name, fileName, pathType); this.tensorType = type; diff --git a/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedOnnxModels.java b/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedOnnxModels.java index 1cc33cc4180..6e50e3c094c 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedOnnxModels.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedOnnxModels.java @@ -5,9 +5,11 @@ import com.yahoo.config.application.api.FileRegistry; import com.yahoo.schema.OnnxModel; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.logging.Logger; @@ -16,13 +18,10 @@ import java.util.logging.Logger; * * @author bratseth */ -public class FileDistributedOnnxModels extends Derived implements OnnxModelsConfig.Producer { +public class FileDistributedOnnxModels { private static final Logger log = Logger.getLogger(FileDistributedOnnxModels.class.getName()); - @Override - public String getDerivedName() { return "onnx-models"; } - private final Map<String, OnnxModel> models; public FileDistributedOnnxModels(FileRegistry fileRegistry, Collection<OnnxModel> models) { @@ -47,30 +46,36 @@ public class FileDistributedOnnxModels extends Derived implements OnnxModelsConf public Map<String, OnnxModel> asMap() { return models; } - public void getConfig(OnnxModelsConfig.Builder builder) { + private static OnnxModelsConfig.Model.Builder toConfig(OnnxModel model) { + OnnxModelsConfig.Model.Builder builder = new OnnxModelsConfig.Model.Builder(); + builder.dry_run_on_setup(true); + builder.name(model.getName()); + builder.fileref(model.getFileReference()); + model.getInputMap().forEach((name, source) -> builder.input(new OnnxModelsConfig.Model.Input.Builder().name(name).source(source))); + model.getOutputMap().forEach((name, as) -> builder.output(new OnnxModelsConfig.Model.Output.Builder().name(name).as(as))); + if (model.getStatelessExecutionMode().isPresent()) + builder.stateless_execution_mode(model.getStatelessExecutionMode().get()); + if (model.getStatelessInterOpThreads().isPresent()) + builder.stateless_interop_threads(model.getStatelessInterOpThreads().get()); + if (model.getStatelessIntraOpThreads().isPresent()) + builder.stateless_intraop_threads(model.getStatelessIntraOpThreads().get()); + if (model.getGpuDevice().isPresent()) { + builder.gpu_device(model.getGpuDevice().get().deviceNumber()); + builder.gpu_device_required(model.getGpuDevice().get().required()); + } + return builder; + } + + public List<OnnxModelsConfig.Model.Builder> getConfig() { + List<OnnxModelsConfig.Model.Builder> cfgList = new ArrayList<>(); for (OnnxModel model : models.values()) { if ("".equals(model.getFileReference())) log.warning("Illegal file reference " + model); // Let tests pass ... we should find a better way else { - OnnxModelsConfig.Model.Builder modelBuilder = new OnnxModelsConfig.Model.Builder(); - modelBuilder.dry_run_on_setup(true); - modelBuilder.name(model.getName()); - modelBuilder.fileref(model.getFileReference()); - model.getInputMap().forEach((name, source) -> modelBuilder.input(new OnnxModelsConfig.Model.Input.Builder().name(name).source(source))); - model.getOutputMap().forEach((name, as) -> modelBuilder.output(new OnnxModelsConfig.Model.Output.Builder().name(name).as(as))); - if (model.getStatelessExecutionMode().isPresent()) - modelBuilder.stateless_execution_mode(model.getStatelessExecutionMode().get()); - if (model.getStatelessInterOpThreads().isPresent()) - modelBuilder.stateless_interop_threads(model.getStatelessInterOpThreads().get()); - if (model.getStatelessIntraOpThreads().isPresent()) - modelBuilder.stateless_intraop_threads(model.getStatelessIntraOpThreads().get()); - if (model.getGpuDevice().isPresent()) { - modelBuilder.gpu_device(model.getGpuDevice().get().deviceNumber()); - modelBuilder.gpu_device_required(model.getGpuDevice().get().required()); - } - builder.model(modelBuilder); + cfgList.add(toConfig(model)); } } + return cfgList; } } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java b/config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java index a781adcefb7..e5e28ccea27 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java @@ -9,6 +9,7 @@ import com.yahoo.schema.document.ImportedComplexField; import com.yahoo.schema.document.ImportedField; import com.yahoo.vespa.config.search.ImportedFieldsConfig; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -22,7 +23,7 @@ import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isMapOfSimple * * @author geirst */ -public class ImportedFields extends Derived implements ImportedFieldsConfig.Producer { +public class ImportedFields extends Derived { private Optional<com.yahoo.schema.document.ImportedFields> importedFields = Optional.empty(); @@ -40,7 +41,6 @@ public class ImportedFields extends Derived implements ImportedFieldsConfig.Prod return "imported-fields"; } - @Override public void getConfig(ImportedFieldsConfig.Builder builder) { // Replace if (importedFields.isPresent()) { @@ -50,6 +50,12 @@ public class ImportedFields extends Derived implements ImportedFieldsConfig.Prod } } + public void export(String toDirectory) throws IOException { + var builder = new ImportedFieldsConfig.Builder(); + getConfig(builder); + export(toDirectory, builder.build()); + } + private static boolean isNestedFieldName(String fieldName) { return fieldName.indexOf('.') != -1; } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java index b0cf4a0013c..c1b698df55f 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java @@ -24,6 +24,7 @@ import com.yahoo.schema.processing.NGramMatch; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.search.config.IndexInfoConfig; +import java.io.IOException; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -34,7 +35,7 @@ import java.util.Set; * * @author bratseth */ -public class IndexInfo extends Derived implements IndexInfoConfig.Producer { +public class IndexInfo extends Derived { private static final String CMD_ATTRIBUTE = "attribute"; private static final String CMD_DEFAULT_POSITION = "default-position"; @@ -314,7 +315,6 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { return true; } - @Override public void getConfig(IndexInfoConfig.Builder builder) { // Append IndexInfoConfig.Indexinfo.Builder iiB = new IndexInfoConfig.Indexinfo.Builder(); @@ -335,6 +335,12 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { builder.indexinfo(iiB); } + public void export(String toDirectory) throws IOException { + var builder = new IndexInfoConfig.Builder(); + getConfig(builder); + export(toDirectory, builder.build()); + } + // TODO: Move this to the FieldSetSettings processor (and rename it) as that already has to look at this. private void addFieldSetCommands(IndexInfoConfig.Indexinfo.Builder iiB, FieldSet fieldSet) { for (String qc : fieldSet.queryCommands()) { diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java index 1c22e80e579..34ed9f2f60c 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java @@ -12,6 +12,7 @@ import com.yahoo.schema.document.FieldSet; import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.vespa.config.search.IndexschemaConfig; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; @@ -24,7 +25,7 @@ import java.util.Map; * * @author geirst */ -public class IndexSchema extends Derived implements IndexschemaConfig.Producer { +public class IndexSchema extends Derived { private final List<IndexField> fields = new ArrayList<>(); private final Map<String, FieldCollection> collections = new LinkedHashMap<>(); @@ -131,13 +132,18 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { return fsB; } - @Override public void getConfig(IndexschemaConfig.Builder icB) { // Replace icB.indexfield(fields.stream().map(IndexSchema::createIndexFieldConfig).toList()); icB.fieldset(fieldSets.values().stream().map(IndexSchema::createFieldSetConfig).toList()); } + public void export(String toDirectory) throws IOException { + var builder = new IndexschemaConfig.Builder(); + getConfig(builder); + export(toDirectory, builder.build()); + } + static List<Field> flattenField(Field field) { DataType fieldType = field.getDataType(); if (fieldType.getPrimitiveType() != null){ diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java index 7245e3f266d..60d27c617f1 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; import com.yahoo.vespa.indexinglanguage.expressions.ClearStateExpression; import com.yahoo.vespa.indexinglanguage.expressions.Expression; -import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; import com.yahoo.vespa.indexinglanguage.expressions.GuardExpression; import com.yahoo.vespa.indexinglanguage.expressions.InputExpression; import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; @@ -22,6 +21,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; import com.yahoo.vespa.indexinglanguage.expressions.TokenizeExpression; import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -35,7 +35,7 @@ import java.util.Set; * * @author bratseth */ -public final class IndexingScript extends Derived implements IlscriptsConfig.Producer { +public final class IndexingScript extends Derived { private final List<String> docFields = new ArrayList<>(); private final List<Expression> expressions = new ArrayList<>(); @@ -93,7 +93,6 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro return "ilscripts"; } - @Override public void getConfig(IlscriptsConfig.Builder configBuilder) { // Append IlscriptsConfig.Ilscript.Builder ilscriptBuilder = new IlscriptsConfig.Ilscript.Builder(); @@ -103,6 +102,12 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro configBuilder.ilscript(ilscriptBuilder); } + public void export(String toDirectory) throws IOException { + var builder = new IlscriptsConfig.Builder(); + getConfig(builder); + export(toDirectory, builder.build()); + } + private static class DropTokenize extends ExpressionConverter { @Override protected boolean shouldConvert(Expression exp) { diff --git a/config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java b/config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java index a39fa642f34..58e77cedaf1 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.config.search.summary.JuniperrcConfig; +import java.io.IOException; import java.util.ArrayList; import java.util.Set; import java.util.stream.Collectors; @@ -15,7 +16,7 @@ import java.util.stream.Collectors; * * @author Simon Thoresen Hult */ -public class Juniperrc extends Derived implements JuniperrcConfig.Producer { +public class Juniperrc extends Derived { private static final int Mb = 1024 * 1024; @@ -43,8 +44,13 @@ public class Juniperrc extends Derived implements JuniperrcConfig.Producer { } } - @Override - protected String getDerivedName() { return "juniperrc"; } + public void export(String toDirectory) throws IOException { + var builder = new JuniperrcConfig.Builder(); + getConfig(builder); + export(toDirectory, builder.build()); + } + + @Override protected String getDerivedName() { return "juniperrc"; } private static JuniperrcConfig.Override.Builder createOverride(String name) { return new JuniperrcConfig.Override.Builder() @@ -55,7 +61,6 @@ public class Juniperrc extends Derived implements JuniperrcConfig.Producer { .surround_max(64*Mb); } - @Override public void getConfig(JuniperrcConfig.Builder builder) { // Replace if (!boldingFields.isEmpty()) { diff --git a/config-model/src/main/java/com/yahoo/schema/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/schema/derived/RankProfileList.java index 0a419768aaf..b08629d68a3 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/RankProfileList.java @@ -4,6 +4,7 @@ package com.yahoo.schema.derived; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.schema.RankingExpressionBody; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.schema.LargeRankingExpressions; import com.yahoo.schema.OnnxModel; @@ -16,6 +17,7 @@ import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -32,7 +34,7 @@ import java.util.concurrent.Future; * * @author bratseth */ -public class RankProfileList extends Derived implements RankProfilesConfig.Producer { +public class RankProfileList extends Derived { private final Map<String, RawRankProfile> rankProfiles; private final FileDistributedConstants constants; @@ -190,32 +192,32 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ public FileDistributedConstants constants() { return constants; } public FileDistributedOnnxModels getOnnxModels() { return onnxModels; } - @Override - public String getDerivedName() { return "rank-profiles"; } + @Override public String getDerivedName() { return "rank-profiles"; } - @Override - public void export(String toDirectory) throws java.io.IOException { - super.export(toDirectory); - onnxModels.export(toDirectory); + public void export(String toDirectory) throws IOException { + export(toDirectory, new RankProfilesConfig.Builder().rankprofile(getRankProfilesConfig()).build()); } - @Override - public void getConfig(RankProfilesConfig.Builder builder) { - for (RawRankProfile rank : rankProfiles.values() ) { - rank.getConfig(builder); - } + public List<RankProfilesConfig.Rankprofile.Builder> getRankProfilesConfig() { + return rankProfiles.values().stream().map(RawRankProfile::getConfig).toList(); + } + + private static RankingExpressionsConfig.Expression.Builder toConfig(RankingExpressionBody expr) { + return new RankingExpressionsConfig.Expression.Builder() + .name(expr.getName()) + .fileref(expr.getFileReference()); } - public void getConfig(RankingExpressionsConfig.Builder builder) { - largeRankingExpressions.expressions().forEach((expr) -> builder.expression.add(new RankingExpressionsConfig.Expression.Builder().name(expr.getName()).fileref(expr.getFileReference()))); + public List<RankingExpressionsConfig.Expression.Builder> getExpressionsConfig() { + return largeRankingExpressions.expressions().stream().map(RankProfileList::toConfig).toList(); } - public void getConfig(RankingConstantsConfig.Builder builder) { - constants.getConfig(builder); + public List<RankingConstantsConfig.Constant.Builder> getConstantsConfig() { + return constants.getConfig(); } - public void getConfig(OnnxModelsConfig.Builder builder) { - onnxModels.getConfig(builder); + public List<OnnxModelsConfig.Model.Builder> getOnnxConfig() { + return onnxModels.getConfig(); } } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java index db76d6397fc..b057624f055 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java @@ -43,7 +43,7 @@ import java.util.Set; * * @author bratseth */ -public class RawRankProfile implements RankProfilesConfig.Producer { +public class RawRankProfile { /** A reusable compressor with default settings */ private static final Compressor compressor = new Compressor(); @@ -130,12 +130,11 @@ public class RawRankProfile implements RankProfilesConfig.Producer { */ public List<Pair<String, String>> configProperties() { return decompress(compressedProperties); } - @Override - public void getConfig(RankProfilesConfig.Builder builder) { + public RankProfilesConfig.Rankprofile.Builder getConfig() { RankProfilesConfig.Rankprofile.Builder b = new RankProfilesConfig.Rankprofile.Builder().name(getName()); getRankProperties(b); buildNormalizers(b); - builder.rankprofile(b); + return b; } @Override diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java index fb6d7fb70ec..19a045ac444 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java @@ -21,6 +21,7 @@ import com.yahoo.schema.RankProfileRegistry; import com.yahoo.schema.Schema; import com.yahoo.searchlib.rankingexpression.Reference; +import java.io.IOException; import java.util.Collection; import java.util.Collections; import java.util.Iterator; @@ -32,7 +33,7 @@ import java.util.Map; * * @author bratseth */ -public final class SchemaInfo extends Derived implements SchemaInfoConfig.Producer { +public final class SchemaInfo extends Derived { private final Schema schema; @@ -63,7 +64,6 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc return rankProfileInfos; } - @Override public void getConfig(SchemaInfoConfig.Builder builder) { // Append var schemaBuilder = new SchemaInfoConfig.Schema.Builder(); @@ -75,6 +75,12 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc builder.schema(schemaBuilder); } + public void export(String toDirectory) throws IOException { + var builder = new SchemaInfoConfig.Builder(); + getConfig(builder); + export(toDirectory, builder.build()); + } + private void addFieldsConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) { for (var field : schema.allFieldsList()) { addFieldConfig(field, schemaBuilder); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/Summaries.java b/config-model/src/main/java/com/yahoo/schema/derived/Summaries.java index 51c648aa1be..1f0c6f7e761 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/Summaries.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/Summaries.java @@ -7,6 +7,7 @@ import com.yahoo.schema.Schema; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.config.search.SummaryConfig; +import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -15,7 +16,7 @@ import java.util.List; * * @author bratseth */ -public class Summaries extends Derived implements SummaryConfig.Producer { +public class Summaries extends Derived { private final boolean useV8GeoPositions; private final List<SummaryClass> summaries; @@ -36,15 +37,18 @@ public class Summaries extends Derived implements SummaryConfig.Producer { public List<SummaryClass> asList() { return summaries; } - @Override - protected String getDerivedName() { return "summary"; } + @Override protected String getDerivedName() { return "summary"; } - @Override public void getConfig(SummaryConfig.Builder builder) { // Replace builder.defaultsummaryid(summaries.isEmpty() ? -1 : summaries.get(0).hashCode()); builder.usev8geopositions(useV8GeoPositions); builder.classes(summaries.stream().map(SummaryClass::getSummaryClassConfig).toList()); - } + } + public void export(String toDirectory) throws IOException { + var builder = new SummaryConfig.Builder(); + getConfig(builder); + export(toDirectory, builder.build()); + } } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java index 799324d26e9..398897c6b78 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; +import java.io.IOException; import java.util.Collections; import java.util.Map; import java.util.logging.Level; @@ -119,11 +120,9 @@ public class SummaryClass extends Derived { return classBuilder; } - @Override - public int hashCode() { return id; } + @Override public int hashCode() { return id; } - @Override - protected String getDerivedName() { return "summary"; } + @Override protected String getDerivedName() { return "summary"; } @Override public String toString() { diff --git a/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java b/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java index b2b2e9d3a14..6992ac70a1a 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java @@ -25,13 +25,14 @@ import com.yahoo.schema.document.SDField; import com.yahoo.schema.processing.TensorFieldProcessor; import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; +import java.io.IOException; import java.util.LinkedHashMap; import java.util.Map; /** * Vertical streaming matcher field specification */ -public class VsmFields extends Derived implements VsmfieldsConfig.Producer { +public class VsmFields extends Derived { private final Map<String, StreamingField> fields=new LinkedHashMap<>(); private final Map<String, StreamingDocumentType> doctypes=new LinkedHashMap<>(); @@ -106,13 +107,18 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { return "vsmfields"; } - @Override public void getConfig(VsmfieldsConfig.Builder vsB) { // Replace vsB.fieldspec(fields.values().stream().map(StreamingField::getFieldSpecConfig).toList()); vsB.documenttype(doctypes.values().stream().map(StreamingDocumentType::getDocTypeConfig).toList()); } + public void export(String toDirectory) throws IOException { + var builder = new VsmfieldsConfig.Builder(); + getConfig(builder); + export(toDirectory, builder.build()); + } + private static boolean isAttributeField(ImmutableSDField field, boolean isStructField, boolean ignoreAttributeAspect) { if (field.doesAttributing()) { return true; diff --git a/config-model/src/main/java/com/yahoo/schema/derived/VsmSummary.java b/config-model/src/main/java/com/yahoo/schema/derived/VsmSummary.java index 248cca33bc6..7507ef95482 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/VsmSummary.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/VsmSummary.java @@ -9,14 +9,18 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; -import java.util.*; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; /** * Vertical streaming matcher summary specification * * @author bratseth */ -public class VsmSummary extends Derived implements VsmsummaryConfig.Producer { +public class VsmSummary extends Derived { private final Map<SummaryField, List<String>> summaryMap = new java.util.LinkedHashMap<>(1); @@ -89,12 +93,10 @@ public class VsmSummary extends Derived implements VsmsummaryConfig.Producer { return ret; } - @Override - public String getDerivedName() { + @Override public String getDerivedName() { return "vsmsummary"; } - @Override public void getConfig(VsmsummaryConfig.Builder vB) { // Replace vB.fieldmap( @@ -105,5 +107,10 @@ public class VsmSummary extends Derived implements VsmsummaryConfig.Producer { ).toList() ); } - + + public void export(String toDirectory) throws IOException { + var builder = new VsmsummaryConfig.Builder(); + getConfig(builder); + export(toDirectory, builder.build()); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index ffd346f704f..62979404025 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -55,25 +55,25 @@ public class ContainerModelEvaluation implements @Override public void getConfig(RankProfilesConfig.Builder builder) { - rankProfileList.getConfig(builder); + builder.rankprofile(rankProfileList.getRankProfilesConfig()); } @Override public void getConfig(RankingConstantsConfig.Builder builder) { - rankProfileList.getConfig(builder); + builder.constant(rankProfileList.getConstantsConfig()); } @Override public void getConfig(OnnxModelsConfig.Builder builder) { if (onnxModels != null) { - onnxModels.getConfig(builder); + builder.model(onnxModels.getConfig()); } else { - rankProfileList.getConfig(builder); + builder.model(rankProfileList.getOnnxConfig()); } } public void getConfig(RankingExpressionsConfig.Builder builder) { - rankProfileList.getConfig(builder); + builder.expression(rankProfileList.getExpressionsConfig()); } public static Handler getHandler() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java index 800e766706b..bc936dc7441 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java @@ -115,27 +115,27 @@ public class ModelsEvaluatorTester { } private static RankProfilesConfig getRankProfilesConfig(RankProfileList rankProfileList) { - RankProfilesConfig.Builder builder = new RankProfilesConfig.Builder(); - rankProfileList.getConfig(builder); - return builder.build(); + return new RankProfilesConfig.Builder() + .rankprofile(rankProfileList.getRankProfilesConfig()) + .build(); } private static RankingConstantsConfig getRankingConstantConfig(RankProfileList rankProfileList) { - RankingConstantsConfig.Builder builder = new RankingConstantsConfig.Builder(); - rankProfileList.getConfig(builder); - return builder.build(); + return new RankingConstantsConfig.Builder() + .constant(rankProfileList.getConstantsConfig()) + .build(); } private static RankingExpressionsConfig getRankingExpressionsConfig(RankProfileList rankProfileList) { - RankingExpressionsConfig.Builder builder = new RankingExpressionsConfig.Builder(); - rankProfileList.getConfig(builder); - return builder.build(); + return new RankingExpressionsConfig.Builder() + .expression(rankProfileList.getExpressionsConfig()) + .build(); } private static OnnxModelsConfig getOnnxModelsConfig(RankProfileList rankProfileList) { - OnnxModelsConfig.Builder builder = new OnnxModelsConfig.Builder(); - rankProfileList.getConfig(builder); - return builder.build(); + return new OnnxModelsConfig.Builder() + .model(rankProfileList.getOnnxConfig()) + .build(); } private static FileAcquirer createFileAcquirer(MockFileRegistry fileRegistry, File appDir) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java index 334f8d4314a..ed571266f1b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java @@ -68,14 +68,20 @@ public class DocumentDatabase extends AnyConfigProducer implements // These methods append as multiple databases join config => TODO will loose information - not good @Override public void getConfig(AttributesConfig.Builder builder) { derivedCfg.getConfig(builder); } - @Override public void getConfig(RankProfilesConfig.Builder builder) { derivedCfg.getRankProfileList().getConfig(builder); } - - // These methods append, TODO unknown usage and consequences - @Override public void getConfig(RankingExpressionsConfig.Builder builder) { derivedCfg.getRankProfileList().getConfig(builder); } - @Override public void getConfig(RankingConstantsConfig.Builder builder) { derivedCfg.getRankProfileList().getConfig(builder); } - @Override public void getConfig(OnnxModelsConfig.Builder builder) { derivedCfg.getRankProfileList().getConfig(builder); } // Below methods will replace config completely + @Override public void getConfig(OnnxModelsConfig.Builder builder) { + builder.model(derivedCfg.getRankProfileList().getOnnxConfig()); + } + @Override public void getConfig(RankingExpressionsConfig.Builder builder) { + builder.expression(derivedCfg.getRankProfileList().getExpressionsConfig()); + } + @Override public void getConfig(RankingConstantsConfig.Builder builder) { + builder.constant(derivedCfg.getRankProfileList().getConstantsConfig()); + } + @Override public void getConfig(RankProfilesConfig.Builder builder) { + builder.rankprofile(derivedCfg.getRankProfileList().getRankProfilesConfig()); + } @Override public void getConfig(IndexschemaConfig.Builder builder) { derivedCfg.getIndexSchema().getConfig(builder); } @Override public void getConfig(JuniperrcConfig.Builder builder) { derivedCfg.getJuniperrc().getConfig(builder); } @Override public void getConfig(SummaryConfig.Builder builder) { derivedCfg.getSummaries().getConfig(builder); } diff --git a/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java index 7686289f11c..7d7ed2b23eb 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java @@ -70,6 +70,7 @@ public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase { DerivedConfiguration.exportDocuments(new DocumentTypes().produce(builder.getModel(), new DocumenttypesConfig.Builder()), path); DerivedConfiguration.exportQueryProfiles(builder.getQueryProfileRegistry(), path); config.exportConstants(path); + config.exportOnnxModels(path); return config; } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/secret/Key.java b/container-disc/src/main/java/com/yahoo/container/jdisc/secret/Key.java new file mode 100644 index 00000000000..3de482b9cc6 --- /dev/null +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/secret/Key.java @@ -0,0 +1,41 @@ +package com.yahoo.container.jdisc.secret; + +import java.util.Objects; + +public class Key { + + private final String keyGroup; + private final String keyName; + + public Key(String keyGroup, String keyName) { + this.keyGroup = keyGroup; + this.keyName = keyName; + } + + public String keyGroup() { + return keyGroup; + } + + public String keyName() { + return keyName; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Key that = (Key) o; + if ( ! (that.keyGroup.equals(keyGroup))) return false; + if ( ! (that.keyName.equals(keyName))) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(keyGroup, keyName); + } + + @Override + public String toString() { return "key group: " + keyGroup + ", key name: " + keyName; } + +} diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/secret/Secret.java b/container-disc/src/main/java/com/yahoo/container/jdisc/secret/Secret.java new file mode 100644 index 00000000000..fef0ba804eb --- /dev/null +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/secret/Secret.java @@ -0,0 +1,58 @@ +package com.yahoo.container.jdisc.secret; + +import com.yahoo.security.YBase64; +import com.yahoo.text.Utf8; + +import java.util.Arrays; +import java.util.Objects; + +public class Secret { + + private final Key key; + private final byte[] secret; + private final int version; + + public Secret(Key key, byte[] secret, int version) { + this.key = key; + this.secret = secret; + this.version = version; + } + + public String keyGroup() { + return key.keyGroup(); + } + + public String keyName() { + return key.keyName(); + } + + public byte[] secret() { + return secret; + } + + public String secretAsString() { return Utf8.toString(secret); } + + /** @return secret value for keys that are auto-rotated by CKMS */ + public byte[] secretAsYbase64Decoded() { return YBase64.decode(secret); } + + public int version() { + return version; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Secret that = (Secret) o; + if ( ! (that.key.equals(key))) return false; + if ( ! (Arrays.equals(that.secret, secret))) return false; + if (that.version != (version)) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(key, version, Arrays.hashCode(secret)); + } + +} diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/secret/TypedSecretStore.java b/container-disc/src/main/java/com/yahoo/container/jdisc/secret/TypedSecretStore.java new file mode 100644 index 00000000000..5bb00e836f5 --- /dev/null +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/secret/TypedSecretStore.java @@ -0,0 +1,18 @@ +package com.yahoo.container.jdisc.secret; + +import com.yahoo.container.jdisc.secretstore.SecretStore; + +import java.util.List; + +public interface TypedSecretStore extends SecretStore { + + Secret getSecret(Key key); + + Secret getSecret(Key key, int version); + + /** Lists the existing versions of this secret (nonnegative integers) */ + default List<Secret> listSecretVersions(Key key) { + throw new UnsupportedOperationException("Secret store does not support listing versions"); + } + +} diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/secret/package-info.java b/container-disc/src/main/java/com/yahoo/container/jdisc/secret/package-info.java new file mode 100644 index 00000000000..c80c6e66066 --- /dev/null +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/secret/package-info.java @@ -0,0 +1,6 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +@ExportPackage +package com.yahoo.container.jdisc.secret; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index f481d58d334..1fe0835306e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -29,9 +29,11 @@ import com.yahoo.yolean.Exceptions; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.ExecutionException; @@ -177,7 +179,7 @@ public class ClusterSearcher extends Searcher { public Result search(Query query, Execution execution) { validateQueryTimeout(query); validateQueryCache(query); - Searcher searcher = server; + var searcher = server; if (searcher == null) { return new Result(query, ErrorMessage.createNoBackendsInService("Could not search")); } @@ -185,17 +187,17 @@ public class ClusterSearcher extends Searcher { return new Result(query, ErrorMessage.createTimeout("No time left for searching")); } - return doSearch(searcher, query, execution); + return doSearch(searcher, query); } @Override public void fill(com.yahoo.search.Result result, String summaryClass, Execution execution) { Query query = result.getQuery(); - Searcher searcher = server; + VespaBackEndSearcher searcher = server; if (searcher != null) { if (query.getTimeLeft() > 0) { - searcher.fill(result, summaryClass, execution); + searcher.fill(result, summaryClass); } else { if (result.hits().getErrorHit() == null) { result.hits().addError(ErrorMessage.createTimeout("No time left to get summaries, query timeout was " + @@ -230,22 +232,21 @@ public class ClusterSearcher extends Searcher { query.getRanking().setQueryCache(false); } - private Result doSearch(Searcher searcher, Query query, Execution execution) { + private Result doSearch(VespaBackEndSearcher searcher, Query query) { if (schemas.size() > 1) { - return searchMultipleDocumentTypes(searcher, query, execution); + return searchMultipleDocumentTypes(searcher, query); } else { - String docType = schemas.iterator().next(); - query.getModel().setRestrict(docType); - return perSchemaSearch(searcher, query, execution); + String schema = schemas.iterator().next(); + query.getModel().setRestrict(schema); + return perSchemaSearch(searcher, schema, query); } } - private Result perSchemaSearch(Searcher searcher, Query query, Execution execution) { + private Result perSchemaSearch(VespaBackEndSearcher searcher, String schema, Query query) { Set<String> restrict = query.getModel().getRestrict(); if (restrict.size() != 1) { throw new IllegalStateException("perSchemaSearch must always be called with 1 schema, got: " + restrict.size()); } - String schema = restrict.iterator().next(); int rerankCount = globalPhaseRanker != null ? globalPhaseRanker.getRerankCount(query, schema) : 0; boolean useGlobalPhase = rerankCount > 0; final int wantOffset = query.getOffset(); @@ -257,7 +258,7 @@ public class ClusterSearcher extends Searcher { query.setOffset(0); query.setHits(useHits); } - Result result = searcher.search(query, execution); + Result result = searcher.search(schema, query); if (useGlobalPhase) { globalPhaseRanker.rerankHits(query, result, schema); result.hits().trim(wantOffset, wantHits); @@ -284,16 +285,17 @@ public class ClusterSearcher extends Searcher { } } - private Result searchMultipleDocumentTypes(Searcher searcher, Query query, Execution execution) { + private Result searchMultipleDocumentTypes(VespaBackEndSearcher searcher, Query query) { Set<String> schemas = resolveSchemas(query); - List<Query> queries = createQueries(query, schemas); - if (queries.size() == 1) { - return perSchemaSearch(searcher, queries.get(0), execution); + Map<String, Query> schemaQueries = createQueries(query, schemas); + if (schemaQueries.size() == 1) { + var entry = schemaQueries.entrySet().iterator().next(); + return perSchemaSearch(searcher, entry.getKey(), entry.getValue()); } else { Result mergedResult = new Result(query); - List<FutureTask<Result>> pending = new ArrayList<>(queries.size()); - for (Query q : queries) { - FutureTask<Result> task = new FutureTask<>(() -> perSchemaSearch(searcher, q, execution)); + List<FutureTask<Result>> pending = new ArrayList<>(schemaQueries.size()); + for (var entry : schemaQueries.entrySet()) { + FutureTask<Result> task = new FutureTask<>(() -> perSchemaSearch(searcher, entry.getKey(), entry.getValue())); try { executor.execute(task); pending.add(task); @@ -309,7 +311,7 @@ public class ClusterSearcher extends Searcher { if (query.getOffset() > 0 || query.getHits() < mergedResult.hits().size()) { if (mergedResult.getHitOrderer() != null) { // Make sure we have the necessary data for sorting - searcher.fill(mergedResult, VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS, execution); + searcher.fill(mergedResult, VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); } mergedResult.hits().trim(query.getOffset(), query.getHits()); query.setOffset(0); // Needed when doing a trim @@ -351,22 +353,24 @@ public class ClusterSearcher extends Searcher { return retval; } - private List<Query> createQueries(Query query, Set<String> docTypes) { + private Map<String, Query> createQueries(Query query, Set<String> schemas) { query.getModel().getQueryTree(); // performance: parse query before cloning such that it is only done once - List<Query> retval = new ArrayList<>(docTypes.size()); - if (docTypes.size() == 1) { - query.getModel().setRestrict(docTypes.iterator().next()); - retval.add(query); - } else if ( ! docTypes.isEmpty() ) { - for (String docType : docTypes) { + if (schemas.size() == 1) { + String schema = schemas.iterator().next(); + query.getModel().setRestrict(schema); + return Map.of(schema, query); + } else if ( ! schemas.isEmpty() ) { + var schemaQueries = new HashMap<String, Query>(); + for (String schema : schemas) { Query q = query.clone(); q.setOffset(0); q.setHits(query.getOffset() + query.getHits()); - q.getModel().setRestrict(docType); - retval.add(q); + q.getModel().setRestrict(schema); + schemaQueries.put(schema, q); } + return schemaQueries; } - return retval; + return Map.of(); } @Override diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java index ddb36e007d3..85b7770a8b2 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -1,8 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; -import com.yahoo.prelude.Ping; -import com.yahoo.prelude.Pong; import com.yahoo.prelude.querytransform.QueryRewrite; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -16,7 +14,6 @@ import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; import com.yahoo.search.schema.SchemaInfo; -import com.yahoo.search.searchchain.Execution; import java.io.IOException; import java.util.Optional; @@ -61,14 +58,6 @@ public class FastSearcher extends VespaBackEndSearcher { this.dispatcher = dispatcher; } - /** - * Pings the backend. Does not propagate to other searchers. - */ - @Override - public Pong ping(Ping ping, Execution execution) { - throw new IllegalStateException("This ping should not have been called."); - } - @Override protected void transformQuery(Query query) { QueryRewrite.rewriteSddocname(query); @@ -83,11 +72,11 @@ public class FastSearcher extends VespaBackEndSearcher { } @Override - public Result doSearch2(Query query, Execution execution) { + public Result doSearch2(String schema, Query query) { if (dispatcher.allGroupsHaveSize1()) forceSinglePassGrouping(query); try (SearchInvoker invoker = getSearchInvoker(query)) { - Result result = invoker.search(query, execution); + Result result = invoker.search(query); injectSource(result.hits()); if (query.properties().getBoolean(Ranking.RANKFEATURES, false)) { @@ -97,7 +86,7 @@ public class FastSearcher extends VespaBackEndSearcher { // contain the data we need. If we fetch the default // one we end up fetching docsums twice unless the // user also requested the default one. - fill(result, query.getPresentation().getSummary(), execution); // ARGH + fill(result, query.getPresentation().getSummary()); // ARGH } return result; } catch (TimeoutException e) { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index 2e635d21f01..340262ed186 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -12,13 +12,11 @@ import com.yahoo.prelude.querytransform.QueryRewrite; import com.yahoo.protect.Validator; import com.yahoo.search.Query; import com.yahoo.search.Result; -import com.yahoo.search.cluster.PingableSearcher; import com.yahoo.search.schema.RankProfile; import com.yahoo.search.grouping.vespa.GroupingExecutor; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; import com.yahoo.search.schema.SchemaInfo; -import com.yahoo.search.searchchain.Execution; import com.yahoo.searchlib.aggregation.Grouping; import java.util.ArrayList; @@ -27,14 +25,13 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.logging.Logger; /** * Superclass for backend searchers. * * @author baldersheim */ -public abstract class VespaBackEndSearcher extends PingableSearcher { +public abstract class VespaBackEndSearcher { /** for vespa-internal use only; consider renaming the summary class */ public static final String SORTABLE_ATTRIBUTES_SUMMARY_CLASS = "attributeprefetch"; @@ -62,16 +59,13 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { /** Sets default document summary class. Default is null */ private void setDefaultDocsumClass(String docsumClass) { defaultDocsumClass = docsumClass; } - public final Logger getLogger() { return super.getLogger(); } - /** * Searches a search cluster * This is an endpoint - searchers will never propagate the search to any nested searcher. * * @param query the query to search - * @param execution the query execution context */ - protected abstract Result doSearch2(Query query, Execution execution); + protected abstract Result doSearch2(String schema, Query query); protected abstract void doPartialFill(Result result, String summaryClass); @@ -154,8 +148,7 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { protected void transformQuery(Query query) { } - @Override - public Result search(Query query, Execution execution) { + public Result search(String schema, Query query) { // query root should not be null here Item root = query.getModel().getQueryTree().getRoot(); if (root == null || root instanceof NullItem) { @@ -183,7 +176,7 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { if (root == null || root instanceof NullItem) // root can become null after resolving and transformation? return new Result(query); - Result result = doSearch2(query, execution); + Result result = doSearch2(schema, query); if (query.getTrace().getLevel() >= 1) query.trace(getName() + " dispatch response: " + result, false, 1); @@ -217,8 +210,7 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { return parts; } - @Override - public void fill(Result result, String summaryClass, Execution execution) { + public void fill(Result result, String summaryClass) { if (result.isFilled(summaryClass)) return; // TODO: Checked in the superclass - remove List<Result> parts = partitionHits(result, summaryClass); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java index be8524bedd2..a246589ec7e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java @@ -111,9 +111,9 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM } @Override - protected InvokerResult getSearchResult(Execution execution) throws IOException { + protected InvokerResult getSearchResult() throws IOException { InvokerResult result = new InvokerResult(query, query.getHits()); - List<LeanHit> merged = Collections.emptyList(); + List<LeanHit> merged = List.of(); long nextTimeout = query.getTimeLeft(); var groupingResultAggregator = new GroupingResultAggregator(); try { @@ -124,7 +124,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM coverageAggregator.getAnsweredNodes() + " responses received"); break; } else { - InvokerResult toMerge = invoker.getSearchResult(execution); + InvokerResult toMerge = invoker.getSearchResult(); merged = mergeResult(result.getResult(), toMerge, merged, groupingResultAggregator); ejectInvoker(invoker); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java index b494b5b3819..74da2388155 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java @@ -35,7 +35,7 @@ public class SearchErrorInvoker extends SearchInvoker { } @Override - protected Object sendSearchRequest(Query query, Object context) throws IOException { + protected Object sendSearchRequest(Query query, Object context) { this.query = query; if (monitor != null) { monitor.responseAvailable(this); @@ -44,7 +44,7 @@ public class SearchErrorInvoker extends SearchInvoker { } @Override - protected InvokerResult getSearchResult(Execution execution) throws IOException { + protected InvokerResult getSearchResult() { Result res = new Result(query, message); if (coverage != null) { res.setCoverage(coverage); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java index ad332494179..4ad89674b7b 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java @@ -31,9 +31,9 @@ public abstract class SearchInvoker extends CloseableInvoker { * nodes, the provided {@link Execution} may be used to retrieve document summaries required * for correct result windowing. */ - public Result search(Query query, Execution execution) throws IOException { + public Result search(Query query) throws IOException { sendSearchRequest(query, null); - InvokerResult result = getSearchResult(execution); + InvokerResult result = getSearchResult(); setFinalStatus(result.getResult().hits().getError() == null); result.complete(); return result.getResult(); @@ -48,7 +48,7 @@ public abstract class SearchInvoker extends CloseableInvoker { */ protected abstract Object sendSearchRequest(Query query, Object context) throws IOException; - protected abstract InvokerResult getSearchResult(Execution execution) throws IOException; + protected abstract InvokerResult getSearchResult() throws IOException; protected void setMonitor(ResponseMonitor<SearchInvoker> monitor) { this.monitor = monitor; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java index e34e5c97bc4..93e09a04180 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java @@ -86,7 +86,7 @@ public class RpcSearchInvoker extends SearchInvoker implements Client.ResponseRe } @Override - protected InvokerResult getSearchResult(Execution execution) throws IOException { + protected InvokerResult getSearchResult() throws IOException { long timeLeftMs = query.getTimeLeft(); if (timeLeftMs <= 0) { return errorResult(query, ErrorMessage.createTimeout("Timeout while waiting for " + getName())); diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingSearcher.java index 180fdc389d8..2f3b48b5683 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingSearcher.java @@ -126,7 +126,7 @@ public class StreamingSearcher extends VespaBackEndSearcher { } @Override - public Result doSearch2(Query query, Execution execution) { + public Result doSearch2(String schema, Query query) { if (query.getTimeLeft() <= 0) return new Result(query, ErrorMessage.createTimeout(String.format("No time left for searching (timeout=%d)", query.getTimeout()))); @@ -135,8 +135,6 @@ public class StreamingSearcher extends VespaBackEndSearcher { return new Result(query, ErrorMessage.createIllegalQuery("Streaming search requires either " + "streaming.groupname or streaming.selection")); } - // Cluster searcher guarantees that there will be one, and only one schema here - String schema = query.getModel().getRestrict().iterator().next(); if (query.getTrace().isTraceable(4)) query.trace("Routing to search cluster " + getSearchClusterName() + " and document type " + schema, 4); @@ -264,7 +262,7 @@ public class StreamingSearcher extends VespaBackEndSearcher { lazyTrace(query, 8, "Returning result ", result); if (skippedHits > 0) { - getLogger().info("skipping " + skippedHits + " hits for query: " + result.getQuery()); + log.info("skipping " + skippedHits + " hits for query: " + result.getQuery()); result.hits().addError(ErrorMessage.createTimeout("Missing hit summary data for " + skippedHits + " hits")); } @@ -366,8 +364,8 @@ public class StreamingSearcher extends VespaBackEndSearcher { } @Override - public Visitor createVisitor(Query query, String searchCluster, Route route, String documentType, int traceLevelOverride) { - return new StreamingVisitor(query, searchCluster, route, documentType, this, traceLevelOverride); + public Visitor createVisitor(Query query, String searchCluster, Route route, String schema, int traceLevelOverride) { + return new StreamingVisitor(query, searchCluster, route, schema, this, traceLevelOverride); } } diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingVisitor.java index 658eea0b526..e638e0cf12a 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingVisitor.java @@ -28,6 +28,7 @@ import com.yahoo.vdslib.VisitorStatistics; import com.yahoo.vespa.objects.BufferSerializer; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -76,13 +77,13 @@ class StreamingVisitor extends VisitorDataHandler implements Visitor { } public StreamingVisitor(Query query, String searchCluster, Route route, - String documentType, VisitorSessionFactory visitorSessionFactory, + String schema, VisitorSessionFactory visitorSessionFactory, int traceLevelOverride) { this.query = query; this.visitorSessionFactory = visitorSessionFactory; this.traceLevelOverride = traceLevelOverride; - setVisitorParameters(searchCluster, route, documentType); + setVisitorParameters(searchCluster, route, schema); } private int inferSessionTraceLevel(Query query) { @@ -112,8 +113,8 @@ class StreamingVisitor extends VisitorDataHandler implements Visitor { return query.properties().getString(streamingSelection); } - private void setVisitorParameters(String searchCluster, Route route, String documentType) { - params.setDocumentSelection(createSelectionString(documentType, createQuerySelectionString())); + private void setVisitorParameters(String searchCluster, Route route, String schema) { + params.setDocumentSelection(createSelectionString(schema, createQuerySelectionString())); params.setTimeoutMs(query.getTimeout()); // Per bucket visitor timeout params.setSessionTimeoutMs(query.getTimeout()); params.setVisitorLibrary("searchvisitor"); @@ -146,7 +147,8 @@ class StreamingVisitor extends VisitorDataHandler implements Visitor { encodeQueryData(query, 0, ed); params.setLibraryParameter("query", ed.getEncodedData()); params.setLibraryParameter("querystackcount", String.valueOf(ed.getReturned())); - params.setLibraryParameter("searchcluster", searchCluster.getBytes()); + params.setLibraryParameter("searchcluster", searchCluster.getBytes(StandardCharsets.UTF_8)); + params.setLibraryParameter("schema", schema.getBytes(StandardCharsets.UTF_8)); if (query.getPresentation().getSummary() != null) { params.setLibraryParameter("summaryclass", query.getPresentation().getSummary()); } else { diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VisitorFactory.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VisitorFactory.java index 10387065128..e33bbd206be 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VisitorFactory.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VisitorFactory.java @@ -11,6 +11,6 @@ import com.yahoo.search.Query; */ interface VisitorFactory { - Visitor createVisitor(Query query, String searchCluster, Route route, String documentType, int traceLevelOverride); + Visitor createVisitor(Query query, String searchCluster, Route route, String schema, int traceLevelOverride); } diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index 8d4e3364ce4..affa6161440 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -206,12 +206,12 @@ public class ClusterSearcherTestCase { } @Override - protected com.yahoo.search.Result doSearch2(Query query, Execution execution) { + protected com.yahoo.search.Result doSearch2(String schema, Query query) { return null; // search() is overriden, this should never be called } @Override - public com.yahoo.search.Result search(Query query, Execution execution) { + public com.yahoo.search.Result search(String schema, Query query) { com.yahoo.search.Result result = new com.yahoo.search.Result(query); List<Hit> hits = getHits(query); if (hits != null) { diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index 20d2a304c3c..8270700a66b 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch.test; -import com.yahoo.component.chain.Chain; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.handler.VipStatus; import com.yahoo.container.protect.Error; @@ -9,9 +8,9 @@ import com.yahoo.prelude.fastsearch.ClusterParams; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.prelude.fastsearch.FastSearcher; import com.yahoo.prelude.fastsearch.SummaryParameters; +import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; -import com.yahoo.search.Searcher; import com.yahoo.search.dispatch.MockDispatcher; import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.dispatch.searchcluster.Node; @@ -24,10 +23,8 @@ import com.yahoo.search.schema.DocumentSummary; import com.yahoo.search.schema.RankProfile; import com.yahoo.search.schema.Schema; import com.yahoo.search.schema.SchemaInfo; -import com.yahoo.search.searchchain.Execution; import org.junit.jupiter.api.Test; -import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.logging.Level; @@ -42,16 +39,18 @@ import static org.junit.jupiter.api.Assertions.*; * @author bratseth */ public class FastSearcherTestCase { + private static final String SCHEMA = "test"; + private static final String CLUSTER = "test"; @Test void testNullQuery() { Logger.getLogger(FastSearcher.class.getName()).setLevel(Level.ALL); FastSearcher fastSearcher = new FastSearcher("container.0", - MockDispatcher.create(Collections.emptyList()), + MockDispatcher.create(List.of()), new SummaryParameters(null), new ClusterParams("testhittype"), - documentdbInfoConfig("test"), - schemaInfo("test")); + documentdbInfoConfig(SCHEMA), + schemaInfo(SCHEMA)); String query = "?junkparam=ignored"; Result result = doSearch(fastSearcher, new Query(query), 0, 10); @@ -63,30 +62,20 @@ public class FastSearcherTestCase { assertEquals(Error.NULL_QUERY.code, message.getCode()); } - private Chain<Searcher> chainedAsSearchChain(Searcher topOfChain) { - List<Searcher> searchers = new ArrayList<>(); - searchers.add(topOfChain); - return new Chain<>(searchers); - } - - private Result doSearch(Searcher searcher, Query query, int offset, int hits) { + private Result doSearch(VespaBackEndSearcher searcher, Query query, int offset, int hits) { query.setOffset(offset); query.setHits(hits); - return createExecution(searcher).search(query); - } - - private Execution createExecution(Searcher searcher) { - return new Execution(chainedAsSearchChain(searcher), Execution.Context.createContextStub()); + return searcher.search(SCHEMA, query); } @Test void testSinglePassGroupingIsForcedWithSingleNodeGroups() { FastSearcher fastSearcher = new FastSearcher("container.0", - MockDispatcher.create(List.of(new Node("test", 0, "host0", 0))), + MockDispatcher.create(List.of(new Node(CLUSTER, 0, "host0", 0))), new SummaryParameters(null), new ClusterParams("testhittype"), - documentdbInfoConfig("test"), - schemaInfo("test")); + documentdbInfoConfig(SCHEMA), + schemaInfo(SCHEMA)); Query q = new Query("?query=foo"); GroupingRequest request1 = GroupingRequest.newInstance(q); request1.setRootOperation(new AllOperation()); @@ -98,18 +87,18 @@ public class FastSearcherTestCase { request2.setRootOperation(all); assertForceSinglePassIs(false, q); - fastSearcher.search(q, new Execution(Execution.Context.createContextStub())); + fastSearcher.search(SCHEMA, q); assertForceSinglePassIs(true, q); } @Test void testRankProfileValidation() { FastSearcher fastSearcher = new FastSearcher("container.0", - MockDispatcher.create(List.of(new Node("test", 0, "host0", 0))), + MockDispatcher.create(List.of(new Node(CLUSTER, 0, "host0", 0))), new SummaryParameters(null), new ClusterParams("testhittype"), - documentdbInfoConfig("test"), - schemaInfo("test")); + documentdbInfoConfig(SCHEMA), + schemaInfo(SCHEMA)); assertFalse(searchError("?query=q", fastSearcher).contains("does not contain requested rank profile")); assertFalse(searchError("?query=q&ranking.profile=default", fastSearcher).contains("does not contain requested rank profile")); assertTrue(searchError("?query=q&ranking.profile=nosuch", fastSearcher).contains("does not contain requested rank profile")); @@ -117,14 +106,14 @@ public class FastSearcherTestCase { @Test void testSummaryNeedsQuery() { - var documentDb = new DocumentdbInfoConfig(new DocumentdbInfoConfig.Builder().documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name("test"))); - var schema = new Schema.Builder("test") + var documentDb = new DocumentdbInfoConfig(new DocumentdbInfoConfig.Builder().documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name(SCHEMA))); + var schema = new Schema.Builder(SCHEMA) .add(new DocumentSummary.Builder("default").build()) .add(new RankProfile.Builder("default").setHasRankFeatures(false) .setHasSummaryFeatures(false) .build()); FastSearcher backend = new FastSearcher("container.0", - MockDispatcher.create(Collections.singletonList(new Node("test", 0, "host0", 0))), + MockDispatcher.create(Collections.singletonList(new Node(CLUSTER, 0, "host0", 0))), new SummaryParameters(null), new ClusterParams("testhittype"), documentDb, @@ -141,14 +130,14 @@ public class FastSearcherTestCase { @Test void testSinglePassGroupingIsNotForcedWithSingleNodeGroups() { - MockDispatcher dispatcher = MockDispatcher.create(List.of(new Node("test", 0, "host0", 0), new Node("test", 2, "host1", 0))); + MockDispatcher dispatcher = MockDispatcher.create(List.of(new Node(CLUSTER, 0, "host0", 0), new Node(CLUSTER, 2, "host1", 0))); FastSearcher fastSearcher = new FastSearcher("container.0", dispatcher, new SummaryParameters(null), new ClusterParams("testhittype"), - documentdbInfoConfig("test"), - schemaInfo("test")); + documentdbInfoConfig(SCHEMA), + schemaInfo(SCHEMA)); Query q = new Query("?query=foo"); GroupingRequest request1 = GroupingRequest.newInstance(q); request1.setRootOperation(new AllOperation()); @@ -160,7 +149,7 @@ public class FastSearcherTestCase { request2.setRootOperation(all); assertForceSinglePassIs(false, q); - fastSearcher.search(q, new Execution(Execution.Context.createContextStub())); + fastSearcher.search(SCHEMA, q); assertForceSinglePassIs(false, q); } @@ -183,7 +172,7 @@ public class FastSearcherTestCase { searchClusterB.name(clusterName); b.searchcluster(searchClusterB); VipStatus vipStatus = new VipStatus(b.build()); - List<Node> nodes_1 = List.of(new Node("test", 0, "host0", 0)); + List<Node> nodes_1 = List.of(new Node(CLUSTER, 0, "host0", 0)); RpcResourcePool rpcPool_1 = new RpcResourcePool(MockDispatcher.toDispatchConfig(), MockDispatcher.toNodesConfig(nodes_1)); MockDispatcher dispatch_1 = MockDispatcher.create(nodes_1, rpcPool_1, vipStatus); dispatch_1.clusterMonitor.shutdown(); @@ -193,12 +182,12 @@ public class FastSearcherTestCase { assertTrue(vipStatus.isInRotation()); //Verify that deconstruct does not touch vipstatus } - private String searchError(String query, Searcher searcher) { + private String searchError(String query, VespaBackEndSearcher searcher) { return search(query, searcher).hits().getError().getDetailedMessage(); } - private Result search(String query, Searcher searcher) { - return searcher.search(new Query(query), new Execution(Execution.Context.createContextStub())); + private Result search(String query, VespaBackEndSearcher searcher) { + return searcher.search(SCHEMA, new Query(query)); } private DocumentdbInfoConfig documentdbInfoConfig(String schemaName) { diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PartialFillTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PartialFillTestCase.java index 27a27e993a8..a033ccdace6 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PartialFillTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PartialFillTestCase.java @@ -2,16 +2,13 @@ package com.yahoo.prelude.fastsearch.test; import com.yahoo.component.chain.Chain; -import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; -import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.search.result.ErrorHit; import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.searchchain.Execution; import org.junit.jupiter.api.Test; import java.util.ArrayList; @@ -27,7 +24,7 @@ public class PartialFillTestCase { public static class FS4 extends VespaBackEndSearcher { public List<Result> history = new ArrayList<>(); - protected Result doSearch2(Query query, Execution execution) { + protected Result doSearch2(String schema, Query query) { return new Result(query); } protected void doPartialFill(Result result, String summaryClass) { @@ -36,7 +33,7 @@ public class PartialFillTestCase { } public static class BadFS4 extends VespaBackEndSearcher { - protected Result doSearch2(Query query, Execution execution) { + protected Result doSearch2(String schema, Query query) { return new Result(query); } protected void doPartialFill(Result result, String summaryClass) { @@ -130,24 +127,18 @@ public class PartialFillTestCase { com.yahoo.search.result.ErrorMessage error = i.next(); switch (n) { case 0: - assertEquals(exp_sub, error); - break; case 1: assertEquals(exp_sub, error); break; default: - assertTrue(false); + fail(); } n++; } } - private Execution createExecution(Searcher searcher) { - return new Execution(chainedAsSearchChain(searcher), Execution.Context.createContextStub()); - } - - private void doFill(Searcher searcher, Result result, String summaryClass) { - createExecution(searcher).fill(result, summaryClass); + private void doFill(VespaBackEndSearcher searcher, Result result, String summaryClass) { + searcher.fill(result, summaryClass); } private Chain<Searcher> chainedAsSearchChain(Searcher topOfChain) { diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java index 8dc74110291..23846db3f0b 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java @@ -18,7 +18,6 @@ import com.yahoo.search.dispatch.searchcluster.Pinger; import com.yahoo.search.dispatch.searchcluster.PongHandler; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import com.yahoo.search.dispatch.searchcluster.SearchGroups; -import com.yahoo.search.searchchain.Execution; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.config.search.DispatchNodesConfig; import com.yahoo.yolean.UncheckedInterruptedException; @@ -243,7 +242,7 @@ public class DispatcherTest { rpcPool.getConnection(node.key()).request(null, null, 0, null, null, 0); return null; }; - @Override protected InvokerResult getSearchResult(Execution execution) { + @Override protected InvokerResult getSearchResult() { return new InvokerResult(new Result(new Query())); } @Override protected void release() { } @@ -288,11 +287,11 @@ public class DispatcherTest { // Start some searches, one against each group, since we have a round-robin policy. SearchInvoker search0 = dispatcher.getSearchInvoker(new Query(), null); - search0.search(new Query(), null); + search0.search(new Query()); // Unknown whether the first or second search hits node0, so we must track that. int offset = nodeIdOfSearcher0.get(); SearchInvoker search1 = dispatcher.getSearchInvoker(new Query(), null); - search1.search(new Query(), null); + search1.search(new Query()); // Wait for the current cluster monitor to be mid-ping-round. doPing.set(true); @@ -330,7 +329,7 @@ public class DispatcherTest { // Next search should hit group0 again, this time on node2. SearchInvoker search2 = dispatcher.getSearchInvoker(new Query(), null); - search2.search(new Query(), null); + search2.search(new Query()); // Searches against nodes 1 and 2 complete. (offset == 0 ? search0 : search1).close(); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java index fac75b1dd68..8ced7d3895e 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java @@ -61,7 +61,7 @@ public class InterleavedSearchInvokerTest { expectedEvents.add(new Event(4900, 100, 1)); expectedEvents.add(new Event(4800, 100, 2)); - invoker.search(query, null); + invoker.search(query); assertTrue(expectedEvents.isEmpty(), "All test scenario events processed"); } @@ -75,7 +75,7 @@ public class InterleavedSearchInvokerTest { expectedEvents.add(new Event(4700, 300, 1)); expectedEvents.add(null); - Result result = invoker.search(query, null); + Result result = invoker.search(query); assertTrue(expectedEvents.isEmpty(), "All test scenario events processed"); assertNull(result.hits().getErrorHit(), "Result is not marked as an error"); @@ -94,7 +94,7 @@ public class InterleavedSearchInvokerTest { expectedEvents.add(new Event(2400, 100, 2)); expectedEvents.add(new Event(0, 0, null)); - Result result = invoker.search(query, null); + Result result = invoker.search(query); assertTrue(expectedEvents.isEmpty(), "All test scenario events processed"); assertNull(result.hits().getErrorHit(), "Result is not marked as an error"); @@ -113,7 +113,7 @@ public class InterleavedSearchInvokerTest { expectedEvents.add(new Event(null, 100, 0)); expectedEvents.add(new Event(null, 200, 1)); - Result result = invoker.search(query, null); + Result result = invoker.search(query); Coverage cov = result.getCoverage(true); assertEquals(100000L, cov.getDocs()); @@ -134,7 +134,7 @@ public class InterleavedSearchInvokerTest { expectedEvents.add(new Event(null, 100, 0)); expectedEvents.add(new Event(null, 200, 1)); - Result result = invoker.search(query, null); + Result result = invoker.search(query); Coverage cov = result.getCoverage(true); assertEquals(23420L, cov.getDocs()); @@ -156,7 +156,7 @@ public class InterleavedSearchInvokerTest { expectedEvents.add(new Event(null, 100, 0)); expectedEvents.add(new Event(null, 200, 1)); - Result result = invoker.search(query, null); + Result result = invoker.search(query); Coverage cov = result.getCoverage(true); assertEquals(9900L, cov.getDocs()); @@ -178,7 +178,7 @@ public class InterleavedSearchInvokerTest { expectedEvents.add(new Event(null, 100, 0)); expectedEvents.add(null); - Result result = invoker.search(query, null); + Result result = invoker.search(query); Coverage cov = result.getCoverage(true); assertEquals(50155L, cov.getDocs()); @@ -213,7 +213,7 @@ public class InterleavedSearchInvokerTest { query.setHits(8); query.properties().set(Dispatcher.topKProbability, topKProbability); SearchInvoker[] invokers = invoker.invokers().toArray(new SearchInvoker[0]); - Result result = invoker.search(query, null); + Result result = invoker.search(query); assertEquals(2, invokers.length); assertEquals(expectedK, ((MockInvoker) invokers[0]).hitsRequested); assertEquals(8, result.hits().size()); @@ -264,7 +264,7 @@ public class InterleavedSearchInvokerTest { void requireThatMergeOfConcreteHitsObeySorting() throws IOException { try (InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5, B5, new Group(0, List.of()))) { query.setHits(12); - Result result = invoker.search(query, null); + Result result = invoker.search(query); assertEquals(10, result.hits().size()); assertEquals(11.0, result.hits().get(0).getRelevance().getScore(), DELTA); assertEquals(1.0, result.hits().get(9).getRelevance().getScore(), DELTA); @@ -273,7 +273,7 @@ public class InterleavedSearchInvokerTest { } try ( InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(B5, A5, new Group(0, List.of()))) { - Result result = invoker.search(query, null); + Result result = invoker.search(query); assertEquals(10, result.hits().size()); assertEquals(11.0, result.hits().get(0).getRelevance().getScore(), DELTA); assertEquals(1.0, result.hits().get(9).getRelevance().getScore(), DELTA); @@ -287,7 +287,7 @@ public class InterleavedSearchInvokerTest { try (InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5, B5, new Group(0, List.of()))) { query.setHits(3); query.setOffset(5); - Result result = invoker.search(query, null); + Result result = invoker.search(query); assertEquals(3, result.hits().size()); assertEquals(7.0, result.hits().get(0).getRelevance().getScore(), DELTA); assertEquals(3.0, result.hits().get(2).getRelevance().getScore(), DELTA); @@ -297,7 +297,7 @@ public class InterleavedSearchInvokerTest { try (InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(B5, A5, new Group(0, List.of()))) { query.setOffset(5); - Result result = invoker.search(query, null); + Result result = invoker.search(query); assertEquals(3, result.hits().size()); assertEquals(7.0, result.hits().get(0).getRelevance().getScore(), DELTA); assertEquals(3.0, result.hits().get(2).getRelevance().getScore(), DELTA); @@ -311,7 +311,7 @@ public class InterleavedSearchInvokerTest { try (InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5Aux, B5Aux, new Group(0, List.of()))) { query.setHits(3); query.setOffset(5); - Result result = invoker.search(query, null); + Result result = invoker.search(query); assertEquals(7, result.hits().size()); assertEquals(7.0, result.hits().get(0).getRelevance().getScore(), DELTA); assertEquals(3.0, result.hits().get(2).getRelevance().getScore(), DELTA); @@ -322,7 +322,7 @@ public class InterleavedSearchInvokerTest { try (InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(B5Aux, A5Aux, new Group(0, List.of()))) { query.setOffset(5); - Result result = invoker.search(query, null); + Result result = invoker.search(query); assertEquals(7, result.hits().size()); assertEquals(7.0, result.hits().get(0).getRelevance().getScore(), DELTA); assertEquals(3.0, result.hits().get(2).getRelevance().getScore(), DELTA); @@ -359,7 +359,7 @@ public class InterleavedSearchInvokerTest { try (InterleavedSearchInvoker invoker = new InterleavedSearchInvoker(Timer.monotonic, invokers, hitEstimator, dispatchConfig, new Group(0, List.of()), Collections.emptySet())) { invoker.responseAvailable(invokers.get(0)); invoker.responseAvailable(invokers.get(1)); - Result result = invoker.search(query, null); + Result result = invoker.search(query); assertEquals(1, ((GroupingListHit) result.hits().get(0)).getGroupingList().size()); } for (SearchInvoker invoker : invokers) { @@ -404,7 +404,7 @@ public class InterleavedSearchInvokerTest { expectedEvents.add(new Event(null, 1, 1)); expectedEvents.add(new Event(null, 100, 0)); - Result result = invoker.search(query, null); + Result result = invoker.search(query); Coverage cov = result.getCoverage(true); assertEquals(50155L, cov.getDocs()); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java index a1521a88fb7..e43f3b0bd9a 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java @@ -40,7 +40,7 @@ class MockInvoker extends SearchInvoker { } @Override - protected InvokerResult getSearchResult(Execution execution) { + protected InvokerResult getSearchResult() { InvokerResult ret = new InvokerResult(query, 10); if (coverage != null) { ret.getResult().setCoverage(coverage); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java index 3f527f34f31..2600d8612f8 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java @@ -9,7 +9,6 @@ import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.dispatch.searchcluster.Node; -import com.yahoo.search.searchchain.Execution; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -45,7 +44,7 @@ public class RpcSearchInvokerTest { assertEquals(10, request.getHits()); assertEquals(3, request.getOffset()); - assertTrue(request.getQueryTreeBlob().size() > 0); + assertFalse(request.getQueryTreeBlob().isEmpty()); var invoker2 = new RpcSearchInvoker(mockSearcher(), compressor, new Node("test", 8, "eight", 1), mockPool, 1000); RpcSearchInvoker.RpcContext context2 = (RpcSearchInvoker.RpcContext) invoker2.sendSearchRequest(q, context); @@ -121,7 +120,7 @@ public class RpcSearchInvokerTest { private VespaBackEndSearcher mockSearcher() { return new VespaBackEndSearcher() { @Override - protected Result doSearch2(Query query, Execution execution) { + protected Result doSearch2(String schema, Query query) { fail("Unexpected call"); return null; } diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java index 3d9a958c757..c7e95f802ab 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java @@ -16,7 +16,6 @@ import com.yahoo.search.Result; import com.yahoo.search.result.Hit; import com.yahoo.search.schema.Schema; import com.yahoo.search.schema.SchemaInfo; -import com.yahoo.search.searchchain.Execution; import com.yahoo.searchlib.aggregation.Grouping; import com.yahoo.vdslib.DocumentSummary; import com.yahoo.vdslib.SearchResult; @@ -161,9 +160,7 @@ public class StreamingSearcherTestCase { } private static Result executeQuery(StreamingSearcher searcher, Query query) { - Execution execution = new Execution(Execution.Context.createContextStub()); - query.getModel().setRestrict("test"); - return searcher.doSearch2(query, execution); + return searcher.doSearch2("test", query); } private static Query[] generateTestQueries(String queryString) { diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingVisitorTest.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingVisitorTest.java index 8be60b7c3b0..30fb33ee8c8 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingVisitorTest.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingVisitorTest.java @@ -23,6 +23,7 @@ import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import static org.junit.jupiter.api.Assertions.*; @@ -76,7 +77,7 @@ public class StreamingVisitorTest { }; } - private class QueryArguments { + private static class QueryArguments { // General query parameters String query = "test"; double timeout = 0.5; @@ -188,25 +189,21 @@ public class StreamingVisitorTest { return query; } - private void verifyVisitorParameters(VisitorParameters params, QueryArguments qa, String searchCluster, String docType, Route route) { + private void verifyVisitorParameters(VisitorParameters params, QueryArguments qa, String searchCluster, String schema, Route route) { //System.out.println("params="+params); // Verify parameters based on properties if (qa.userId != null) { - assertEquals(docType + " and ( id.user=="+qa.userId + " )", params.getDocumentSelection()); + assertEquals(schema + " and ( id.user=="+qa.userId + " )", params.getDocumentSelection()); } else if (qa.groupName != null) { - assertEquals(docType + " and ( id.group==\""+qa.groupName+"\" )", params.getDocumentSelection()); + assertEquals(schema + " and ( id.group==\""+qa.groupName+"\" )", params.getDocumentSelection()); } else if ((qa.selection == null) || qa.selection.isEmpty()) { - assertEquals(docType, params.getDocumentSelection()); + assertEquals(schema, params.getDocumentSelection()); } else { - assertEquals(docType + " and ( " + qa.selection + " )", params.getDocumentSelection()); + assertEquals(schema + " and ( " + qa.selection + " )", params.getDocumentSelection()); } assertEquals(qa.from, params.getFromTimestamp()); assertEquals(qa.to, params.getToTimestamp()); - if (qa.priority != null) { - assertEquals(qa.priority, params.getPriority()); - } else { - assertEquals(DocumentProtocol.Priority.VERY_HIGH, params.getPriority()); - } + assertEquals(Objects.requireNonNullElse(qa.priority, DocumentProtocol.Priority.VERY_HIGH), params.getPriority()); if (qa.maxBucketsPerVisitor != 0) { assertEquals(qa.maxBucketsPerVisitor, params.getMaxBucketsPerVisitor()); } else { @@ -227,17 +224,10 @@ public class StreamingVisitorTest { //System.err.println("query="+new String(params.getLibraryParameters().get("querystackcount"))); assertNotNull(params.getLibraryParameters().get("querystackcount")); // TODO: Check contents assertEquals(searchCluster, new String(params.getLibraryParameters().get("searchcluster"))); - if (qa.summary != null) { - assertEquals(qa.summary, new String(params.getLibraryParameters().get("summaryclass"))); - } else { - assertEquals("default", new String(params.getLibraryParameters().get("summaryclass"))); - } + assertEquals(schema, new String(params.getLibraryParameters().get("schema"))); + assertEquals(Objects.requireNonNullElse(qa.summary, "default"), new String(params.getLibraryParameters().get("summaryclass"))); assertEquals(Integer.toString(qa.offset+qa.hits), new String(params.getLibraryParameters().get("summarycount"))); - if (qa.profile != null) { - assertEquals(qa.profile, new String(params.getLibraryParameters().get("rankprofile"))); - } else { - assertEquals("default", new String(params.getLibraryParameters().get("rankprofile"))); - } + assertEquals(Objects.requireNonNullElse(qa.profile, "default"), new String(params.getLibraryParameters().get("rankprofile"))); //System.err.println("queryflags="+new String(params.getLibraryParameters().get("queryflags"))); assertNotNull(params.getLibraryParameters().get("queryflags")); // TODO: Check contents if (qa.location != null) { diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index f6f6646eb76..a9e17bbb45c 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -65,7 +65,7 @@ <assertj.vespa.version>3.25.3</assertj.vespa.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> - <aws-sdk.vespa.version>1.12.674</aws-sdk.vespa.version> + <aws-sdk.vespa.version>1.12.675</aws-sdk.vespa.version> <athenz.vespa.version>1.11.53</athenz.vespa.version> <!-- Athenz END --> @@ -122,7 +122,7 @@ <mockito.vespa.version>5.11.0</mockito.vespa.version> <mojo-executor.vespa.version>2.4.0</mojo-executor.vespa.version> <netty.vespa.version>4.1.107.Final</netty.vespa.version> - <netty-tcnative.vespa.version>2.0.63.Final</netty-tcnative.vespa.version> + <netty-tcnative.vespa.version>2.0.65.Final</netty-tcnative.vespa.version> <onnxruntime.vespa.version>1.17.1</onnxruntime.vespa.version> <opennlp.vespa.version>2.3.2</opennlp.vespa.version> <opentest4j.vespa.version>1.3.0</opentest4j.vespa.version> diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java index 180173789ae..1836f25f8d2 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java @@ -40,6 +40,7 @@ public class MbusRequestContext implements RequestContext, ResponseHandler { private final static Logger log = Logger.getLogger(MbusRequestContext.class.getName()); private final static CopyOnWriteHashMap<String, URI> uriCache = new CopyOnWriteHashMap<>(); + private final AtomicBoolean deserialized = new AtomicBoolean(false); private final AtomicBoolean responded = new AtomicBoolean(false); private final ProcessingFactory processingFactory; diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java index 030f95e380b..41d7f7920ca 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java @@ -81,7 +81,12 @@ class ProcessingFactory { log.fine(() -> "Unable to get document factory component '" + componentId + "' from document factory registry."); return document; } - return cdf.getDocumentCopy(document.getDataType().getName(), document, document.getId()); + try { + return cdf.getDocumentCopy(document.getDataType().getName(), document, document.getId()); + } + catch (RuntimeException e) { + throw new IllegalArgumentException("error in document with id '" + document.getId() + "'", e); + } } private ContainerDocumentConfig.Doctype getDocumentConfig(String name) { diff --git a/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java index 8b4b94f6bbf..d09967f973f 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java @@ -29,6 +29,9 @@ import java.util.Objects; */ public class StringFieldValue extends FieldValue { + // TODO: remove this, it's a temporary workaround for invalid data stored before unicode validation was fixed + private static final boolean replaceInvalidUnicode = System.getProperty("vespa.replace_invalid_unicode", "false").equals("true"); + private static class Factory extends PrimitiveDataType.Factory { @Override public FieldValue create() { return new StringFieldValue(); } @Override public FieldValue create(String value) { return new StringFieldValue(value); } @@ -56,16 +59,17 @@ public class StringFieldValue extends FieldValue { setValue(value); } - private static void validateTextString(String value) { + private static String validateTextString(String value) { if ( ! Text.isValidTextString(value)) { - throw new IllegalArgumentException("The string field value contains illegal code point 0x" + - Integer.toHexString(Text.validateTextString(value).getAsInt()).toUpperCase()); + if (replaceInvalidUnicode) return Text.stripInvalidCharacters(value); + else throw new IllegalArgumentException("The string field value contains illegal code point 0x" + + Integer.toHexString(Text.validateTextString(value).getAsInt()).toUpperCase()); } + return value; } private void setValue(String value) { - validateTextString(value); - this.value = value; + this.value = validateTextString(value); } /** diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index ffc6cf75ab0..a5453ac5273 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -131,6 +131,7 @@ vespa_define_module( src/tests/engine/proto_rpc_adapter src/tests/expression/attributenode src/tests/expression/current_index_setup + src/tests/expression/interpolated_document_field_lookup_node src/tests/features src/tests/features/beta src/tests/features/bm25 diff --git a/searchlib/src/tests/expression/interpolated_document_field_lookup_node/CMakeLists.txt b/searchlib/src/tests/expression/interpolated_document_field_lookup_node/CMakeLists.txt new file mode 100644 index 00000000000..267848d9468 --- /dev/null +++ b/searchlib/src/tests/expression/interpolated_document_field_lookup_node/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_interpolated_document_field_lookup_node_test_app + SOURCES + interpolated_document_field_lookup_node_test.cpp + DEPENDS + searchlib_test +) +vespa_add_test(NAME searchlib_interpolated_document_field_lookup_node_test_app COMMAND searchlib_interpolated_document_field_lookup_node_test_app) diff --git a/searchlib/src/tests/expression/interpolated_document_field_lookup_node/interpolated_document_field_lookup_node_test.cpp b/searchlib/src/tests/expression/interpolated_document_field_lookup_node/interpolated_document_field_lookup_node_test.cpp new file mode 100644 index 00000000000..c131efcacb2 --- /dev/null +++ b/searchlib/src/tests/expression/interpolated_document_field_lookup_node/interpolated_document_field_lookup_node_test.cpp @@ -0,0 +1,98 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/document/fieldvalue/arrayfieldvalue.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/fieldvalue/doublefieldvalue.h> +#include <vespa/document/repo/configbuilder.h> +#include <vespa/searchlib/expression/constantnode.h> +#include <vespa/searchlib/expression/floatresultnode.h> +#include <vespa/searchlib/expression/interpolated_document_field_lookup_node.h> +#include <vespa/searchlib/test/doc_builder.h> +#include <vespa/vespalib/gtest/gtest.h> + +using document::DataType; +using document::Document; +using document::DoubleFieldValue; +using search::expression::ConstantNode; +using search::expression::DocumentAccessorNode; +using search::expression::FloatResultNode; +using search::expression::InterpolatedDocumentFieldLookupNode; +using search::test::DocBuilder; + +const vespalib::string field_name("f"); + +struct Fixture { + DocBuilder _builder; + std::unique_ptr<Document> _doc; + std::unique_ptr<DocumentAccessorNode> _node; + + Fixture(); + ~Fixture(); + + Fixture& setup_doc(std::vector<double> field_value); + Fixture& setup_node(double lookup_value); + double evaluate(); +}; + +Fixture::Fixture() + : _builder([](auto& header) + { header.addField(field_name, document::config_builder::Array(DataType::T_DOUBLE)); }), + _doc(_builder.make_document("id:ns:searchdocument::0")), + _node() +{ +} + +Fixture::~Fixture() = default; + + +Fixture& +Fixture::setup_doc(std::vector<double> field_value) +{ + auto array = _builder.make_array(field_name); + for (auto& v : field_value) { + array.add(DoubleFieldValue(v)); + } + _doc->setValue(field_name, array); + return *this; +} + +Fixture& +Fixture::setup_node(double lookup_value) +{ + _node = std::make_unique<InterpolatedDocumentFieldLookupNode>(field_name, std::make_unique<ConstantNode>(std::make_unique<FloatResultNode>(lookup_value))); + _node->prepare(true); + _node->setDocType(_doc->getType()); + _node->setDoc(*_doc); + return *this; +} + +double +Fixture::evaluate() +{ + EXPECT_TRUE(_node->execute()); + return _node->getResult()->getFloat(); +} + + +class InterpolatedDocumentFieldLookupNodeTest : public Fixture, + public ::testing::Test +{ +protected: + InterpolatedDocumentFieldLookupNodeTest(); + ~InterpolatedDocumentFieldLookupNodeTest() override; +}; + +InterpolatedDocumentFieldLookupNodeTest::InterpolatedDocumentFieldLookupNodeTest() = default; +InterpolatedDocumentFieldLookupNodeTest::~InterpolatedDocumentFieldLookupNodeTest() = default; + +TEST_F(InterpolatedDocumentFieldLookupNodeTest, test_interpolated_lookup_in_document_field) +{ + EXPECT_EQ(0.0, setup_doc({ 2, 10 }).setup_node(1.0).evaluate()); + EXPECT_EQ(0.0, setup_node(2.0).evaluate()); + EXPECT_EQ(0.3125, setup_node(4.5).evaluate()); + EXPECT_EQ(1.0, setup_node(10).evaluate()); + EXPECT_EQ(1.0, setup_node(11).evaluate()); + EXPECT_EQ(2.5, setup_doc({1.5, 5.25, 8.0, 14.0 }).evaluate()); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h b/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h index ce5a24d9c2f..53f2be88c1f 100644 --- a/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h +++ b/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h @@ -22,6 +22,8 @@ inline double array_cost(double my_est, size_t num_children) { * The following tests were executed on a machine with an Intel Xeon 2.5 GHz CPU with 48 cores and 256 Gb of memory: * ./searchlib_iterator_benchmark_test_app --gtest_filter='*analyze_term_search*' * + * TODO: Add details on OR benchmarking. + * * The benchmark summary shows the 'average ms per cost' of the different benchmark cases. * The following constants and formulas were derived to balance 'average ms per cost' to be similar * across the different benchmark cases. @@ -29,7 +31,7 @@ inline double array_cost(double my_est, size_t num_children) { // Non-strict cost of lookup based matching in an attribute (not fast-search). inline double lookup_cost(size_t num_indirections) { - return 1.0 + (num_indirections * 4.0); + return 1.0 + (num_indirections * 1.0); } // Strict cost of lookup based matching in an attribute (not fast-search). @@ -39,7 +41,7 @@ inline double lookup_strict_cost(size_t num_indirections) { // Non-strict cost of matching in a btree posting list (e.g. fast-search attribute or memory index field). inline double btree_cost() { - return 7.0; + return 1.0; } // Strict cost of matching in a btree posting list (e.g. fast-search attribute or memory index field). @@ -49,7 +51,7 @@ inline double btree_strict_cost(double my_est) { // Non-strict cost of matching in a disk index posting list. inline double disk_index_cost() { - return 12.0; + return 1.5; } // Strict cost of matching in a disk index posting list. diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index 52dd1c52fc2..dc58b607848 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -55,15 +55,24 @@ using vespalib::string; namespace { -std::optional<vespalib::string> +vespalib::stringref extract_search_cluster(const vdslib::Parameters& params) { - Parameters::ValueRef searchClusterBlob; - if (params.lookup("searchcluster", searchClusterBlob)) { - LOG(spam, "Received searchcluster blob of %zd bytes", searchClusterBlob.size()); - return {{searchClusterBlob.data(), searchClusterBlob.size()}}; + Parameters::ValueRef searchCluster; + if (params.lookup("searchcluster", searchCluster)) { + LOG(spam, "Received searchcluster blob of %zd bytes", searchCluster.size()); } - return std::nullopt; + return searchCluster; +} + +vespalib::stringref +extract_schema(const vdslib::Parameters& params) +{ + Parameters::ValueRef schema; + if (params.lookup("schema", schema)) { + LOG(spam, "Received searchcluster blob of %zd bytes", schema.size()); + } + return schema; } std::shared_ptr<const SearchEnvironmentSnapshot> @@ -71,8 +80,11 @@ get_search_environment_snapshot(VisitorEnvironment& v_env, const Parameters& par { auto& env = dynamic_cast<SearchEnvironment&>(v_env); auto search_cluster = extract_search_cluster(params); - if (search_cluster.has_value()) { - return env.get_snapshot(search_cluster.value()); + if ( !search_cluster.empty()) { + auto schema = extract_schema(params); + return schema.empty() + ? env.get_snapshot(search_cluster) + : env.get_snapshot(search_cluster + "/" + schema); } return {}; } diff --git a/vespalib/src/tests/btree/CMakeLists.txt b/vespalib/src/tests/btree/CMakeLists.txt index bf4edc9e5e9..3b6f012a888 100644 --- a/vespalib/src/tests/btree/CMakeLists.txt +++ b/vespalib/src/tests/btree/CMakeLists.txt @@ -20,6 +20,7 @@ vespa_add_executable(vespalib_btreeaggregation_test_app TEST btreeaggregation_test.cpp DEPENDS vespalib + GTest::gtest ) vespa_add_test(NAME vespalib_btreeaggregation_test_app COMMAND vespalib_btreeaggregation_test_app COST 30) vespa_add_executable(vespalib_iteratespeed_app diff --git a/vespalib/src/tests/btree/btreeaggregation_test.cpp b/vespalib/src/tests/btree/btreeaggregation_test.cpp index b33242b918e..34c1ca76772 100644 --- a/vespalib/src/tests/btree/btreeaggregation_test.cpp +++ b/vespalib/src/tests/btree/btreeaggregation_test.cpp @@ -16,8 +16,8 @@ #include <vespa/vespalib/btree/btreeaggregator.hpp> #include <vespa/vespalib/datastore/buffer_type.hpp> #include <vespa/vespalib/datastore/compaction_strategy.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/test/btree/btree_printer.h> -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/rand48.h> #include <iostream> @@ -216,115 +216,95 @@ cleanup(GenerationHandler & g, cleanup(g, m); } -class Test : public vespalib::TestApp { -private: +class BTreeAggregationTest : public ::testing::Test { +protected: + BTreeAggregationTest(); + ~BTreeAggregationTest() override; template <typename Tree> bool assertTree(const std::string & exp, const Tree &t); template <typename Tree> bool - assertAggregated(const MockTree &m, const Tree &t); + assertAggregated(const MockTree &m, const Tree &t, const vespalib::string& label); template <typename TreeStore> bool - assertAggregated(const MockTree &m, const TreeStore &s, EntryRef ref); + assertAggregated(const MockTree &m, const TreeStore &s, EntryRef ref, const vespalib::string& label); void buildSubTree(const std::vector<LeafPair> &sub, size_t numEntries); - - void requireThatNodeInsertWorks(); - void keys_are_aggregated_correctly_on_node_insertions(); - void requireThatNodeSplitInsertWorks(); - void keys_are_aggregated_correctly_when_node_split_on_insert(); - void requireThatTreeInsertWorks(); - void requireThatNodeStealWorks(); - void requireThatNodeRemoveWorks(); - void keys_are_aggregated_correctly_on_node_removal(); - void requireThatWeCanInsertAndRemoveFromTree(); - void requireThatSortedTreeInsertWorks(); - void requireThatCornerCaseTreeFindWorks(); - void requireThatBasicTreeIteratorWorks(); - void requireThatTreeIteratorAssignWorks(); - void requireThatUpdateOfKeyWorks(); - void requireThatUpdateOfDataWorks(); - void requireThatFrozenViewProvidesAggregatedValues(); - - void - requireThatSmallNodesWorks(); -public: - int Main() override; }; +BTreeAggregationTest::BTreeAggregationTest() = default; +BTreeAggregationTest::~BTreeAggregationTest() = default; template<typename Tree> bool -Test::assertTree(const std::string &exp, const Tree &t) +BTreeAggregationTest::assertTree(const std::string &exp, const Tree &t) { std::stringstream ss; test::BTreePrinter<std::stringstream, typename Tree::NodeAllocatorType> printer(ss, t.getAllocator()); printer.print(t.getRoot()); - if (!EXPECT_EQUAL(exp, ss.str())) return false; - return true; + bool failed = false; + EXPECT_EQ(exp, ss.str()) << (failed = true, ""); + return !failed; } template <typename Tree> bool -Test::assertAggregated(const MockTree &m, const Tree &t) +BTreeAggregationTest::assertAggregated(const MockTree &m, const Tree &t, const vespalib::string& label) { + SCOPED_TRACE(label); const MinMaxAggregated &ta(t.getAggregated()); + bool failed = false; if (t.getRoot().valid()) { - return - EXPECT_FALSE(m._rtree.empty()) && - EXPECT_EQUAL(m._rtree.rbegin()->first, - ta.getMax()) && - EXPECT_EQUAL(m._rtree.begin()->first, - ta.getMin()); + EXPECT_FALSE(m._rtree.empty()) << (failed = true, ""); + if (failed) { + return false; + } + EXPECT_EQ(m._rtree.rbegin()->first, ta.getMax()) << (failed = true, ""); + EXPECT_EQ(m._rtree.begin()->first, ta.getMin()) << (failed = true, ""); } else { - return EXPECT_TRUE(m._rtree.empty()) && - EXPECT_EQUAL(std::numeric_limits<int32_t>::min(), - ta.getMax()) && - EXPECT_EQUAL(std::numeric_limits<int32_t>::max(), - ta.getMin()); + EXPECT_TRUE(m._rtree.empty()) << (failed = true, ""); + EXPECT_EQ(std::numeric_limits<int32_t>::min(), ta.getMax()) << (failed = true, ""); + EXPECT_EQ(std::numeric_limits<int32_t>::max(), ta.getMin()) << (failed = true, ""); } + return !failed; } template <typename TreeStore> bool -Test::assertAggregated(const MockTree &m, const TreeStore &s, EntryRef ref) +BTreeAggregationTest::assertAggregated(const MockTree &m, const TreeStore &s, EntryRef ref, const vespalib::string& label) { + SCOPED_TRACE(label); typename TreeStore::Iterator i(s.begin(ref)); MinMaxAggregated sa(s.getAggregated(ref)); const MinMaxAggregated &ia(i.getAggregated()); + bool failed = false; if (ref.valid()) { - return - EXPECT_FALSE(m._rtree.empty()) && - EXPECT_EQUAL(m._rtree.rbegin()->first, - ia.getMax()) && - EXPECT_EQUAL(m._rtree.begin()->first, - ia.getMin()) && - EXPECT_EQUAL(m._rtree.rbegin()->first, - sa.getMax()) && - EXPECT_EQUAL(m._rtree.begin()->first, - sa.getMin()); + EXPECT_FALSE(m._rtree.empty()) << (failed = true, ""); + if (failed) { + return false; + } + EXPECT_EQ(m._rtree.rbegin()->first, ia.getMax()) << (failed = true, ""); + EXPECT_EQ(m._rtree.begin()->first, ia.getMin()) << (failed = true, ""); + EXPECT_EQ(m._rtree.rbegin()->first, sa.getMax()) << (failed = true, ""); + EXPECT_EQ(m._rtree.begin()->first, sa.getMin()) << (failed = true, ""); } else { - return EXPECT_TRUE(m._rtree.empty()) && - EXPECT_EQUAL(std::numeric_limits<int32_t>::min(), - ia.getMax()) && - EXPECT_EQUAL(std::numeric_limits<int32_t>::max(), - ia.getMin()) && - EXPECT_EQUAL(std::numeric_limits<int32_t>::min(), - sa.getMax()) && - EXPECT_EQUAL(std::numeric_limits<int32_t>::max(), - sa.getMin()); + EXPECT_TRUE(m._rtree.empty()) << (failed = true, ""); + EXPECT_EQ(std::numeric_limits<int32_t>::min(), ia.getMax()) << (failed = true, ""); + EXPECT_EQ(std::numeric_limits<int32_t>::max(), ia.getMin()) << (failed = true, ""); + EXPECT_EQ(std::numeric_limits<int32_t>::min(), sa.getMax()) << (failed =true, ""); + EXPECT_EQ(std::numeric_limits<int32_t>::max(), sa.getMin()) << (failed =true, ""); } + return !failed; } -void -Test::requireThatNodeInsertWorks() +TEST_F(BTreeAggregationTest, require_that_node_insert_works) { MyTree t; t.insert(20, 102); @@ -336,7 +316,8 @@ Test::requireThatNodeInsertWorks() EXPECT_TRUE(assertTree("{{10:101,20:102,30:103,40:104[min=101,max=104]}}", t)); } -void Test::keys_are_aggregated_correctly_on_node_insertions() { +TEST_F(BTreeAggregationTest, keys_are_aggregated_correctly_on_node_insertions) +{ MyKeyAggrTree t; t.insert(20, 102); EXPECT_TRUE(assertTree("{{20:102[min=20,max=20]}}", t)); @@ -367,8 +348,7 @@ populateLeafNode(Tree &t) populateTree(t, 4, 2); } -void -Test::requireThatNodeSplitInsertWorks() +TEST_F(BTreeAggregationTest, require_that_node_split_insert_works) { { // new entry in current node MyTree t; @@ -396,7 +376,8 @@ Test::requireThatNodeSplitInsertWorks() } } -void Test::keys_are_aggregated_correctly_when_node_split_on_insert() { +TEST_F(BTreeAggregationTest, keys_are_aggregated_correctly_when_node_split_on_insert) +{ { // new entry in current node MyKeyAggrTree t; populateLeafNode(t); @@ -423,8 +404,7 @@ void Test::keys_are_aggregated_correctly_when_node_split_on_insert() { } } -void -Test::requireThatTreeInsertWorks() +TEST_F(BTreeAggregationTest, require_that_tree_insert_works) { { // multi level node split MyTree t; @@ -523,8 +503,7 @@ struct BTreeStealTraits } -void -Test::requireThatNodeStealWorks() +TEST_F(BTreeAggregationTest, require_that_node_steal_works) { typedef BTree<MyKey, int32_t, btree::MinMaxAggregated, @@ -602,8 +581,7 @@ Test::requireThatNodeStealWorks() } } -void -Test::requireThatNodeRemoveWorks() +TEST_F(BTreeAggregationTest, require_that_node_remove_works) { MyTree t; populateLeafNode(t); @@ -615,7 +593,8 @@ Test::requireThatNodeRemoveWorks() EXPECT_TRUE(assertTree("{{5:105[min=105,max=105]}}", t)); } -void Test::keys_are_aggregated_correctly_on_node_removal() { +TEST_F(BTreeAggregationTest, keys_are_aggregated_correctly_on_node_removal) +{ MyKeyAggrTree t; populateLeafNode(t); t.remove(3); @@ -640,7 +619,7 @@ generateData(std::vector<LeafPair> & data, size_t numEntries) } void -Test::buildSubTree(const std::vector<LeafPair> &sub, +BTreeAggregationTest::buildSubTree(const std::vector<LeafPair> &sub, size_t numEntries) { GenerationHandler g; @@ -660,31 +639,31 @@ Test::buildSubTree(const std::vector<LeafPair> &sub, assert(numEntries == tree.size()); assert(tree.isValid()); - TEST_DO(EXPECT_TRUE(assertAggregated(mock, tree))); - EXPECT_EQUAL(numEntries, tree.size()); + EXPECT_TRUE(assertAggregated(mock, tree, "build_sub_tree")); + EXPECT_EQ(numEntries, tree.size()); EXPECT_TRUE(tree.isValid()); MyTree::Iterator itr = tree.begin(); MyTree::Iterator ritr = itr; if (numEntries > 0) { EXPECT_TRUE(ritr.valid()); - EXPECT_EQUAL(0u, ritr.position()); + EXPECT_EQ(0u, ritr.position()); --ritr; EXPECT_TRUE(!ritr.valid()); - EXPECT_EQUAL(numEntries, ritr.position()); + EXPECT_EQ(numEntries, ritr.position()); --ritr; EXPECT_TRUE(ritr.valid()); - EXPECT_EQUAL(numEntries - 1, ritr.position()); + EXPECT_EQ(numEntries - 1, ritr.position()); } else { EXPECT_TRUE(!ritr.valid()); - EXPECT_EQUAL(0u, ritr.position()); + EXPECT_EQ(0u, ritr.position()); --ritr; EXPECT_TRUE(!ritr.valid()); - EXPECT_EQUAL(0u, ritr.position()); + EXPECT_EQ(0u, ritr.position()); } for (size_t i = 0; i < numEntries; ++i) { EXPECT_TRUE(itr.valid()); - EXPECT_EQUAL(sorted[i].first, itr.getKey()); - EXPECT_EQUAL(sorted[i].second, itr.getData()); + EXPECT_EQ(sorted[i].first, itr.getKey()); + EXPECT_EQ(sorted[i].second, itr.getData()); ++itr; } EXPECT_TRUE(!itr.valid()); @@ -693,22 +672,21 @@ Test::buildSubTree(const std::vector<LeafPair> &sub, --ritr; for (size_t i = 0; i < numEntries; ++i) { EXPECT_TRUE(ritr.valid()); - EXPECT_EQUAL(sorted[numEntries - 1 - i].first, ritr.getKey()); - EXPECT_EQUAL(sorted[numEntries - 1 - i].second, ritr.getData()); + EXPECT_EQ(sorted[numEntries - 1 - i].first, ritr.getKey()); + EXPECT_EQ(sorted[numEntries - 1 - i].second, ritr.getData()); --ritr; } EXPECT_TRUE(!ritr.valid()); } -void -Test::requireThatWeCanInsertAndRemoveFromTree() +TEST_F(BTreeAggregationTest, require_that_we_can_insert_and_remove_from_tree) { GenerationHandler g; MyTree tree; MockTree mock; std::vector<LeafPair> exp; std::vector<LeafPair> sorted; - TEST_DO(EXPECT_TRUE(assertAggregated(mock, tree))); + EXPECT_TRUE(assertAggregated(mock, tree, "insert_and_remove_1")); size_t numEntries = 1000; generateData(exp, numEntries); sorted = exp; @@ -722,15 +700,15 @@ Test::requireThatWeCanInsertAndRemoveFromTree() EXPECT_TRUE(tree.insert(num, val)); EXPECT_TRUE(!tree.insert(num, val)); mock.insert(num, val); - TEST_DO(EXPECT_TRUE(assertAggregated(mock, tree))); + EXPECT_TRUE(assertAggregated(mock, tree, "insert_and_remove_2")); for (size_t j = 0; j <= i; ++j) { //LOG(info, "find[%zu](%d)", j, exp[j].first._val); MyTree::Iterator itr = tree.find(exp[j].first); EXPECT_TRUE(itr.valid()); - EXPECT_EQUAL(exp[j].first, itr.getKey()); - EXPECT_EQUAL(exp[j].second, itr.getData()); + EXPECT_EQ(exp[j].first, itr.getKey()); + EXPECT_EQ(exp[j].second, itr.getData()); } - EXPECT_EQUAL(i + 1u, tree.size()); + EXPECT_EQ(i + 1u, tree.size()); EXPECT_TRUE(tree.isValid()); buildSubTree(exp, i + 1); } @@ -745,36 +723,36 @@ Test::requireThatWeCanInsertAndRemoveFromTree() ++itre; if (numEntries > 0) { EXPECT_TRUE(ritr.valid()); - EXPECT_EQUAL(0u, ritr.position()); + EXPECT_EQ(0u, ritr.position()); --ritr; EXPECT_TRUE(!ritr.valid()); - EXPECT_EQUAL(numEntries, ritr.position()); + EXPECT_EQ(numEntries, ritr.position()); --ritr; EXPECT_TRUE(ritr.valid()); - EXPECT_EQUAL(numEntries - 1, ritr.position()); + EXPECT_EQ(numEntries - 1, ritr.position()); } else { EXPECT_TRUE(!ritr.valid()); - EXPECT_EQUAL(0u, ritr.position()); + EXPECT_EQ(0u, ritr.position()); --ritr; EXPECT_TRUE(!ritr.valid()); - EXPECT_EQUAL(0u, ritr.position()); + EXPECT_EQ(0u, ritr.position()); } MyTree::Iterator pitr = itr; for (size_t i = 0; i < numEntries; ++i) { ssize_t si = i; ssize_t sileft = numEntries - i; EXPECT_TRUE(itr.valid()); - EXPECT_EQUAL(i, itr.position()); - EXPECT_EQUAL(sileft, itre - itr); - EXPECT_EQUAL(-sileft, itr - itre); - EXPECT_EQUAL(sileft, itre2 - itr); - EXPECT_EQUAL(-sileft, itr - itre2); - EXPECT_EQUAL(si, itr - tree.begin()); - EXPECT_EQUAL(-si, tree.begin() - itr); - EXPECT_EQUAL(i != 0, itr - pitr); - EXPECT_EQUAL(-(i != 0), pitr - itr); - EXPECT_EQUAL(sorted[i].first, itr.getKey()); - EXPECT_EQUAL(sorted[i].second, itr.getData()); + EXPECT_EQ(i, itr.position()); + EXPECT_EQ(sileft, itre - itr); + EXPECT_EQ(-sileft, itr - itre); + EXPECT_EQ(sileft, itre2 - itr); + EXPECT_EQ(-sileft, itr - itre2); + EXPECT_EQ(si, itr - tree.begin()); + EXPECT_EQ(-si, tree.begin() - itr); + EXPECT_EQ(i != 0, itr - pitr); + EXPECT_EQ(-(i != 0), pitr - itr); + EXPECT_EQ(sorted[i].first, itr.getKey()); + EXPECT_EQ(sorted[i].second, itr.getData()); pitr = itr; ++itr; ritr = itr; @@ -783,12 +761,12 @@ Test::requireThatWeCanInsertAndRemoveFromTree() EXPECT_TRUE(ritr == pitr); } EXPECT_TRUE(!itr.valid()); - EXPECT_EQUAL(numEntries, itr.position()); + EXPECT_EQ(numEntries, itr.position()); ssize_t sNumEntries = numEntries; - EXPECT_EQUAL(sNumEntries, itr - tree.begin()); - EXPECT_EQUAL(-sNumEntries, tree.begin() - itr); - EXPECT_EQUAL(1, itr - pitr); - EXPECT_EQUAL(-1, pitr - itr); + EXPECT_EQ(sNumEntries, itr - tree.begin()); + EXPECT_EQ(-sNumEntries, tree.begin() - itr); + EXPECT_EQ(1, itr - pitr); + EXPECT_EQ(-1, pitr - itr); } // compact full tree by calling incremental compaction methods in a loop { @@ -818,52 +796,50 @@ Test::requireThatWeCanInsertAndRemoveFromTree() EXPECT_TRUE(!tree.remove(num)); EXPECT_TRUE(tree.isValid()); mock.erase(num); - TEST_DO(EXPECT_TRUE(assertAggregated(mock, tree))); + EXPECT_TRUE(assertAggregated(mock, tree, "insert_and_remove3")); for (size_t j = i + 1; j < numEntries; ++j) { MyTree::Iterator itr = tree.find(exp[j].first); EXPECT_TRUE(itr.valid()); - EXPECT_EQUAL(exp[j].first, itr.getKey()); - EXPECT_EQUAL(exp[j].second, itr.getData()); + EXPECT_EQ(exp[j].first, itr.getKey()); + EXPECT_EQ(exp[j].second, itr.getData()); } - EXPECT_EQUAL(numEntries - 1 - i, tree.size()); + EXPECT_EQ(numEntries - 1 - i, tree.size()); } } -void -Test::requireThatSortedTreeInsertWorks() +TEST_F(BTreeAggregationTest, require_that_sorted_tree_insert_works) { { MyTree tree; MockTree mock; - TEST_DO(EXPECT_TRUE(assertAggregated(mock, tree))); + EXPECT_TRUE(assertAggregated(mock, tree, "sorted_tree_insert1")); for (int i = 0; i < 1000; ++i) { EXPECT_TRUE(tree.insert(i, toVal(i))); mock.insert(i, toVal(i)); MyTree::Iterator itr = tree.find(i); EXPECT_TRUE(itr.valid()); - EXPECT_EQUAL(toVal(i), itr.getData()); + EXPECT_EQ(toVal(i), itr.getData()); EXPECT_TRUE(tree.isValid()); - TEST_DO(EXPECT_TRUE(assertAggregated(mock, tree))); + EXPECT_TRUE(assertAggregated(mock, tree, "sorted_tree_insert2")); } } { MyTree tree; MockTree mock; - TEST_DO(EXPECT_TRUE(assertAggregated(mock, tree))); + EXPECT_TRUE(assertAggregated(mock, tree, "sorted_tree_insert3")); for (int i = 1000; i > 0; --i) { EXPECT_TRUE(tree.insert(i, toVal(i))); mock.insert(i, toVal(i)); MyTree::Iterator itr = tree.find(i); EXPECT_TRUE(itr.valid()); - EXPECT_EQUAL(toVal(i), itr.getData()); + EXPECT_EQ(toVal(i), itr.getData()); EXPECT_TRUE(tree.isValid()); - TEST_DO(EXPECT_TRUE(assertAggregated(mock, tree))); + EXPECT_TRUE(assertAggregated(mock, tree, "sorted_tree_insert4")); } } } -void -Test::requireThatCornerCaseTreeFindWorks() +TEST_F(BTreeAggregationTest, require_that_corner_case_tree_find_works) { GenerationHandler g; MyTree tree; @@ -874,8 +850,7 @@ Test::requireThatCornerCaseTreeFindWorks() EXPECT_TRUE(!tree.find(1000).valid()); // higher than highest } -void -Test::requireThatBasicTreeIteratorWorks() +TEST_F(BTreeAggregationTest, require_that_basic_tree_iterator_works) { GenerationHandler g; MyTree tree; @@ -890,27 +865,24 @@ Test::requireThatBasicTreeIteratorWorks() size_t ei = 0; MyTree::Iterator itr = tree.begin(); MyTree::Iterator ritr; - EXPECT_EQUAL(1000u, itr.size()); + EXPECT_EQ(1000u, itr.size()); for (; itr.valid(); ++itr) { //LOG(info, "itr(%d, %s)", itr.getKey(), itr.getData().c_str()); - EXPECT_EQUAL(UNWRAP(exp[ei].first), UNWRAP(itr.getKey())); - EXPECT_EQUAL(exp[ei].second, itr.getData()); + EXPECT_EQ(UNWRAP(exp[ei].first), UNWRAP(itr.getKey())); + EXPECT_EQ(exp[ei].second, itr.getData()); ei++; ritr = itr; } - EXPECT_EQUAL(numEntries, ei); + EXPECT_EQ(numEntries, ei); for (; ritr.valid(); --ritr) { --ei; //LOG(info, "itr(%d, %s)", itr.getKey(), itr.getData().c_str()); - EXPECT_EQUAL(UNWRAP(exp[ei].first), UNWRAP(ritr.getKey())); - EXPECT_EQUAL(exp[ei].second, ritr.getData()); + EXPECT_EQ(UNWRAP(exp[ei].first), UNWRAP(ritr.getKey())); + EXPECT_EQ(exp[ei].second, ritr.getData()); } } - - -void -Test::requireThatTreeIteratorAssignWorks() +TEST_F(BTreeAggregationTest, require_that_tree_iterator_assign_works) { GenerationHandler g; MyTree tree; @@ -923,9 +895,9 @@ Test::requireThatTreeIteratorAssignWorks() EXPECT_TRUE(itr == itr2); int expNum = i; for (; itr2.valid(); ++itr2) { - EXPECT_EQUAL(expNum++, UNWRAP(itr2.getKey())); + EXPECT_EQ(expNum++, UNWRAP(itr2.getKey())); } - EXPECT_EQUAL(1000, expNum); + EXPECT_EQ(1000, expNum); } } @@ -940,8 +912,7 @@ struct UpdKeyComp { } }; -void -Test::requireThatUpdateOfKeyWorks() +TEST_F(BTreeAggregationTest, require_that_update_of_key_works) { using UpdKeyTree = BTree<int, BTreeNoLeafData, btree::NoAggregated, UpdKeyComp &>; using UpdKeyTreeIterator = UpdKeyTree::Iterator; @@ -951,7 +922,7 @@ Test::requireThatUpdateOfKeyWorks() for (int i = 0; i < 1000; i+=2) { EXPECT_TRUE(t.insert(i, BTreeNoLeafData(), cmp1)); } - EXPECT_EQUAL(0u, cmp1._numErrors); + EXPECT_EQ(0u, cmp1._numErrors); for (int i = 0; i < 1000; i+=2) { UpdKeyTreeIterator itr = t.find(i, cmp1); itr.writeKey(i + 1); @@ -961,23 +932,22 @@ Test::requireThatUpdateOfKeyWorks() UpdKeyTreeIterator itr = t.find(i, cmp2); EXPECT_TRUE(itr.valid()); } - EXPECT_EQUAL(0u, cmp2._numErrors); + EXPECT_EQ(0u, cmp2._numErrors); } -void -Test::requireThatUpdateOfDataWorks() +TEST_F(BTreeAggregationTest, require_that_update_of_data_works) { GenerationHandler g; MyTree t; MockTree mock; MyAggrCalc ac; MyTree::NodeAllocatorType &manager = t.getAllocator(); - TEST_DO(EXPECT_TRUE(assertAggregated(mock, t))); + EXPECT_TRUE(assertAggregated(mock, t, "update_data1")); for (int i = 0; i < 1000; i+=2) { EXPECT_TRUE(t.insert(i, toVal(i))); mock.insert(i, toVal(i)); - TEST_DO(EXPECT_TRUE(assertAggregated(mock, t))); + EXPECT_TRUE(assertAggregated(mock, t, "udate_data2")); } freezeTree(g, manager); for (int i = 0; i < 1000; i+=2) { @@ -985,31 +955,31 @@ Test::requireThatUpdateOfDataWorks() MyTree::Iterator itr2 = itr; t.thaw(itr); itr.updateData(toHighVal(i), ac); - EXPECT_EQUAL(toHighVal(i), itr.getData()); - EXPECT_EQUAL(toVal(i), itr2.getData()); + EXPECT_EQ(toHighVal(i), itr.getData()); + EXPECT_EQ(toVal(i), itr2.getData()); mock.erase(i); mock.insert(i, toHighVal(i)); - TEST_DO(EXPECT_TRUE(assertAggregated(mock, t))); + EXPECT_TRUE(assertAggregated(mock, t, "update_data3")); freezeTree(g, manager); itr = t.find(i); itr2 = itr; t.thaw(itr); itr.updateData(toLowVal(i), ac); - EXPECT_EQUAL(toLowVal(i), itr.getData()); - EXPECT_EQUAL(toHighVal(i), itr2.getData()); + EXPECT_EQ(toLowVal(i), itr.getData()); + EXPECT_EQ(toHighVal(i), itr2.getData()); mock.erase(i); mock.insert(i, toLowVal(i)); - TEST_DO(EXPECT_TRUE(assertAggregated(mock, t))); + EXPECT_TRUE(assertAggregated(mock, t, "update_data4")); freezeTree(g, manager); itr = t.find(i); itr2 = itr; t.thaw(itr); itr.updateData(toVal(i), ac); - EXPECT_EQUAL(toVal(i), itr.getData()); - EXPECT_EQUAL(toLowVal(i), itr2.getData()); + EXPECT_EQ(toVal(i), itr.getData()); + EXPECT_EQ(toLowVal(i), itr2.getData()); mock.erase(i); mock.insert(i, toVal(i)); - TEST_DO(EXPECT_TRUE(assertAggregated(mock, t))); + EXPECT_TRUE(assertAggregated(mock, t, "update_data5")); freezeTree(g, manager); } } @@ -1030,68 +1000,67 @@ remove(MyTreeStore& s, EntryRef& root, MyTreeStore::KeyType removal) } -void -Test::requireThatSmallNodesWorks() +TEST_F(BTreeAggregationTest, require_that_small_nodes_works) { GenerationHandler g; MyTreeStore s; MockTree mock; EntryRef root; - EXPECT_EQUAL(0u, s.size(root)); + EXPECT_EQ(0u, s.size(root)); EXPECT_TRUE(s.isSmallArray(root)); - TEST_DO(EXPECT_TRUE(assertAggregated(mock, s, root))); + EXPECT_TRUE(assertAggregated(mock, s, root, "small1")); insert(s, root, {40, toVal(40)}); mock.insert(40, toVal(40)); - EXPECT_EQUAL(1u, s.size(root)); + EXPECT_EQ(1u, s.size(root)); EXPECT_TRUE(s.isSmallArray(root)); - TEST_DO(EXPECT_TRUE(assertAggregated(mock, s, root))); + EXPECT_TRUE(assertAggregated(mock, s, root, "small2")); insert(s, root, {20, toVal(20)}); mock.insert(20, toVal(20)); - EXPECT_EQUAL(2u, s.size(root)); + EXPECT_EQ(2u, s.size(root)); EXPECT_TRUE(s.isSmallArray(root)); - TEST_DO(EXPECT_TRUE(assertAggregated(mock, s, root))); + EXPECT_TRUE(assertAggregated(mock, s, root, "small3")); insert(s, root, {60, toVal(60)}); mock.insert(60, toVal(60)); - EXPECT_EQUAL(3u, s.size(root)); + EXPECT_EQ(3u, s.size(root)); EXPECT_TRUE(s.isSmallArray(root)); - TEST_DO(EXPECT_TRUE(assertAggregated(mock, s, root))); + EXPECT_TRUE(assertAggregated(mock, s, root, "small4")); insert(s, root, {50, toVal(50)}); mock.insert(50, toVal(50)); - EXPECT_EQUAL(4u, s.size(root)); + EXPECT_EQ(4u, s.size(root)); EXPECT_TRUE(s.isSmallArray(root)); - TEST_DO(EXPECT_TRUE(assertAggregated(mock, s, root))); + EXPECT_TRUE(assertAggregated(mock, s, root, "small5")); for (uint32_t i = 0; i < 100; ++i) { insert(s, root, {int(1000 + i), 42}); mock.insert(1000 + i, 42); - EXPECT_EQUAL(5u + i, s.size(root)); - EXPECT_EQUAL(5u + i <= MyTreeStore::clusterLimit, s.isSmallArray(root)); - TEST_DO(EXPECT_TRUE(assertAggregated(mock, s, root))); + EXPECT_EQ(5u + i, s.size(root)); + EXPECT_EQ(5u + i <= MyTreeStore::clusterLimit, s.isSmallArray(root)); + EXPECT_TRUE(assertAggregated(mock, s, root, "small6")); } remove(s, root, 40); mock.erase(40); - EXPECT_EQUAL(103u, s.size(root)); + EXPECT_EQ(103u, s.size(root)); EXPECT_TRUE(!s.isSmallArray(root)); - TEST_DO(EXPECT_TRUE(assertAggregated(mock, s, root))); + EXPECT_TRUE(assertAggregated(mock, s, root, "small7")); remove(s, root, 20); mock.erase(20); - EXPECT_EQUAL(102u, s.size(root)); + EXPECT_EQ(102u, s.size(root)); EXPECT_TRUE(!s.isSmallArray(root)); - TEST_DO(EXPECT_TRUE(assertAggregated(mock, s, root))); + EXPECT_TRUE(assertAggregated(mock, s, root, "small8")); remove(s, root, 50); mock.erase(50); - EXPECT_EQUAL(101u, s.size(root)); + EXPECT_EQ(101u, s.size(root)); EXPECT_TRUE(!s.isSmallArray(root)); - TEST_DO(EXPECT_TRUE(assertAggregated(mock, s, root))); + EXPECT_TRUE(assertAggregated(mock, s, root, "small9")); for (uint32_t i = 0; i < 100; ++i) { remove(s, root, 1000 + i); mock.erase(1000 + i); - EXPECT_EQUAL(100 - i, s.size(root)); - EXPECT_EQUAL(100 - i <= MyTreeStore::clusterLimit, s.isSmallArray(root)); - TEST_DO(EXPECT_TRUE(assertAggregated(mock, s, root))); + EXPECT_EQ(100 - i, s.size(root)); + EXPECT_EQ(100 - i <= MyTreeStore::clusterLimit, s.isSmallArray(root)); + EXPECT_TRUE(assertAggregated(mock, s, root, "small10")); } - EXPECT_EQUAL(1u, s.size(root)); + EXPECT_EQ(1u, s.size(root)); EXPECT_TRUE(s.isSmallArray(root)); s.clear(root); @@ -1102,8 +1071,7 @@ Test::requireThatSmallNodesWorks() s.reclaim_memory(g.get_oldest_used_generation()); } -void -Test::requireThatFrozenViewProvidesAggregatedValues() +TEST_F(BTreeAggregationTest, require_that_frozen_view_provides_aggregated_values) { MyTree t; t.insert(20, 102); @@ -1114,39 +1082,13 @@ Test::requireThatFrozenViewProvidesAggregatedValues() t.getAllocator().freeze(); auto new_view = t.getFrozenView(); auto new_aggregated = new_view.getAggregated(); - EXPECT_EQUAL(new_aggregated.getMin(), 101); - EXPECT_EQUAL(new_aggregated.getMax(), 104); + EXPECT_EQ(new_aggregated.getMin(), 101); + EXPECT_EQ(new_aggregated.getMax(), 104); auto old_aggregated = old_view.getAggregated(); - EXPECT_EQUAL(old_aggregated.getMin(), std::numeric_limits<int32_t>::max()); - EXPECT_EQUAL(old_aggregated.getMax(), std::numeric_limits<int32_t>::min()); -} - -int -Test::Main() -{ - TEST_INIT("btreeaggregation_test"); - - requireThatNodeInsertWorks(); - keys_are_aggregated_correctly_on_node_insertions(); - requireThatNodeSplitInsertWorks(); - keys_are_aggregated_correctly_when_node_split_on_insert(); - requireThatTreeInsertWorks(); - requireThatNodeStealWorks(); - requireThatNodeRemoveWorks(); - keys_are_aggregated_correctly_on_node_removal(); - requireThatWeCanInsertAndRemoveFromTree(); - requireThatSortedTreeInsertWorks(); - requireThatCornerCaseTreeFindWorks(); - requireThatBasicTreeIteratorWorks(); - requireThatTreeIteratorAssignWorks(); - requireThatUpdateOfKeyWorks(); - requireThatUpdateOfDataWorks(); - TEST_DO(requireThatSmallNodesWorks()); - TEST_DO(requireThatFrozenViewProvidesAggregatedValues()); - - TEST_DONE(); + EXPECT_EQ(old_aggregated.getMin(), std::numeric_limits<int32_t>::max()); + EXPECT_EQ(old_aggregated.getMax(), std::numeric_limits<int32_t>::min()); } } -TEST_APPHOOK(vespalib::btree::Test); +GTEST_MAIN_RUN_ALL_TESTS() |