diff options
50 files changed, 737 insertions, 285 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java index ad6eb038058..7cb0a088f5f 100644 --- a/config-model/src/main/java/com/yahoo/schema/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java @@ -1019,6 +1019,9 @@ public class RankProfile implements Cloneable { var recorder = new InputRecorder(needInputs); recorder.transform(globalPhaseRanking.function().getBody(), context); for (String input : needInputs) { + if (input.startsWith("constant(") || input.startsWith("query(")) { + continue; + } try { addMatchFeatures(new FeatureList(input)); } catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) { diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java index b0f63ebb732..4e7988a2006 100644 --- a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java @@ -3,16 +3,13 @@ package com.yahoo.schema.expressiontransforms; import com.yahoo.schema.FeatureNames; import com.yahoo.schema.RankProfile; -import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.searchlib.rankingexpression.parser.ParseException; import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; -import java.io.StringReader; import java.util.Set; /** @@ -86,13 +83,7 @@ public class InputRecorder extends ExpressionTransformer<RankProfileTransformCon throw new IllegalArgumentException("missing onnx model: " + arg); } for (String onnxInput : model.getInputMap().values()) { - var reader = new StringReader(onnxInput); - try { - var asExpression = new RankingExpression(reader); - transform(asExpression.getRoot(), context); - } catch (ParseException e) { - throw new IllegalArgumentException("illegal onnx input '" + onnxInput + "': " + e.getMessage()); - } + neededInputs.add(onnxInput); } return; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 86c48407775..14c25ee7452 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -7,6 +7,7 @@ import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.search.pagetemplates.PageTemplatesConfig; import com.yahoo.search.query.profile.config.QueryProfilesConfig; +import com.yahoo.search.ranking.RankProfilesEvaluatorFactory; import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; @@ -56,6 +57,8 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> owningCluster.addComponent(Component.fromClassAndBundle(CompiledQueryProfileRegistry.class, SEARCH_AND_DOCPROC_BUNDLE)); owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.schema.SchemaInfo.class, SEARCH_AND_DOCPROC_BUNDLE)); owningCluster.addComponent(Component.fromClassAndBundle(SearchStatusExtension.class, SEARCH_AND_DOCPROC_BUNDLE)); + owningCluster.addComponent(Component.fromClassAndBundle(RankProfilesEvaluatorFactory.class, SEARCH_AND_DOCPROC_BUNDLE)); + owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.ranking.GlobalPhaseRanker.class, SEARCH_AND_DOCPROC_BUNDLE)); cluster.addSearchAndDocprocBundles(); } @@ -68,9 +71,16 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> /** Adds a Dispatcher component to the owning container cluster for each search cluster */ private void initializeDispatchers(Collection<SearchCluster> searchClusters) { for (SearchCluster searchCluster : searchClusters) { - if ( ! ( searchCluster instanceof IndexedSearchCluster)) continue; - var dispatcher = new DispatcherComponent((IndexedSearchCluster)searchCluster); - owningCluster.addComponent(dispatcher); + if (searchCluster instanceof IndexedSearchCluster indexed) { + var dispatcher = new DispatcherComponent(indexed); + owningCluster.addComponent(dispatcher); + for (var documentDb : indexed.getDocumentDbs()) { + var factory = new RankProfilesEvaluatorComponent(documentDb); + if (! owningCluster.getComponentsMap().containsKey(factory.getComponentId())) { + owningCluster.addComponent(factory); + } + } + } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankProfilesEvaluatorComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankProfilesEvaluatorComponent.java new file mode 100644 index 00000000000..75a2802ee53 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankProfilesEvaluatorComponent.java @@ -0,0 +1,49 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search; + +import com.yahoo.config.model.producer.AnyConfigProducer; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.search.ranking.RankProfilesEvaluator; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; +import com.yahoo.vespa.model.container.ContainerModelEvaluation; +import com.yahoo.vespa.model.container.PlatformBundles; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.search.DocumentDatabase; + +public class RankProfilesEvaluatorComponent + extends Component<AnyConfigProducer, ComponentModel> + implements + RankProfilesConfig.Producer, + RankingConstantsConfig.Producer, + RankingExpressionsConfig.Producer, + OnnxModelsConfig.Producer +{ + private final DocumentDatabase ddb; + + public RankProfilesEvaluatorComponent(DocumentDatabase db) { + super(toComponentModel(db.getSchemaName())); + ddb = db; + } + + private static ComponentModel toComponentModel(String p) { + String myComponentId = "ranking-expression-evaluator." + p; + return new ComponentModel(myComponentId, + RankProfilesEvaluator.class.getName(), + PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE); + } + + @Override + public void getConfig(RankProfilesConfig.Builder builder) { ddb.getConfig(builder); } + + @Override + public void getConfig(RankingExpressionsConfig.Builder builder) { ddb.getConfig(builder); } + + @Override + public void getConfig(RankingConstantsConfig.Builder builder) { ddb.getConfig(builder); } + + @Override + public void getConfig(OnnxModelsConfig.Builder builder) { ddb.getConfig(builder); } +} diff --git a/config-model/src/test/derived/globalphase_onnx_inside/files/ax_plus_b.onnx b/config-model/src/test/derived/globalphase_onnx_inside/files/ax_plus_b.onnx new file mode 100644 index 00000000000..17282d13dc3 --- /dev/null +++ b/config-model/src/test/derived/globalphase_onnx_inside/files/ax_plus_b.onnx @@ -0,0 +1,23 @@ +:© + +matrix_X +vector_AXA"MatMul + +XA +vector_Bvector_Y"AddlrZ +matrix_X + + +Z +vector_A + + +Z +vector_B + + +b +vector_Y + + +B
\ No newline at end of file diff --git a/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg b/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg new file mode 100644 index 00000000000..35bb1ccc3d2 --- /dev/null +++ b/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg @@ -0,0 +1,34 @@ +rankprofile[].name "default" +rankprofile[].fef.property[].name "rankingExpression(handicap).rankingScript" +rankprofile[].fef.property[].value "query(yy)" +rankprofile[].fef.property[].name "rankingExpression(handicap).type" +rankprofile[].fef.property[].value "tensor(d0[2])" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "rankingExpression(firstphase)" +rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" +rankprofile[].fef.property[].value "reduce(attribute(aa), sum)" +rankprofile[].fef.property[].name "vespa.rank.globalphase" +rankprofile[].fef.property[].value "rankingExpression(globalphase)" +rankprofile[].fef.property[].name "rankingExpression(globalphase).rankingScript" +rankprofile[].fef.property[].value "reduce(constant(ww) * (onnx(inside).foobar - rankingExpression(handicap)), sum)" +rankprofile[].fef.property[].name "vespa.match.feature" +rankprofile[].fef.property[].value "attribute(aa)" +rankprofile[].fef.property[].name "vespa.globalphase.rerankcount" +rankprofile[].fef.property[].value "13" +rankprofile[].fef.property[].name "vespa.type.attribute.aa" +rankprofile[].fef.property[].value "tensor(d1[3])" +rankprofile[].fef.property[].name "vespa.type.query.bb" +rankprofile[].fef.property[].value "tensor(d0[2])" +rankprofile[].fef.property[].name "vespa.type.query.yy" +rankprofile[].fef.property[].value "tensor(d0[2])" +rankprofile[].name "unranked" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "value(0)" +rankprofile[].fef.property[].name "vespa.hitcollector.heapsize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.hitcollector.arraysize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.dump.ignoredefaultfeatures" +rankprofile[].fef.property[].value "true" +rankprofile[].fef.property[].name "vespa.type.attribute.aa" +rankprofile[].fef.property[].value "tensor(d1[3])" diff --git a/config-model/src/test/derived/globalphase_onnx_inside/test.sd b/config-model/src/test/derived/globalphase_onnx_inside/test.sd new file mode 100644 index 00000000000..c38e318ce6b --- /dev/null +++ b/config-model/src/test/derived/globalphase_onnx_inside/test.sd @@ -0,0 +1,42 @@ +schema test { + + document test { + field aa type tensor(d1[3]) { + indexing: attribute + } + } + + constant xx { + file: files/const_xx.json + type: tensor(d0[2],d1[3]) + } + constant ww { + file: files/const_ww.json + type: tensor(d0[2]) + } + + rank-profile default { + inputs { + query(bb) tensor(d0[2]) + query(yy) tensor(d0[2]) + } + onnx-model inside { + file: files/ax_plus_b.onnx + input vector_A: attribute(aa) + input matrix_X: constant(xx) + input vector_B: query(bb) + output vector_Y: foobar + } + first-phase { + expression: sum(attribute(aa)) + } + function handicap() { + expression: query(yy) + } + global-phase { + rerank-count: 13 + expression: sum(constant(ww) * (onnx(inside).foobar - handicap)) + } + } + +} diff --git a/config-model/src/test/derived/rankingexpression/rank-profiles.cfg b/config-model/src/test/derived/rankingexpression/rank-profiles.cfg index ea8bc5f77e6..e3947e9e46f 100644 --- a/config-model/src/test/derived/rankingexpression/rank-profiles.cfg +++ b/config-model/src/test/derived/rankingexpression/rank-profiles.cfg @@ -410,8 +410,6 @@ rankprofile[].fef.property[].value "rankingExpression(globalphase)" rankprofile[].fef.property[].name "rankingExpression(globalphase).rankingScript" rankprofile[].fef.property[].value "rankingExpression(myplus) + reduce(rankingExpression(mymul), sum) + firstPhase" rankprofile[].fef.property[].name "vespa.match.feature" -rankprofile[].fef.property[].value "query(fromq)" -rankprofile[].fef.property[].name "vespa.match.feature" rankprofile[].fef.property[].value "firstPhase" rankprofile[].fef.property[].name "vespa.match.feature" rankprofile[].fef.property[].value "attribute(t1)" diff --git a/config-model/src/test/java/com/yahoo/schema/derived/GlobalPhaseOnnxModelsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/GlobalPhaseOnnxModelsTestCase.java new file mode 100644 index 00000000000..2ff33dd70d8 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/GlobalPhaseOnnxModelsTestCase.java @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.jupiter.api.Test; +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Tests exporting with global-phase and ONNX models + * + * @author arnej + */ +public class GlobalPhaseOnnxModelsTestCase extends AbstractExportingTestCase { + + @Test + void testModelInRankProfile() throws IOException, ParseException { + assertCorrectDeriving("globalphase_onnx_inside"); + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 729aebf2fc2..7787d7d7702 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -20,6 +20,7 @@ import com.yahoo.search.Searcher; import com.yahoo.search.config.ClusterConfig; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.query.ParameterParser; +import com.yahoo.search.ranking.GlobalPhaseRanker; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; @@ -64,6 +65,7 @@ public class ClusterSearcher extends Searcher { private final VespaBackEndSearcher server; private final Executor executor; + private final GlobalPhaseRanker globalPhaseHelper; @Inject public ClusterSearcher(ComponentId id, @@ -73,10 +75,12 @@ public class ClusterSearcher extends Searcher { DocumentdbInfoConfig documentDbConfig, SchemaInfo schemaInfo, ComponentRegistry<Dispatcher> dispatchers, + GlobalPhaseRanker globalPhaseHelper, VipStatus vipStatus, VespaDocumentAccess access) { super(id); this.executor = executor; + this.globalPhaseHelper = globalPhaseHelper; int searchClusterIndex = clusterConfig.clusterId(); searchClusterName = clusterConfig.clusterName(); QrSearchersConfig.Searchcluster searchClusterConfig = getSearchClusterConfigFromClusterName(qrsConfig, searchClusterName); @@ -159,7 +163,9 @@ public class ClusterSearcher extends Searcher { maxQueryCacheTimeout = DEFAULT_MAX_QUERY_CACHE_TIMEOUT; server = searcher; this.executor = executor; + this.globalPhaseHelper = null; } + /** Do not use, for internal testing purposes only. **/ ClusterSearcher(Set<String> schemas) { this(schemas, null, null); @@ -169,7 +175,7 @@ public class ClusterSearcher extends Searcher { public void fill(com.yahoo.search.Result result, String summaryClass, Execution execution) { Query query = result.getQuery(); - VespaBackEndSearcher searcher = server; + Searcher searcher = server; if (searcher != null) { if (query.getTimeLeft() > 0) { searcher.fill(result, summaryClass, execution); @@ -190,7 +196,7 @@ public class ClusterSearcher extends Searcher { public Result search(Query query, Execution execution) { validateQueryTimeout(query); validateQueryCache(query); - VespaBackEndSearcher searcher = server; + Searcher searcher = server; if (searcher == null) { return new Result(query, ErrorMessage.createNoBackendsInService("Could not search")); } @@ -228,8 +234,21 @@ public class ClusterSearcher extends Searcher { } else { String docType = schemas.iterator().next(); query.getModel().setRestrict(docType); - return searcher.search(query, execution); + return perSchemaSearch(searcher, query, execution); + } + } + + private Result perSchemaSearch(Searcher searcher, Query query, Execution execution) { + Set<String> restrict = query.getModel().getRestrict(); + if (restrict.size() != 1) { + throw new IllegalStateException("perSchemaSearch must always be called with 1 schema, got: " + restrict.size()); + } + String schema = restrict.iterator().next(); + Result result = searcher.search(query, execution); + if (globalPhaseHelper != null) { + globalPhaseHelper.process(query, result, schema); } + return result; } private static void processResult(Query query, FutureTask<Result> task, Result mergedResult) { @@ -248,12 +267,12 @@ public class ClusterSearcher extends Searcher { Set<String> schemas = resolveSchemas(query, execution.context().getIndexFacts()); List<Query> queries = createQueries(query, schemas); if (queries.size() == 1) { - return searcher.search(queries.get(0), execution); + return perSchemaSearch(searcher, queries.get(0), execution); } else { Result mergedResult = new Result(query); List<FutureTask<Result>> pending = new ArrayList<>(queries.size()); for (Query q : queries) { - FutureTask<Result> task = new FutureTask<>(() -> searcher.search(q, execution)); + FutureTask<Result> task = new FutureTask<>(() -> perSchemaSearch(searcher, q, execution)); try { executor.execute(task); pending.add(task); diff --git a/container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java new file mode 100644 index 00000000000..d2edb776c92 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java @@ -0,0 +1,14 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import com.yahoo.tensor.Tensor; + +import java.util.Collection; + +interface Evaluator { + Collection<String> needInputs(); + + Evaluator bind(String name, Tensor value); + + double evaluateScore(); +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java new file mode 100644 index 00000000000..87213362acd --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java @@ -0,0 +1,118 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import ai.vespa.models.evaluation.FunctionEvaluator; +import ai.vespa.models.evaluation.Model; +import com.yahoo.component.annotation.Inject; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.tensor.Tensor; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Logger; +import java.util.function.Supplier; + +public class GlobalPhaseRanker { + + private static final Logger logger = Logger.getLogger(GlobalPhaseRanker.class.getName()); + private final RankProfilesEvaluatorFactory factory; + private final Set<String> skipProcessing = new HashSet<>(); + private final Map<String, Supplier<FunctionEvaluator>> scorers = new HashMap<>(); + + @Inject + public GlobalPhaseRanker(RankProfilesEvaluatorFactory factory) { + this.factory = factory; + logger.info("using factory: " + factory); + } + + public void process(Query query, Result result, String schema) { + var functionEvaluatorSource = underlying(query, schema); + if (functionEvaluatorSource == null) { + return; + } + var prepared = findFromQuery(query, functionEvaluatorSource.get().function().arguments()); + Supplier<Evaluator> supplier = () -> { + var evaluator = functionEvaluatorSource.get(); + var simple = new SimpleEvaluator(evaluator); + for (var entry : prepared) { + simple.bind(entry.name(), entry.value()); + } + return simple; + }; + // TODO need to get rerank-count somehow + int rerank = 7; + ResultReranker.rerankHits(result, new HitRescorer(supplier), rerank); + } + + record NameAndValue(String name, Tensor value) { } + + /* do this only once per query: */ + List<NameAndValue> findFromQuery(Query query, List<String> needInputs) { + List<NameAndValue> result = new ArrayList<>(); + var ranking = query.getRanking(); + var rankFeatures = ranking.getFeatures(); + var rankProps = ranking.getProperties().asMap(); + for (String needed : needInputs) { + var optRef = com.yahoo.searchlib.rankingexpression.Reference.simple(needed); + if (optRef.isEmpty()) continue; + var ref = optRef.get(); + if (ref.name().equals("constant")) { + // XXX in theory, we should be able to avoid this + result.add(new NameAndValue(needed, null)); + continue; + } + if (ref.isSimple() && ref.name().equals("query")) { + String queryFeatureName = ref.simpleArgument().get(); + // searchers are recommended to place query features here: + var feature = rankFeatures.getTensor(queryFeatureName); + if (feature.isPresent()) { + result.add(new NameAndValue(needed, feature.get())); + } else { + // but other ways of setting query features end up in the properties: + var objList = rankProps.get(queryFeatureName); + if (objList != null && objList.size() == 1 && objList.get(0) instanceof Tensor t) { + result.add(new NameAndValue(needed, t)); + } + } + } + } + return result; + } + + private Supplier<FunctionEvaluator> underlying(Query query, String schema) { + String rankProfile = query.getRanking().getProfile(); + String key = schema + " with rank profile " + rankProfile; + if (skipProcessing.contains(key)) { + return null; + } + Supplier<FunctionEvaluator> supplier = scorers.get(key); + if (supplier != null) { + return supplier; + } + try { + var proxy = factory.proxyForSchema(schema); + var model = proxy.modelForRankProfile(rankProfile); + supplier = () -> model.evaluatorOf("globalphase"); + if (supplier.get() == null) { + supplier = null; + } + } catch (IllegalArgumentException e) { + logger.info("no global-phase for " + key + " because: " + e.getMessage()); + supplier = null; + } + if (supplier == null) { + skipProcessing.add(key); + } else { + scorers.put(key, supplier); + } + return supplier; + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java b/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java new file mode 100644 index 00000000000..ebdbbb693f1 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java @@ -0,0 +1,56 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import com.yahoo.search.result.FeatureData; +import com.yahoo.search.result.Hit; + +import java.util.function.Supplier; +import java.util.logging.Logger; + +class HitRescorer { + + private static final Logger logger = Logger.getLogger(HitRescorer.class.getName()); + + private final Supplier<Evaluator> evaluatorSource; + + public HitRescorer(Supplier<Evaluator> evaluatorSource) { + this.evaluatorSource = evaluatorSource; + } + + boolean rescoreHit(Hit hit) { + var features = hit.getField("matchfeatures"); + if (features instanceof FeatureData matchFeatures) { + var scorer = evaluatorSource.get(); + for (String argName : scorer.needInputs()) { + var asTensor = matchFeatures.getTensor(argName); + if (asTensor == null) { + asTensor = matchFeatures.getTensor(alternate(argName)); + } + if (asTensor != null) { + scorer.bind(argName, asTensor); + } else { + logger.warning("Missing match-feature for Evaluator argument: " + argName); + return false; + } + } + double newScore = scorer.evaluateScore(); + hit.setRelevance(newScore); + return true; + } else { + logger.warning("Hit without match-features: " + hit); + return false; + } + } + + private static final String RE_PREFIX = "rankingExpression("; + private static final String RE_SUFFIX = ")"; + private static final int RE_PRE_LEN = RE_PREFIX.length(); + private static final int RE_SUF_LEN = RE_SUFFIX.length(); + + static String alternate(String argName) { + if (argName.startsWith(RE_PREFIX) && argName.endsWith(RE_SUFFIX)) { + return argName.substring(RE_PRE_LEN, argName.length() - RE_SUF_LEN); + } + return argName; + } +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java new file mode 100644 index 00000000000..ccb9b9837fe --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java @@ -0,0 +1,53 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.ranking; + +import ai.vespa.models.evaluation.FunctionEvaluator; +import ai.vespa.models.evaluation.Model; +import ai.vespa.models.evaluation.ModelsEvaluator; +import com.yahoo.api.annotations.Beta; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.annotation.Inject; +import com.yahoo.filedistribution.fileacquirer.FileAcquirer; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; + +/** + * proxy for model-evaluation components + * @author arnej + */ +@Beta +public class RankProfilesEvaluator extends AbstractComponent { + + private final ModelsEvaluator evaluator; + + @Inject + public RankProfilesEvaluator( + RankProfilesConfig rankProfilesConfig, + RankingConstantsConfig constantsConfig, + RankingExpressionsConfig expressionsConfig, + OnnxModelsConfig onnxModelsConfig, + FileAcquirer fileAcquirer) + { + this.evaluator = new ModelsEvaluator( + rankProfilesConfig, + constantsConfig, + expressionsConfig, + onnxModelsConfig, + fileAcquirer); + } + + public Model modelForRankProfile(String rankProfile) { + var m = evaluator.models().get(rankProfile); + if (m == null) { + throw new IllegalArgumentException("unknown rankprofile: " + rankProfile); + } + return m; + } + + public FunctionEvaluator evaluatorForFunction(String rankProfile, String functionName) { + return modelForRankProfile(rankProfile).evaluatorOf(functionName); + } +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluatorFactory.java b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluatorFactory.java new file mode 100644 index 00000000000..edb05ed9788 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluatorFactory.java @@ -0,0 +1,40 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.ranking; + +import com.yahoo.api.annotations.Beta; +import com.yahoo.component.annotation.Inject; +import com.yahoo.component.provider.ComponentRegistry; + +/** + * factory for model-evaluation proxies + * @author arnej + */ +@Beta +public class RankProfilesEvaluatorFactory { + + private final ComponentRegistry<RankProfilesEvaluator> registry; + + @Inject + public RankProfilesEvaluatorFactory(ComponentRegistry<RankProfilesEvaluator> registry) { + this.registry = registry; + } + + public RankProfilesEvaluator proxyForSchema(String schemaName) { + var component = registry.getComponent("ranking-expression-evaluator." + schemaName); + if (component == null) { + throw new IllegalArgumentException("ranking expression evaluator for schema '" + schemaName + "' not found"); + } + return component; + } + + public String toString() { + var buf = new StringBuilder(); + buf.append(this.getClass().getName()).append(" containing: ["); + for (var id : registry.allComponentsById().keySet()) { + buf.append(" ").append(id.toString()); + } + buf.append(" ]"); + return buf.toString(); + } +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java b/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java new file mode 100644 index 00000000000..11b3fa7390a --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java @@ -0,0 +1,91 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.logging.Logger; + +class ResultReranker { + + private static final Logger logger = Logger.getLogger(ResultReranker.class.getName()); + + // scale and adjust the score according to the range + // of the original and final score values to avoid that + // a score from the backend is larger than finalScores_low + static class Ranges { + private double initialScores_high = -Double.MAX_VALUE; + private double initialScores_low = Double.MAX_VALUE; + private double finalScores_high = -Double.MAX_VALUE; + private double finalScores_low = Double.MAX_VALUE; + + boolean valid() { + return (initialScores_high >= initialScores_low + && + finalScores_high >= finalScores_low); + } + void withInitialScore(double score) { + if (score < initialScores_low) initialScores_low = score; + if (score > initialScores_high) initialScores_high = score; + } + void withFinalScore(double score) { + if (score < finalScores_low) finalScores_low = score; + if (score > finalScores_high) finalScores_high = score; + } + private double initialRange() { + double r = initialScores_high - initialScores_low; + if (r < 1.0) r = 1.0; + return r; + } + private double finalRange() { + double r = finalScores_high - finalScores_low; + if (r < 1.0) r = 1.0; + return r; + } + double scale() { return finalRange() / initialRange(); } + double bias() { return finalScores_low - initialScores_low * scale(); } + } + + static void rerankHits(Result result, HitRescorer hitRescorer, int rerankCount) { + List<Hit> hitsToRescore = new ArrayList<>(); + // consider doing recursive iteration explicitly instead of using deepIterator? + for (var iterator = result.hits().deepIterator(); iterator.hasNext();) { + Hit hit = iterator.next(); + if (hit.isMeta() || hit instanceof HitGroup) { + continue; + } + // what about hits inside grouping results? + // they are inside GroupingListHit, we won't recurse into it; so we won't see them. + hitsToRescore.add(hit); + } + // we can't be 100% certain that hits were sorted according to relevance: + hitsToRescore.sort(Comparator.naturalOrder()); + var ranges = new Ranges(); + for (var iterator = hitsToRescore.iterator(); rerankCount > 0 && iterator.hasNext(); ) { + Hit hit = iterator.next(); + double oldScore = hit.getRelevance().getScore(); + boolean didRerank = hitRescorer.rescoreHit(hit); + if (didRerank) { + ranges.withInitialScore(oldScore); + ranges.withFinalScore(hit.getRelevance().getScore()); + --rerankCount; + iterator.remove(); + } + } + // if any hits are left in the list, they need rescaling: + if (ranges.valid()) { + double scale = ranges.scale(); + double bias = ranges.bias(); + for (Hit hit : hitsToRescore) { + double oldScore = hit.getRelevance().getScore(); + hit.setRelevance(oldScore * scale + bias); + } + } + result.hits().sort(); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java new file mode 100644 index 00000000000..f247eab1649 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java @@ -0,0 +1,51 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import ai.vespa.models.evaluation.FunctionEvaluator; +import com.yahoo.search.result.FeatureData; +import com.yahoo.search.result.Hit; +import com.yahoo.tensor.Tensor; + +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +class SimpleEvaluator implements Evaluator { + + private final FunctionEvaluator evaluator; + private final Set<String> neededInputs; + + public SimpleEvaluator(FunctionEvaluator prototype) { + this.evaluator = prototype; + this.neededInputs = new HashSet<String>(prototype.function().arguments()); + } + + @Override + public Collection<String> needInputs() { return List.copyOf(neededInputs); } + + @Override + public SimpleEvaluator bind(String name, Tensor value) { + if (value != null) evaluator.bind(name, value); + neededInputs.remove(name); + return this; + } + + @Override + public double evaluateScore() { + return evaluator.evaluate().asDouble(); + } + + @Override + public String toString() { + var buf = new StringBuilder(); + buf.append("SimpleEvaluator("); + buf.append(evaluator.function().toString()); + buf.append(")["); + for (String arg : neededInputs) { + buf.append("{").append(arg).append("}"); + } + buf.append("]"); + return buf.toString(); + } +} diff --git a/fastos/src/tests/mazeserver.cpp b/container-search/src/main/java/com/yahoo/search/ranking/package-info.java index b62e98645f2..a86a5c1e52f 100644 --- a/fastos/src/tests/mazeserver.cpp +++ b/container-search/src/main/java/com/yahoo/search/ranking/package-info.java @@ -1,4 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#define DO_MAZE_SERVER 1 -#include "sockettest.cpp" +@ExportPackage +package com.yahoo.search.ranking; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index 5df8d2e5444..06ae9923dae 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -464,6 +464,7 @@ public class ClusterSearcherTestCase { documentDbConfig.build(), new SchemaInfo(List.of(schema.build()), Map.of()), dispatchers, + null, vipStatus, null); } diff --git a/fastos/CMakeLists.txt b/fastos/CMakeLists.txt index c17752e234c..60813c569e4 100644 --- a/fastos/CMakeLists.txt +++ b/fastos/CMakeLists.txt @@ -4,5 +4,4 @@ vespa_define_module( src/vespa/fastos TESTS - src/tests ) diff --git a/fastos/src/tests/.gitignore b/fastos/src/tests/.gitignore deleted file mode 100644 index ccb5f0210ab..00000000000 --- a/fastos/src/tests/.gitignore +++ /dev/null @@ -1,17 +0,0 @@ -/Makefile -/backtracetest -/backtracetest.log -/filetest -/filetest.log -/processtest -/processtest.log -/sockettest -/sockettest.log -/threadtest -/threadtest.log -/timetest -/timetest.log -/typetest -/typetest.log -/usecputest -*test_app diff --git a/fastos/src/tests/CMakeLists.txt b/fastos/src/tests/CMakeLists.txt deleted file mode 100644 index 7d4b014b6b3..00000000000 --- a/fastos/src/tests/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(fastos_filetest_app TEST - SOURCES - filetest.cpp - DEPENDS - fastos -) -vespa_add_test(NAME fastos_filetest_app NO_VALGRIND COMMAND fastos_filetest_app) -vespa_add_executable(fastos_typetest_app TEST - SOURCES - typetest.cpp - DEPENDS - fastos -) -vespa_add_test(NAME fastos_typetest_app NO_VALGRIND COMMAND fastos_typetest_app) diff --git a/fastos/src/tests/coretest2.cpp b/fastos/src/tests/coretest2.cpp deleted file mode 100644 index bd93623922b..00000000000 --- a/fastos/src/tests/coretest2.cpp +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - - - - -static void -bomb(void) -{ - char *p; - - p = nullptr; - *p = 4; -} - -void *BomberRun(void *arg) -{ - (void) arg; - bomb(); - return nullptr; -}; - -static int -bombMain(void) -{ - pthread_t thread; - void *ret; - - (void) pthread_create(&thread, nullptr, BomberRun, nullptr); - ret = nullptr; - (void) pthread_join(thread, &ret); - return (0); -} - - -int -main(int argc, char **argv) -{ - (void) argc; - (void) argv; - return bombMain(); -} diff --git a/fastos/src/tests/performancetest.cpp b/fastos/src/tests/performancetest.cpp deleted file mode 100644 index f566979957a..00000000000 --- a/fastos/src/tests/performancetest.cpp +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <stdlib.h> - -#include "tests.h" - -void PerformanceTest (char *buffer); - -int main (int argc, char **argv) -{ - (void)argc; - (void)argv; - - void (*test)(char *buffer) = PerformanceTest; - - test(nullptr); - return 0; -} - -void PerformanceTest (char *buffer) -{ - // Cause exception - *static_cast<char *>(nullptr) = 'e'; - -#if 1 - FastOS_File file("test.txt"); - - if(file.OpenReadOnly()) - { - file.Read(buffer, 20); - file.Write2(buffer, 20); - file.Read(buffer, 20); - file.Write2(buffer, 20); - file.Read(buffer, 20); - file.Write2(buffer, 20); - } -#else - - int filedes = open("test.txt", O_RDONLY, 0664); - - if(filedes != -1) - { - write(filedes, buffer, 20); - read(filedes, buffer, 20); - write(filedes, buffer, 20); - read(filedes, buffer, 20); - write(filedes, buffer, 20); - read(filedes, buffer, 20); - write(filedes, buffer, 20); - - close(filedes); - } -#endif -} - diff --git a/fastos/src/tests/typetest.cpp b/fastos/src/tests/typetest.cpp deleted file mode 100644 index 7871275e17c..00000000000 --- a/fastos/src/tests/typetest.cpp +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "tests.h" -#include <vespa/fastos/file.h> - -class TypeTest : public BaseTest -{ -private: - - void ObjectSizeTest () - { - TestHeader("Object Sizes (bytes)"); - - Progress(true, "FastOS_DirectoryScan %d", sizeof(FastOS_DirectoryScan)); - Progress(true, "FastOS_File: %d", sizeof(FastOS_File)); - Progress(true, "FastOS_StatInfo %d", sizeof(FastOS_StatInfo)); - - PrintSeparator(); - } - -public: - virtual ~TypeTest() {}; - - int Main () override - { - printf("grep for the string '%s' to detect failures.\n\n", failString); - - ObjectSizeTest(); - - PrintSeparator(); - printf("END OF TEST (%s)\n", _argv[0]); - - return allWasOk() ? 0 : 1; - } -}; - - -int main (int argc, char **argv) -{ - setvbuf(stdout, nullptr, _IOLBF, 8192); - TypeTest app; - return app.Entry(argc, argv); -} - diff --git a/fastos/src/vespa/fastos/CMakeLists.txt b/fastos/src/vespa/fastos/CMakeLists.txt index c37eb43b884..29810a4f296 100644 --- a/fastos/src/vespa/fastos/CMakeLists.txt +++ b/fastos/src/vespa/fastos/CMakeLists.txt @@ -1,10 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(fastos_objects OBJECT SOURCES - file.cpp - file_rw_ops.cpp - linux_file.cpp - unix_file.cpp ) vespa_add_library(fastos diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp index da7b8f71fdf..9135dd86a16 100644 --- a/metrics/src/vespa/metrics/metricmanager.cpp +++ b/metrics/src/vespa/metrics/metricmanager.cpp @@ -25,6 +25,7 @@ using Config = MetricsmanagerConfig; using vespalib::IllegalStateException; using vespalib::IllegalArgumentException; using vespalib::make_string_short::fmt; +using vespalib::count_ms; MetricManager::ConsumerSpec::ConsumerSpec() = default; MetricManager::ConsumerSpec::~ConsumerSpec() = default; @@ -654,7 +655,7 @@ MetricManager::updatePeriodicMetrics(const MetricLockGuard & guard, time_t updat { assertMetricLockLocked(guard); time_t nextUpdateTime = std::numeric_limits<time_t>::max(); - time_t preTime = _timer->getTimeInMilliSecs(); + time_point preTime = _timer->getTimeInMilliSecs(); for (auto hook : _periodicUpdateHooks) { if (hook->_nextCall <= updateTime) { hook->updateMetrics(guard); @@ -668,13 +669,13 @@ MetricManager::updatePeriodicMetrics(const MetricLockGuard & guard, time_t updat } else { hook->_nextCall += hook->_period; } - time_t postTime = _timer->getTimeInMilliSecs(); - _periodicHookLatency.addValue(postTime - preTime); + time_point postTime = _timer->getTimeInMilliSecs(); + _periodicHookLatency.addValue(count_ms(postTime - preTime)); preTime = postTime; } else if (outOfSchedule) { hook->updateMetrics(guard); - time_t postTime = _timer->getTimeInMilliSecs(); - _periodicHookLatency.addValue(postTime - preTime); + time_point postTime = _timer->getTimeInMilliSecs(); + _periodicHookLatency.addValue(count_ms(postTime - preTime)); preTime = postTime; } nextUpdateTime = std::min(nextUpdateTime, hook->_nextCall); @@ -687,11 +688,11 @@ void MetricManager::updateSnapshotMetrics(const MetricLockGuard & guard) { assertMetricLockLocked(guard); - time_t preTime = _timer->getTimeInMilliSecs(); + time_point preTime = _timer->getTimeInMilliSecs(); for (const auto & hook : _snapshotUpdateHooks) { hook->updateMetrics(guard); - time_t postTime = _timer->getTimeInMilliSecs(); - _snapshotHookLatency.addValue(postTime - preTime); + time_point postTime = _timer->getTimeInMilliSecs(); + _snapshotHookLatency.addValue(count_ms(postTime - preTime)); preTime = postTime; } } @@ -708,7 +709,7 @@ MetricManager::forceEventLogging() void MetricManager::reset(time_t currentTime) { - time_t preTime = _timer->getTimeInMilliSecs(); + time_point preTime = _timer->getTimeInMilliSecs(); // Resetting implies visiting metrics, which needs to grab metric lock // to avoid conflict with adding/removal of metrics std::lock_guard waiterLock(_waiter); @@ -717,8 +718,8 @@ MetricManager::reset(time_t currentTime) snapshot->reset(currentTime); } _totalMetrics->reset(currentTime); - time_t postTime = _timer->getTimeInMilliSecs(); - _resetLatency.addValue(postTime - preTime); + time_point postTime = _timer->getTimeInMilliSecs(); + _resetLatency.addValue(count_ms(postTime - preTime)); } void @@ -806,7 +807,7 @@ MetricManager::takeSnapshots(const MetricLockGuard & guard, time_t timeToProcess static_cast<uint64_t>(_snapshots[0]->getToTime())); return; } - time_t preTime = _timer->getTimeInMilliSecs(); + time_point preTime = _timer->getTimeInMilliSecs(); LOG(debug, "Updating %s snapshot and total metrics at time %" PRIu64 ".", _snapshots[0]->getName().c_str(), static_cast<uint64_t>(timeToProcess)); MetricSnapshot& firstTarget(_snapshots[0]->getNextTarget()); @@ -838,8 +839,8 @@ MetricManager::takeSnapshots(const MetricLockGuard & guard, time_t timeToProcess _snapshots[i]->getName().c_str(), static_cast<uint64_t>(timeToProcess)); } } - time_t postTime = _timer->getTimeInMilliSecs(); - _snapshotLatency.addValue(postTime - preTime); + time_point postTime = _timer->getTimeInMilliSecs(); + _snapshotLatency.addValue(count_ms(postTime - preTime)); } MemoryConsumption::UP diff --git a/metrics/src/vespa/metrics/metricmanager.h b/metrics/src/vespa/metrics/metricmanager.h index 2c595186ea4..3c81d9abd6a 100644 --- a/metrics/src/vespa/metrics/metricmanager.h +++ b/metrics/src/vespa/metrics/metricmanager.h @@ -68,7 +68,7 @@ public: struct Timer { virtual ~Timer() = default; virtual time_t getTime() const; - virtual time_t getTimeInMilliSecs() const { return getTime() * 1000; } + time_point getTimeInMilliSecs() const { return time_point(vespalib::from_s(getTime())); } }; /** diff --git a/metrics/src/vespa/metrics/updatehook.h b/metrics/src/vespa/metrics/updatehook.h index 9fa0d52027e..f355197bbb7 100644 --- a/metrics/src/vespa/metrics/updatehook.h +++ b/metrics/src/vespa/metrics/updatehook.h @@ -1,10 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <vespa/vespalib/util/time.h> #include <mutex> namespace metrics { +using time_point = vespalib::steady_time; + class MetricLockGuard { public: MetricLockGuard(std::mutex & mutex); diff --git a/storage/src/tests/common/metricstest.cpp b/storage/src/tests/common/metricstest.cpp index 78fa32e24e5..76c6a9cf991 100644 --- a/storage/src/tests/common/metricstest.cpp +++ b/storage/src/tests/common/metricstest.cpp @@ -53,7 +53,6 @@ namespace { framework::Clock& _clock; explicit MetricClock(framework::Clock& c) : _clock(c) {} [[nodiscard]] time_t getTime() const override { return vespalib::count_s(_clock.getMonotonicTime().time_since_epoch()); } - [[nodiscard]] time_t getTimeInMilliSecs() const override { return vespalib::count_ms(_clock.getMonotonicTime().time_since_epoch()); } }; } @@ -85,10 +84,7 @@ void MetricsTest::SetUp() { _metricManager->registerMetric(guard, *_topSet); } - _metricsConsumer = std::make_unique<StatusMetricConsumer>( - _node->getComponentRegister(), - *_metricManager, - "status"); + _metricsConsumer = std::make_unique<StatusMetricConsumer>(_node->getComponentRegister(), *_metricManager, "status"); _filestorMetrics = std::make_shared<FileStorMetrics>(); _filestorMetrics->initDiskMetrics(1, 1); diff --git a/storage/src/tests/storageserver/statereportertest.cpp b/storage/src/tests/storageserver/statereportertest.cpp index d6b528e5a25..2570b5232eb 100644 --- a/storage/src/tests/storageserver/statereportertest.cpp +++ b/storage/src/tests/storageserver/statereportertest.cpp @@ -54,7 +54,6 @@ struct MetricClock : public metrics::MetricManager::Timer framework::Clock& _clock; explicit MetricClock(framework::Clock& c) : _clock(c) {} [[nodiscard]] time_t getTime() const override { return vespalib::count_s(_clock.getMonotonicTime().time_since_epoch()); } - [[nodiscard]] time_t getTimeInMilliSecs() const override { return vespalib::count_ms(_clock.getMonotonicTime().time_since_epoch()); } }; } @@ -85,11 +84,8 @@ void StateReporterTest::SetUp() { _metricManager->registerMetric(guard, *_topSet); } - _stateReporter = std::make_unique<StateReporter>( - _node->getComponentRegister(), - *_metricManager, - _generationFetcher, - "status"); + _stateReporter = std::make_unique<StateReporter>(_node->getComponentRegister(), *_metricManager, + _generationFetcher, "status"); _filestorMetrics = std::make_shared<FileStorMetrics>(); _filestorMetrics->initDiskMetrics(1, 1); @@ -125,20 +121,14 @@ vespalib::Slime slime; \ #define ASSERT_GENERATION(jsonData, component, generation) \ { \ PARSE_JSON(jsonData); \ - ASSERT_EQ( \ - generation, \ - slime.get()["config"][component]["generation"].asDouble()); \ + ASSERT_EQ(generation, slime.get()["config"][component]["generation"].asDouble()); \ } #define ASSERT_NODE_STATUS(jsonData, code, message) \ { \ PARSE_JSON(jsonData); \ - ASSERT_EQ( \ - vespalib::string(code), \ - slime.get()["status"]["code"].asString().make_string()); \ - ASSERT_EQ( \ - vespalib::string(message), \ - slime.get()["status"]["message"].asString().make_string()); \ + ASSERT_EQ(vespalib::string(code), slime.get()["status"]["code"].asString().make_string()); \ + ASSERT_EQ(vespalib::string(message), slime.get()["status"]["message"].asString().make_string()); \ } #define ASSERT_METRIC_GET_PUT(jsonData, expGetCount, expPutCount) \ @@ -148,16 +138,11 @@ vespalib::Slime slime; \ double putCount = -1; \ size_t metricCount = slime.get()["metrics"]["values"].children(); \ for (size_t j=0; j<metricCount; j++) { \ - const vespalib::string name = slime.get()["metrics"]["values"][j]["name"] \ - .asString().make_string(); \ - if (name.compare("vds.filestor.allthreads.get.count") == 0) \ - { \ - getCount = slime.get()["metrics"]["values"][j]["values"]["count"] \ - .asDouble(); \ - } else if (name.compare("vds.filestor.allthreads.put.count") == 0) \ - { \ - putCount = slime.get()["metrics"]["values"][j]["values"]["count"] \ - .asDouble(); \ + const vespalib::string name = slime.get()["metrics"]["values"][j]["name"].asString().make_string(); \ + if (name.compare("vds.filestor.allthreads.get.count") == 0) { \ + getCount = slime.get()["metrics"]["values"][j]["values"]["count"].asDouble(); \ + } else if (name.compare("vds.filestor.allthreads.put.count") == 0) { \ + putCount = slime.get()["metrics"]["values"][j]["values"]["count"].asDouble(); \ } \ } \ ASSERT_EQ(expGetCount, getCount); \ @@ -226,8 +211,7 @@ TEST_F(StateReporterTest, report_metrics) { for (uint32_t i = 0; i < 6; ++i) { _clock->addSecondsToTime(60); _metricManager->timeChangedNotification(); - while (int64_t(_metricManager->getLastProcessedTime()) < vespalib::count_s(_clock->getMonotonicTime().time_since_epoch())) - { + while (int64_t(_metricManager->getLastProcessedTime()) < vespalib::count_s(_clock->getMonotonicTime().time_since_epoch())) { std::this_thread::sleep_for(1ms); } } diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index a09abb25f7a..9f8456afc37 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -33,34 +33,36 @@ namespace storage { namespace { - using vespalib::getLastErrorString; +using vespalib::getLastErrorString; - void writePidFile(const vespalib::string& pidfile) - { - ssize_t rv = -1; - vespalib::string mypid = vespalib::make_string("%d\n", getpid()); - size_t lastSlash = pidfile.rfind('/'); - if (lastSlash != vespalib::string::npos) { - std::filesystem::create_directories(std::filesystem::path(pidfile.substr(0, lastSlash))); - } - int fd = open(pidfile.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); - if (fd != -1) { - rv = write(fd, mypid.c_str(), mypid.size()); - close(fd); - } - if (rv < 1) { - LOG(warning, "Failed to write pidfile '%s': %s", - pidfile.c_str(), getLastErrorString().c_str()); - } +void +writePidFile(const vespalib::string& pidfile) +{ + ssize_t rv = -1; + vespalib::string mypid = vespalib::make_string("%d\n", getpid()); + size_t lastSlash = pidfile.rfind('/'); + if (lastSlash != vespalib::string::npos) { + std::filesystem::create_directories(std::filesystem::path(pidfile.substr(0, lastSlash))); + } + int fd = open(pidfile.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd != -1) { + rv = write(fd, mypid.c_str(), mypid.size()); + close(fd); } + if (rv < 1) { + LOG(warning, "Failed to write pidfile '%s': %s", + pidfile.c_str(), getLastErrorString().c_str()); + } +} - void removePidFile(const vespalib::string& pidfile) - { - if (unlink(pidfile.c_str()) != 0) { - LOG(warning, "Failed to delete pidfile '%s': %s", - pidfile.c_str(), getLastErrorString().c_str()); - } +void +removePidFile(const vespalib::string& pidfile) +{ + if (unlink(pidfile.c_str()) != 0) { + LOG(warning, "Failed to delete pidfile '%s': %s", + pidfile.c_str(), getLastErrorString().c_str()); } +} } // End of anonymous namespace @@ -429,7 +431,8 @@ StorageNode::shutdown() LOG(debug, "Done shutting down node"); } -void StorageNode::configure(std::unique_ptr<StorServerConfig> config) { +void +StorageNode::configure(std::unique_ptr<StorServerConfig> config) { log_config_received(*config); // When we get config, we try to grab the config lock to ensure noone // else is doing configuration work, and then we write the new config @@ -445,7 +448,8 @@ void StorageNode::configure(std::unique_ptr<StorServerConfig> config) { } } -void StorageNode::configure(std::unique_ptr<UpgradingConfig> config) { +void +StorageNode::configure(std::unique_ptr<UpgradingConfig> config) { log_config_received(*config); { std::lock_guard configLockGuard(_configLock); @@ -457,7 +461,8 @@ void StorageNode::configure(std::unique_ptr<UpgradingConfig> config) { } } -void StorageNode::configure(std::unique_ptr<StorDistributionConfig> config) { +void +StorageNode::configure(std::unique_ptr<StorDistributionConfig> config) { log_config_received(*config); { std::lock_guard configLockGuard(_configLock); @@ -486,7 +491,8 @@ StorageNode::configure(std::unique_ptr<document::config::DocumenttypesConfig> co } } -void StorageNode::configure(std::unique_ptr<BucketspacesConfig> config) { +void +StorageNode::configure(std::unique_ptr<BucketspacesConfig> config) { log_config_received(*config); { std::lock_guard configLockGuard(_configLock); diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 2720d8786cb..8f1b687449a 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -26,6 +26,8 @@ vespa_define_module( src/apps/vespa-validate-hostname TESTS + ${VESPALIB_DIRECTIO_TESTDIR} + ${VESPALIB_PROCESS_MEMORY_STATS_TESTDIR} src/tests/alloc src/tests/approx src/tests/array @@ -38,9 +40,9 @@ vespa_define_module( src/tests/bits src/tests/box src/tests/btree - src/tests/btree/btree_store src/tests/btree/btree-scan-speed src/tests/btree/btree-stress + src/tests/btree/btree_store src/tests/clock src/tests/component src/tests/compress @@ -73,7 +75,6 @@ vespa_define_module( src/tests/datastore/unique_store src/tests/datastore/unique_store_dictionary src/tests/datastore/unique_store_string_allocator - ${VESPALIB_DIRECTIO_TESTDIR} src/tests/detect_type_benchmark src/tests/dotproduct src/tests/drop-file-from-cache @@ -86,6 +87,9 @@ vespa_define_module( src/tests/executor_idle_tracking src/tests/explore_modern_cpp src/tests/false + src/tests/fastlib/io + src/tests/fastlib/text + src/tests/fastos src/tests/fiddle src/tests/fileheader src/tests/floatingpointtype @@ -95,6 +99,7 @@ vespa_define_module( src/tests/guard src/tests/host_name src/tests/hwaccelrated + src/tests/invokeservice src/tests/io/fileutil src/tests/io/mapped_file_input src/tests/issue @@ -134,7 +139,6 @@ vespa_define_module( src/tests/printable src/tests/priority_queue src/tests/process - ${VESPALIB_PROCESS_MEMORY_STATS_TESTDIR} src/tests/programoptions src/tests/random src/tests/referencecounter @@ -144,14 +148,14 @@ vespa_define_module( src/tests/runnable_pair src/tests/rusage src/tests/sequencedtaskexecutor - src/tests/shutdownguard - src/tests/singleexecutor src/tests/sha1 src/tests/shared_operation_throttler src/tests/shared_string_repo src/tests/sharedptr + src/tests/shutdownguard src/tests/signalhandler src/tests/simple_thread_bundle + src/tests/singleexecutor src/tests/slime src/tests/slime/external_data_value src/tests/slime/summary-feature-benchmark @@ -204,17 +208,15 @@ vespa_define_module( src/tests/util/string_escape src/tests/valgrind src/tests/visit_ranges - src/tests/invokeservice src/tests/wakeup src/tests/xmlserializable src/tests/zcurve - src/tests/fastlib/io - src/tests/fastlib/text LIBS src/vespa/fastlib/io src/vespa/fastlib/text src/vespa/fastlib/text/apps + src/vespa/fastos src/vespa/vespalib src/vespa/vespalib/btree src/vespa/vespalib/component diff --git a/vespalib/src/tests/fastos/CMakeLists.txt b/vespalib/src/tests/fastos/CMakeLists.txt new file mode 100644 index 00000000000..6ea986ac0b0 --- /dev/null +++ b/vespalib/src/tests/fastos/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(fastos_file_test_app TEST + SOURCES + file_test.cpp + DEPENDS + vespalib + GTest::GTest +) +vespa_add_test(NAME fastos_file_test_app COMMAND fastos_file_test_app) diff --git a/fastos/src/tests/filetest.cpp b/vespalib/src/tests/fastos/file_test.cpp index d0f8bbfd98b..d0f8bbfd98b 100644 --- a/fastos/src/tests/filetest.cpp +++ b/vespalib/src/tests/fastos/file_test.cpp diff --git a/fastos/src/tests/hello.txt b/vespalib/src/tests/fastos/hello.txt index 62a405393a6..62a405393a6 100644 --- a/fastos/src/tests/hello.txt +++ b/vespalib/src/tests/fastos/hello.txt diff --git a/fastos/src/tests/tests.h b/vespalib/src/tests/fastos/tests.h index 9cd7a10ab48..9cd7a10ab48 100644 --- a/fastos/src/tests/tests.h +++ b/vespalib/src/tests/fastos/tests.h diff --git a/vespalib/src/vespa/fastlib/io/CMakeLists.txt b/vespalib/src/vespa/fastlib/io/CMakeLists.txt index f21cf27b21e..9f6211f3e19 100644 --- a/vespalib/src/vespa/fastlib/io/CMakeLists.txt +++ b/vespalib/src/vespa/fastlib/io/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(fastlib_io OBJECT +vespa_add_library(vespalib_fastlib_io OBJECT SOURCES bufferedfile.cpp DEPENDS diff --git a/vespalib/src/vespa/fastlib/text/CMakeLists.txt b/vespalib/src/vespa/fastlib/text/CMakeLists.txt index d6cb8c29305..06573ee814d 100644 --- a/vespalib/src/vespa/fastlib/text/CMakeLists.txt +++ b/vespalib/src/vespa/fastlib/text/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(fastlib_text OBJECT +vespa_add_library(vespalib_fastlib_text OBJECT SOURCES unicodeutil.cpp normwordfolder.cpp diff --git a/vespalib/src/vespa/fastos/CMakeLists.txt b/vespalib/src/vespa/fastos/CMakeLists.txt new file mode 100644 index 00000000000..2b7a2c3b905 --- /dev/null +++ b/vespalib/src/vespa/fastos/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(vespalib_fastos OBJECT + SOURCES + file.cpp + file_rw_ops.cpp + linux_file.cpp + unix_file.cpp + DEPENDS +) diff --git a/fastos/src/vespa/fastos/file.cpp b/vespalib/src/vespa/fastos/file.cpp index fdbacb570b4..fdbacb570b4 100644 --- a/fastos/src/vespa/fastos/file.cpp +++ b/vespalib/src/vespa/fastos/file.cpp diff --git a/fastos/src/vespa/fastos/file.h b/vespalib/src/vespa/fastos/file.h index 1cf6fee71dd..1cf6fee71dd 100644 --- a/fastos/src/vespa/fastos/file.h +++ b/vespalib/src/vespa/fastos/file.h diff --git a/fastos/src/vespa/fastos/file_rw_ops.cpp b/vespalib/src/vespa/fastos/file_rw_ops.cpp index 79fe95b21f2..79fe95b21f2 100644 --- a/fastos/src/vespa/fastos/file_rw_ops.cpp +++ b/vespalib/src/vespa/fastos/file_rw_ops.cpp diff --git a/fastos/src/vespa/fastos/file_rw_ops.h b/vespalib/src/vespa/fastos/file_rw_ops.h index 4f7aa6f082f..4f7aa6f082f 100644 --- a/fastos/src/vespa/fastos/file_rw_ops.h +++ b/vespalib/src/vespa/fastos/file_rw_ops.h diff --git a/fastos/src/vespa/fastos/linux_file.cpp b/vespalib/src/vespa/fastos/linux_file.cpp index 6fb29782957..6fb29782957 100644 --- a/fastos/src/vespa/fastos/linux_file.cpp +++ b/vespalib/src/vespa/fastos/linux_file.cpp diff --git a/fastos/src/vespa/fastos/linux_file.h b/vespalib/src/vespa/fastos/linux_file.h index 2481b163210..2481b163210 100644 --- a/fastos/src/vespa/fastos/linux_file.h +++ b/vespalib/src/vespa/fastos/linux_file.h diff --git a/fastos/src/vespa/fastos/types.h b/vespalib/src/vespa/fastos/types.h index 69dd3e5231c..69dd3e5231c 100644 --- a/fastos/src/vespa/fastos/types.h +++ b/vespalib/src/vespa/fastos/types.h diff --git a/fastos/src/vespa/fastos/unix_file.cpp b/vespalib/src/vespa/fastos/unix_file.cpp index 7c4cde19125..7c4cde19125 100644 --- a/fastos/src/vespa/fastos/unix_file.cpp +++ b/vespalib/src/vespa/fastos/unix_file.cpp diff --git a/fastos/src/vespa/fastos/unix_file.h b/vespalib/src/vespa/fastos/unix_file.h index 31e45f8d2fa..31e45f8d2fa 100644 --- a/fastos/src/vespa/fastos/unix_file.h +++ b/vespalib/src/vespa/fastos/unix_file.h diff --git a/vespalib/src/vespa/vespalib/CMakeLists.txt b/vespalib/src/vespa/vespalib/CMakeLists.txt index 9e31084c067..63876d442ef 100644 --- a/vespalib/src/vespa/vespalib/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/CMakeLists.txt @@ -30,8 +30,9 @@ vespa_add_library(vespalib $<TARGET_OBJECTS:vespalib_vespalib_time> $<TARGET_OBJECTS:vespalib_vespalib_trace> $<TARGET_OBJECTS:vespalib_vespalib_util> - $<TARGET_OBJECTS:fastlib_io> - $<TARGET_OBJECTS:fastlib_text> + $<TARGET_OBJECTS:vespalib_fastlib_io> + $<TARGET_OBJECTS:vespalib_fastlib_text> + $<TARGET_OBJECTS:vespalib_fastos> INSTALL lib64 DEPENDS ${VESPA_GCC_LIB} |