diff options
7 files changed, 40 insertions, 27 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index e3afdec85b9..abca424a838 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -85,6 +85,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useBucketExecutorForBucketMove() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useBucketExecutorForPruneRemoved() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useExternalRankExpressions() { return false; } + @ModelFeatureFlag(owners = {"baldersheim"}) default boolean distributeExternalRankExpressions() { return false; } @ModelFeatureFlag(owners = {"geirst"}) default boolean enableFeedBlockInDistributor() { return true; } @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default double maxDeadBytesRatio() { return 0.05; } @ModelFeatureFlag(owners = {"hmusum"}, removeAfter = "7.406") default int clusterControllerMaxHeapSizeInMb() { return 128; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 66e728957f3..2a530b78b86 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -100,6 +100,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public int numDistributorStripes() { return numDistributorStripes; } @Override public boolean allowDisableMtls() { return allowDisableMtls; } @Override public boolean useExternalRankExpressions() { return useExternalRankExpression; } + @Override public boolean distributeExternalRankExpressions() { return useExternalRankExpression; } public TestProperties useExternalRankExpression(boolean value) { useExternalRankExpression = value; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 8b467288326..97b537066ee 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -713,18 +713,18 @@ public class RankProfile implements Cloneable { } private Reader openRankingExpressionReader(String expName, String expression) { - if ( ! expression.startsWith("file:")) return new StringReader(expression); + if (!expression.startsWith("file:")) return new StringReader(expression); String fileName = extractFileName(expression); File file = new File(fileName); - if ( ! file.isAbsolute() && file.getPath().contains("/")) // See ticket 4102122 - throw new IllegalArgumentException("In " + getName() +", " + expName + ", ranking references file '" + file + - "' in subdirectory, which is not supported."); - - /* TODO balder: Disabled until end-2-end verified - rankExpressionFiles().add(new RankExpressionFile(getUniqueExpressionName(expName), fileName)); - externalFileExpressions.add(expName); - */ + if (!file.isAbsolute() && file.getPath().contains("/")) // See ticket 4102122 + throw new IllegalArgumentException("In " + getName() + ", " + expName + ", ranking references file '" + file + + "' in subdirectory, which is not supported."); + + if (search.getDeployProperties().featureFlags().distributeExternalRankExpressions()) { + rankExpressionFiles().add(new RankExpressionFile(getUniqueExpressionName(expName), fileName)); + externalFileExpressions.add(expName); + } return search.getRankingExpression(fileName); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index 370315ad08b..125e25470df 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -105,7 +105,7 @@ public class ExportingTestCase extends AbstractExportingTestCase { @Test public void testRankExpression() throws IOException, ParseException { assertCorrectDeriving("rankexpression", null, - new TestProperties().useExternalRankExpression(true), new TestableDeployLogger()); + new TestProperties().useExternalRankExpression(false), new TestableDeployLogger()); } @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index 289a380a85f..ff17a211e17 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -3,6 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.collections.Pair; import com.yahoo.config.FileReference; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.search.query.profile.QueryProfileRegistry; @@ -30,10 +31,11 @@ public class RankingExpressionsTestCase extends SchemaTestCase { @Test public void testFunctions() throws IOException, ParseException { + ModelContext.Properties deployProperties = new TestProperties().useExternalRankExpression(true); RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); Search search = SearchBuilder.createFromDirectory("src/test/examples/rankingexpressionfunction", new TestableDeployLogger(), - new TestProperties(), + deployProperties, rankProfileRegistry).getSearch(); RankProfile functionsRankProfile = rankProfileRegistry.get(search, "macros"); Map<String, RankProfile.RankingExpressionFunction> functions = functionsRankProfile.getFunctions(); @@ -47,26 +49,25 @@ public class RankingExpressionsTestCase extends SchemaTestCase { functions.get("artistmatch").function().getBody().getRoot().toString()); assertEquals(0, functions.get("artistmatch").function().arguments().size()); - List<Pair<String, String>> rankProperties = new RawRankProfile(functionsRankProfile, - new QueryProfileRegistry(), - new ImportedMlModels(), - new AttributeFields(search)).configProperties(); - assertEquals(6, rankProperties.size()); - - assertEquals("rankingExpression(titlematch$).rankingScript", rankProperties.get(2).getFirst()); - assertEquals("var1 * var2 + 890", rankProperties.get(2).getSecond()); - - assertEquals("rankingExpression(artistmatch).rankingScript", rankProperties.get(3).getFirst()); - assertEquals("78 + closeness(distance)", rankProperties.get(3).getSecond()); + RawRankProfile rawRankProfile = new RawRankProfile(functionsRankProfile, new QueryProfileRegistry(), + new ImportedMlModels(), new AttributeFields(search), deployProperties); + List<Pair<String, String>> rankProperties = rawRankProfile.configProperties(); + assertEquals(5, rankProperties.size()); - assertEquals("rankingExpression(firstphase).rankingScript", rankProperties.get(5).getFirst()); - assertEquals("0.8 + 0.2 * rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c) + 0.8 * rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6) * closeness(distance)", rankProperties.get(5).getSecond()); + assertEquals("rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c).rankingScript", rankProperties.get(0).getFirst()); + assertEquals("4 * 5 + 890", rankProperties.get(0).getSecond()); assertEquals("rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6).rankingScript", rankProperties.get(1).getFirst()); assertEquals("7 * 8 + 890", rankProperties.get(1).getSecond()); - assertEquals("rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c).rankingScript", rankProperties.get(0).getFirst()); - assertEquals("4 * 5 + 890", rankProperties.get(0).getSecond()); + assertEquals("rankingExpression(artistmatch).rankingScript", rankProperties.get(2).getFirst()); + assertEquals("78 + closeness(distance)", rankProperties.get(2).getSecond()); + + assertEquals("vespa.rank.firstphase", rankProperties.get(3).getFirst()); + assertEquals("rankingExpression(firstphase)", rankProperties.get(3).getSecond()); + + assertEquals("rankingExpression(firstphase).rankingScript", rankProperties.get(4).getFirst()); + assertEquals("0.8 + 0.2 * rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c) + 0.8 * rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6) * closeness(distance)", rankProperties.get(4).getSecond()); } @Test(expected = IllegalArgumentException.class) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index f44694f5066..195f9dbf8a7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -176,6 +176,7 @@ public class ModelContextImpl implements ModelContext { private final ToIntFunction<ClusterSpec.Type> jvmOmitStackTraceInFastThrow; private final boolean enableCustomAclMapping; private final boolean useExternalRankExpression; + private final boolean distributeExternalRankExpressions; private final int numDistributorStripes; public FeatureFlags(FlagSource source, ApplicationId appId) { @@ -199,7 +200,8 @@ public class ModelContextImpl implements ModelContext { this.jvmOmitStackTraceInFastThrow = type -> flagValueAsInt(source, appId, type, PermanentFlags.JVM_OMIT_STACK_TRACE_IN_FAST_THROW); this.enableCustomAclMapping = flagValue(source, appId, Flags.ENABLE_CUSTOM_ACL_MAPPING); this.numDistributorStripes = flagValue(source, appId, Flags.NUM_DISTRIBUTOR_STRIPES); - this.useExternalRankExpression = flagValue(source, appId, Flags.USE_EXTERNAL_RANK_EXPRESSION);; + this.useExternalRankExpression = flagValue(source, appId, Flags.USE_EXTERNAL_RANK_EXPRESSION); + this.distributeExternalRankExpressions = flagValue(source, appId, Flags.DISTRIBUTE_EXTERNAL_RANK_EXPRESSION); } @Override public Optional<NodeResources> dedicatedClusterControllerFlavor() { return Optional.ofNullable(dedicatedClusterControllerFlavor); } @@ -225,6 +227,7 @@ public class ModelContextImpl implements ModelContext { @Override public boolean enableCustomAclMapping() { return enableCustomAclMapping; } @Override public int numDistributorStripes() { return numDistributorStripes; } @Override public boolean useExternalRankExpressions() { return useExternalRankExpression; } + @Override public boolean distributeExternalRankExpressions() { return distributeExternalRankExpressions; } private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index dff6833c015..5693159dfd5 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -231,6 +231,13 @@ public class Flags { "Takes effect on next internal redeployment", APPLICATION_ID); + public static final UnboundBooleanFlag DISTRIBUTE_EXTERNAL_RANK_EXPRESSION = defineFeatureFlag( + "distribute-external-rank-expression", false, + List.of("baldersheim"), "2021-05-27", "2021-07-01", + "Whether to use distributed external rank expression files by filedistribution", + "Takes effect on next internal redeployment", + APPLICATION_ID); + public static final UnboundBooleanFlag ENABLE_ROUTING_CORE_DUMP = defineFeatureFlag( "enable-routing-core-dumps", false, List.of("tokle"), "2021-04-16", "2021-08-01", |