diff options
Diffstat (limited to 'config-model/src/test/java/com/yahoo/searchdefinition')
12 files changed, 112 insertions, 36 deletions
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java index 056fc27f067..c2bea606bef 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java @@ -1,10 +1,15 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import org.junit.Ignore; import org.junit.Test; +import java.util.function.Function; +import java.util.regex.Pattern; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * Tests rank feature names. @@ -43,4 +48,47 @@ public class FeatureNamesTestCase { FeatureNames.asQueryFeature("foo.bar").toString()); } + @Test + public void testLegalFeatureNames() { + assertTrue(FeatureNames.notNeedQuotes("_")); + assertFalse(FeatureNames.notNeedQuotes("-")); + assertTrue(FeatureNames.notNeedQuotes("_-")); + assertTrue(FeatureNames.notNeedQuotes("0_-azAZxy98-_")); + assertFalse(FeatureNames.notNeedQuotes("0_-azAZxy98-_+")); + } + + @Test + @Ignore + /* + * Unignore to verify performance + * 2021/09/05 performance was a factor of 5.25 + * 'Identifier handcoded validity check took 4301ms + * Identifier regexp validity check took 22609ms' + */ + public void benchMarkPatternMatching() { + Pattern identifierRegexp = Pattern.compile("[A-Za-z0-9_][A-Za-z0-9_-]*"); + String[] strings = new String[1000]; + for (int i = 0; i < strings.length; i++) { + strings[i] = i + "-legal_string" + i; + } + + countValid(strings, 1000, "handcoded warmup", FeatureNames::notNeedQuotes); + countValid(strings, 1000, "regexp warmup", (s) -> identifierRegexp.matcher(s).matches()); + + countValid(strings, 100000, "handcoded", FeatureNames::notNeedQuotes); + countValid(strings, 100000, "regexp", (s) -> identifierRegexp.matcher(s).matches()); + } + + private void countValid(String [] strings, int numReps, String text, Function<String, Boolean> func) { + long start = System.nanoTime(); + int validCount = 0; + for (int i = 0; i < numReps; i++) { + for (String s : strings) { + if (func.apply(s)) validCount++; + } + } + long end = System.nanoTime(); + assertEquals(strings.length * numReps, validCount); + System.out.println("Identifier " + text + " validity check took " + (end - start)/1000000 + "ms"); + } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java index 4453f327bb4..2490359ba1e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java @@ -1,12 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.parser.ParseException; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.yolean.Exceptions; import org.junit.Test; @@ -27,7 +24,7 @@ public class IncorrectRankingExpressionFileRefTestCase extends SchemaTestCase { Search search = SearchBuilder.buildFromFile("src/test/examples/incorrectrankingexpressionfileref.sd", registry, new QueryProfileRegistry()); - new DerivedConfiguration(search, new BaseDeployLogger(), new TestProperties(), registry, new QueryProfileRegistry(), new ImportedMlModels()); // cause rank profile parsing + new DerivedConfiguration(search, registry); // cause rank profile parsing fail("parsing should have failed"); } catch (IllegalArgumentException e) { String message = Exceptions.toMessageString(e); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java index c1fe5e42dfa..8c0c594c0cb 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java @@ -1,12 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.parser.ParseException; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.yolean.Exceptions; import org.junit.Test; @@ -27,7 +23,7 @@ public class RankingExpressionValidationTestCase extends SchemaTestCase { try { RankProfileRegistry registry = new RankProfileRegistry(); Search search = importWithExpression(expression, registry); - new DerivedConfiguration(search, new BaseDeployLogger(), new TestProperties(), registry, new QueryProfileRegistry(), new ImportedMlModels()); // cause rank profile parsing + new DerivedConfiguration(search, registry); // cause rank profile parsing fail("No exception on incorrect ranking expression " + expression); } catch (IllegalArgumentException e) { // Success diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java index 95137e2a080..737598a08d8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.deploy.TestProperties; @@ -49,15 +50,14 @@ public abstract class AbstractExportingTestCase extends SchemaTestCase { properties, builder.getRankProfileRegistry(), builder.getQueryProfileRegistry(), - new ImportedMlModels()); + new ImportedMlModels(), new InThreadExecutorService()); return export(dirName, builder, config); } DerivedConfiguration derive(String dirName, SearchBuilder builder, Search search) throws IOException { DerivedConfiguration config = new DerivedConfiguration(search, builder.getRankProfileRegistry(), - builder.getQueryProfileRegistry(), - new ImportedMlModels()); + builder.getQueryProfileRegistry()); return export(dirName, builder, config); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java index 139dae65c63..ffcdc076d77 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java @@ -9,7 +9,6 @@ import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import org.junit.Test; /** @@ -32,7 +31,7 @@ public class EmptyRankProfileTestCase extends SchemaTestCase { doc.addField(new SDField("c", DataType.STRING)); search = SearchBuilder.buildFromRawSearch(search, rankProfileRegistry, new QueryProfileRegistry()); - new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(), new ImportedMlModels()); + new DerivedConfiguration(search, rankProfileRegistry); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index 370315ad08b..138fb333621 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -105,7 +105,7 @@ public class ExportingTestCase extends AbstractExportingTestCase { @Test public void testRankExpression() throws IOException, ParseException { assertCorrectDeriving("rankexpression", null, - new TestProperties().useExternalRankExpression(true), new TestableDeployLogger()); + new TestProperties().useExternalRankExpression(true).largeRankExpressionLimit(1024), new TestableDeployLogger()); } @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java index 2e36319bd8c..e9e618cb333 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java @@ -11,7 +11,6 @@ import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.processing.Processing; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; @@ -42,7 +41,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true, false); - DerivedConfiguration derived=new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(), new ImportedMlModels()); + DerivedConfiguration derived=new DerivedConfiguration(search, rankProfileRegistry); // Check attribute fields derived.getAttributeFields(); // TODO: assert content @@ -73,7 +72,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); search = SearchBuilder.buildFromRawSearch(search, rankProfileRegistry, new QueryProfileRegistry()); - DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(),new ImportedMlModels()); + DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry); // Check il script addition assertIndexing(Arrays.asList("clear_state | guard { input a | tokenize normalize stem:\"BEST\" | index a; }", @@ -100,7 +99,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { field2.setLiteralBoost(20); search = SearchBuilder.buildFromRawSearch(search, rankProfileRegistry, new QueryProfileRegistry()); - new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(), new ImportedMlModels()); + new DerivedConfiguration(search, rankProfileRegistry); assertIndexing(Arrays.asList("clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }", "clear_state | guard { input body | tokenize normalize stem:\"BEST\" | summary body | index body; }", "clear_state | guard { input title | tokenize | index title_literal; }", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankProfilesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankProfilesTestCase.java index d0712aaeb4d..861ae10cdf6 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankProfilesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankProfilesTestCase.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -14,6 +15,6 @@ import java.io.IOException; public class RankProfilesTestCase extends AbstractExportingTestCase { @Test public void testRankProfiles() throws IOException, ParseException { - assertCorrectDeriving("rankprofiles"); + assertCorrectDeriving("rankprofiles", null, new TestProperties(), new TestableDeployLogger()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java index 83e11c365f8..51961a6083d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java @@ -1,11 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; -import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.parser.ParseException; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import org.junit.Test; import java.io.File; @@ -33,10 +31,7 @@ public class SimpleInheritTestCase extends AbstractExportingTestCase { toDir.mkdirs(); deleteContent(toDir); - DerivedConfiguration config = new DerivedConfiguration(search, - builder.getRankProfileRegistry(), - new QueryProfileRegistry(), - new ImportedMlModels()); + DerivedConfiguration config = new DerivedConfiguration(search, builder.getRankProfileRegistry()); config.export(toDirName); checkDir(toDirName, expectedResultsDirName); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java index 364eb9dc014..4f7a6118296 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java @@ -3,14 +3,12 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.document.DataType; -import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.processing.Processing; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; @@ -34,7 +32,7 @@ public class TypeConversionTestCase extends SchemaTestCase { document.addField(a); new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true, false); - DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(), new ImportedMlModels()); + DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry); IndexInfo indexInfo = derived.getIndexInfo(); assertFalse(indexInfo.hasCommand("default", "compact-to-term")); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java index 809ccdb3a3a..259d0d67106 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java @@ -1,16 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.parser.ParseException; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import org.junit.Test; import java.io.IOException; @@ -90,7 +86,7 @@ public class ImplicitSearchFieldsTestCase extends SchemaTestCase { sb.importFile("src/test/examples/nextgen/simple.sd"); sb.build(); assertNotNull(sb.getSearch()); - new DerivedConfiguration(sb.getSearch(), new BaseDeployLogger(), new TestProperties(), sb.getRankProfileRegistry(), new QueryProfileRegistry(), new ImportedMlModels()); + new DerivedConfiguration(sb.getSearch(), sb.getRankProfileRegistry()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index 9dd569a2f64..00ac5ac5405 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.collections.Pair; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.search.query.profile.QueryProfileRegistry; @@ -22,10 +21,12 @@ import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import org.junit.Test; import java.io.IOException; +import java.util.Arrays; import java.util.List; import java.util.Map; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class RankingExpressionsTestCase extends SchemaTestCase { @@ -75,7 +76,53 @@ public class RankingExpressionsTestCase extends SchemaTestCase { public void testThatIncludingFileInSubdirFails() throws IOException, ParseException { RankProfileRegistry registry = new RankProfileRegistry(); Search search = createSearch("src/test/examples/rankingexpressioninfile", new TestProperties(), registry); - new DerivedConfiguration(search, new BaseDeployLogger(), new TestProperties(), registry, new QueryProfileRegistry(), new ImportedMlModels()); // rank profile parsing happens during deriving + new DerivedConfiguration(search, registry); // rank profile parsing happens during deriving } + private void verifyProfile(RankProfile profile, List<String> expectedFunctions, List<Pair<String, String>> rankProperties, + LargeRankExpressions largeExpressions, QueryProfileRegistry queryProfiles, ImportedMlModels models, + AttributeFields attributes, ModelContext.Properties properties) { + var functions = profile.getFunctions(); + assertEquals(expectedFunctions.size(), functions.size()); + for (String func : expectedFunctions) { + assertTrue(functions.containsKey(func)); + } + + RawRankProfile raw = new RawRankProfile(profile, largeExpressions, queryProfiles, models, attributes, properties); + assertEquals(rankProperties.size(), raw.configProperties().size()); + for (int i = 0; i < rankProperties.size(); i++) { + assertEquals(rankProperties.get(i).getFirst(), raw.configProperties().get(i).getFirst()); + assertEquals(rankProperties.get(i).getSecond(), raw.configProperties().get(i).getSecond()); + } + } + + private void verifySearch(Search search, RankProfileRegistry rankProfileRegistry, LargeRankExpressions largeExpressions, + QueryProfileRegistry queryProfiles, ImportedMlModels models, ModelContext.Properties properties) + { + AttributeFields attributes = new AttributeFields(search); + + verifyProfile(rankProfileRegistry.get(search, "base"), Arrays.asList("large_f", "large_m"), + Arrays.asList(new Pair<>("rankingExpression(large_f).expressionName", "base.large_f"), new Pair<>("rankingExpression(large_m).expressionName", "base.large_m")), + largeExpressions, queryProfiles, models, attributes, properties); + for (String child : Arrays.asList("child_a", "child_b")) { + verifyProfile(rankProfileRegistry.get(search, child), Arrays.asList("large_f", "large_m", "large_local_f", "large_local_m"), + Arrays.asList(new Pair<>("rankingExpression(large_f).expressionName", child + ".large_f"), new Pair<>("rankingExpression(large_m).expressionName", child + ".large_m"), + new Pair<>("rankingExpression(large_local_f).expressionName", child + ".large_local_f"), new Pair<>("rankingExpression(large_local_m).expressionName", child + ".large_local_m"), + new Pair<>("vespa.rank.firstphase", "rankingExpression(firstphase)"), new Pair<>("rankingExpression(firstphase).expressionName", child + ".firstphase")), + largeExpressions, queryProfiles, models, attributes, properties); + } + } + + @Test + public void testLargeInheritedFunctions() throws IOException, ParseException { + ModelContext.Properties properties = new TestProperties().useExternalRankExpression(true).largeRankExpressionLimit(50); + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + LargeRankExpressions largeExpressions = new LargeRankExpressions(new MockFileRegistry()); + QueryProfileRegistry queryProfiles = new QueryProfileRegistry(); + ImportedMlModels models = new ImportedMlModels(); + Search search = createSearch("src/test/examples/largerankingexpressions", properties, rankProfileRegistry); + verifySearch(search, rankProfileRegistry, largeExpressions, queryProfiles, models, properties); + // Need to verify that second derivation works as that will happen if same sd is used in multiple content clusters + verifySearch(search, rankProfileRegistry, largeExpressions, queryProfiles, models, properties); + } } |