diff options
Diffstat (limited to 'config-model/src/test/java/com/yahoo/schema/processing')
50 files changed, 5789 insertions, 0 deletions
diff --git a/config-model/src/test/java/com/yahoo/schema/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java b/config-model/src/test/java/com/yahoo/schema/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java new file mode 100644 index 00000000000..0d64dd5c953 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java @@ -0,0 +1,75 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.schema.DocumentReference; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.TestableDeployLogger; +import com.yahoo.schema.document.ImportedField; +import com.yahoo.schema.document.ImportedFields; +import com.yahoo.schema.document.ImportedSimpleField; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import org.junit.Test; + +import java.util.Collections; + +import static org.junit.Assert.assertEquals; + +/** + * @author bjorncs + */ +public class AddAttributeTransformToSummaryOfImportedFieldsTest { + + private static final String IMPORTED_FIELD_NAME = "imported_myfield"; + private static final String DOCUMENT_NAME = "mydoc"; + private static final String SUMMARY_NAME = "mysummary"; + + @Test + public void attribute_summary_transform_applied_to_summary_field_of_imported_field() { + Schema schema = createSearchWithDocument(DOCUMENT_NAME); + schema.setImportedFields(createSingleImportedField(IMPORTED_FIELD_NAME)); + schema.addSummary(createDocumentSummary(IMPORTED_FIELD_NAME, schema)); + + AddAttributeTransformToSummaryOfImportedFields processor = new AddAttributeTransformToSummaryOfImportedFields( + schema, null, null, null); + processor.process(true, false); + SummaryField summaryField = schema.getSummaries().get(SUMMARY_NAME).getSummaryField(IMPORTED_FIELD_NAME); + SummaryTransform actualTransform = summaryField.getTransform(); + assertEquals(SummaryTransform.ATTRIBUTE, actualTransform); + } + + private static Schema createSearch(String documentType) { + return new Schema(documentType, MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); + } + + private static Schema createSearchWithDocument(String documentName) { + Schema schema = createSearch(documentName); + SDDocumentType document = new SDDocumentType(documentName, schema); + schema.addDocument(document); + return schema; + } + + private static ImportedFields createSingleImportedField(String fieldName) { + Schema targetSchema = createSearchWithDocument("target_doc"); + var doc = targetSchema.getDocument(); + SDField targetField = new SDField(doc, "target_field", DataType.INT); + DocumentReference documentReference = new DocumentReference(new Field("reference_field"), targetSchema); + ImportedField importedField = new ImportedSimpleField(fieldName, documentReference, targetField); + return new ImportedFields(Collections.singletonMap(fieldName, importedField)); + } + + private static DocumentSummary createDocumentSummary(String fieldName, Schema schema) { + DocumentSummary summary = new DocumentSummary("mysummary", schema); + summary.add(new SummaryField(fieldName, DataType.INT)); + return summary; + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/AdjustPositionSummaryFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/AdjustPositionSummaryFieldsTestCase.java new file mode 100644 index 00000000000..103d08b39a8 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/AdjustPositionSummaryFieldsTestCase.java @@ -0,0 +1,260 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.document.DataType; +import com.yahoo.document.PositionDataType; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +public class AdjustPositionSummaryFieldsTestCase { + + @Test + public void test_pos_summary() { + SearchModel model = new SearchModel(false); + model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, "pos"); + model.resolve(); + model.assertSummaryField("my_pos", PositionDataType.INSTANCE, SummaryTransform.GEOPOS, "pos_zcurve"); + model.assertSummaryField("my_pos.position", DataType.getArray(DataType.STRING), SummaryTransform.POSITIONS, "pos_zcurve"); + model.assertSummaryField("my_pos.distance", DataType.INT, SummaryTransform.DISTANCE, "pos_zcurve"); + } + + @Test + public void test_imported_pos_summary() { + SearchModel model = new SearchModel(); + model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, null); + model.resolve(); + model.assertSummaryField("my_pos", PositionDataType.INSTANCE, SummaryTransform.GEOPOS, "my_pos_zcurve"); + model.assertSummaryField("my_pos.position", DataType.getArray(DataType.STRING), SummaryTransform.POSITIONS, "my_pos_zcurve"); + model.assertSummaryField("my_pos.distance", DataType.INT, SummaryTransform.DISTANCE, "my_pos_zcurve"); + } + + @Test + public void test_imported_pos_summary_bad_source() { + SearchModel model = new SearchModel(); + model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, "pos"); + model.resolve(); + // SummaryFieldsMustHaveValidSource processing not run in this test. + model.assertSummaryField("my_pos", PositionDataType.INSTANCE, SummaryTransform.NONE, "pos"); + model.assertNoSummaryField("my_pos.position"); + model.assertNoSummaryField("my_pos.distance"); + } + + @Test + public void test_imported_pos_summary_bad_datatype() { + SearchModel model = new SearchModel(); + model.addSummaryField("my_pos", DataType.getArray(PositionDataType.INSTANCE), null, "pos"); + model.resolve(); + model.assertSummaryField("my_pos", DataType.getArray(PositionDataType.INSTANCE), SummaryTransform.NONE, "pos"); + model.assertNoSummaryField("my_pos.position"); + model.assertNoSummaryField("my_pos.distance"); + } + + @Test + public void test_pos_summary_no_attr_no_rename() { + SearchModel model = new SearchModel(false, false, false); + model.addSummaryField("pos", PositionDataType.INSTANCE, null, "pos"); + model.resolve(); + model.assertSummaryField("pos", PositionDataType.INSTANCE, SummaryTransform.NONE, "pos"); + model.assertNoSummaryField("pos.position"); + model.assertNoSummaryField("pos.distance"); + } + + @Test + public void test_pos_default_summary_no_attr_no_rename() { + SearchModel model = new SearchModel(false, false, false); + model.resolve(); + assertNull(model.childSchema.getSummary("default")); // ImplicitSummaries processing not run in this test + } + + @Test + public void test_pos_summary_no_rename() { + SearchModel model = new SearchModel(false, true, false); + model.addSummaryField("pos", PositionDataType.INSTANCE, null, "pos"); + model.resolve(); + model.assertSummaryField("pos", PositionDataType.INSTANCE, SummaryTransform.GEOPOS, "pos_zcurve"); + model.assertSummaryField("pos.position", DataType.getArray(DataType.STRING), SummaryTransform.POSITIONS, "pos_zcurve"); + model.assertSummaryField("pos.distance", DataType.INT, SummaryTransform.DISTANCE, "pos_zcurve"); + } + + @SuppressWarnings("deprecation") + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void test_pos_summary_no_attr() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For schema 'child', field 'my_pos': No position attribute 'pos_zcurve'"); + SearchModel model = new SearchModel(false, false, false); + model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, "pos"); + model.resolve(); + } + + @Test + public void test_pos_summary_bad_attr() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For schema 'child', field 'my_pos': No position attribute 'pos_zcurve'"); + SearchModel model = new SearchModel(false, false, true); + model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, "pos"); + model.resolve(); + } + + @Test + public void test_imported_pos_summary_no_attr() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For schema 'child', import field 'my_pos_zcurve': " + + "Field 'pos_zcurve' via reference field 'ref': Not found"); + SearchModel model = new SearchModel(true, false, false); + model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, null); + model.resolve(); + } + + @Test + public void test_imported_pos_summary_bad_attr() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For schema 'child', field 'my_pos': " + + "No position attribute 'my_pos_zcurve'"); + SearchModel model = new SearchModel(true, false, true); + model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, null); + model.resolve(); + } + + @Test + public void test_my_pos_position_summary_bad_datatype() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For schema 'child', field 'my_pos.position': " + + "exists with type 'datatype string (code: 2)', should be of type 'datatype Array<string> (code: -1486737430)"); + SearchModel model = new SearchModel(); + model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, null); + model.addSummaryField("my_pos.position", DataType.STRING, null, "pos"); + model.resolve(); + } + + @Test + public void test_my_pos_position_summary_bad_transform() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For schema 'child', field 'my_pos.position': " + + "has summary transform 'none', should have transform 'positions'"); + SearchModel model = new SearchModel(); + model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, null); + model.addSummaryField("my_pos.position", DataType.getArray(DataType.STRING), null, "pos"); + model.resolve(); + } + + @Test + public void test_my_pos_position_summary_bad_source() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For schema 'child', field 'my_pos.position': " + + "has source '[source field 'pos']', should have source 'source field 'my_pos_zcurve''"); + SearchModel model = new SearchModel(); + model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, null); + model.addSummaryField("my_pos.position", DataType.getArray(DataType.STRING), SummaryTransform.POSITIONS, "pos"); + model.resolve(); + } + + static class SearchModel extends ParentChildSearchModel { + + SearchModel() { + this(true); + } + + SearchModel(boolean importedPos) { + this(importedPos, true, false); + } + + SearchModel(boolean importedPos, boolean setupPosAttr, boolean setupBadAttr) { + super(); + if (importedPos) { + createPositionField(parentSchema, setupPosAttr, setupBadAttr); + } + addRefField(childSchema, parentSchema, "ref"); + if (importedPos) { + addImportedField("my_pos", "ref", "pos"); + } else { + createPositionField(childSchema, setupPosAttr, setupBadAttr); + } + } + + private void createPositionField(Schema schema, boolean setupPosAttr, boolean setupBadAttr) { + String ilScript = setupPosAttr ? "{ summary | attribute }" : "{ summary }"; + var doc = schema.getDocument(); + doc.addField(createField(doc, "pos", PositionDataType.INSTANCE, ilScript)); + if (setupBadAttr) { + doc.addField(createField(doc, "pos_zcurve", DataType.LONG, "{ attribute }")); + } + } + + void addSummaryField(String fieldName, DataType dataType, SummaryTransform transform, String source) { + addSummaryField("my_summary", fieldName, dataType, transform, source); + } + + public void addSummaryField(String summaryName, String fieldName, DataType dataType, SummaryTransform transform, String source) { + DocumentSummary summary = childSchema.getSummary(summaryName); + if (summary == null) { + summary = new DocumentSummary(summaryName, childSchema); + childSchema.addSummary(summary); + } + SummaryField summaryField = new SummaryField(fieldName, dataType); + if (source != null) { + summaryField.addSource(source); + } + if (transform != null) { + summaryField.setTransform(transform); + } + summary.add(summaryField); + } + + public void assertNoSummaryField(String fieldName) { + assertNoSummaryField("my_summary", fieldName); + } + + public void assertNoSummaryField(String summaryName, String fieldName) { + DocumentSummary summary = childSchema.getSummary(summaryName); + assertNotNull(summary); + SummaryField summaryField = summary.getSummaryField(fieldName); + assertNull(summaryField); + } + + public void assertSummaryField(String fieldName, DataType dataType, SummaryTransform transform, String source) { + assertSummaryField("my_summary", fieldName, dataType, transform, source); + } + + public void assertSummaryField(String summaryName, String fieldName, DataType dataType, SummaryTransform transform, String source) { + DocumentSummary summary = childSchema.getSummary(summaryName); + assertNotNull(summary); + SummaryField summaryField = summary.getSummaryField(fieldName); + assertNotNull(summaryField); + assertEquals(dataType, summaryField.getDataType()); + assertEquals(transform, summaryField.getTransform()); + if (source == null) { + assertEquals(0, summaryField.getSourceCount()); + } else { + assertEquals(1, summaryField.getSourceCount()); + assertEquals(source, summaryField.getSingleSource()); + } + } + + public void resolve() { + resolve(parentSchema); + resolve(childSchema); + } + + private static void resolve(Schema schema) { + new CreatePositionZCurve(schema, null, null, null).process(true, false); + assertNotNull(schema.temporaryImportedFields().get()); + assertFalse(schema.importedFields().isPresent()); + new ImportedFieldsResolver(schema, null, null, null).process(true, false); + assertNotNull(schema.importedFields().get()); + new AdjustPositionSummaryFields(schema, null, null, null).process(true, false); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/AssertIndexingScript.java b/config-model/src/test/java/com/yahoo/schema/processing/AssertIndexingScript.java new file mode 100644 index 00000000000..82650598f29 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/AssertIndexingScript.java @@ -0,0 +1,43 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.IndexingScript; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.parser.ParseException; + +import java.util.LinkedList; +import java.util.List; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author Simon Thoresen Hult + */ +public abstract class AssertIndexingScript { + + public static void assertIndexing(List<String> expected, Schema schema) { + assertIndexing(expected, new IndexingScript(schema).expressions()); + } + + public static void assertIndexing(List<String> expected, IndexingScript script) { + assertIndexing(expected, script.expressions()); + } + + public static void assertIndexing(List<String> expected, Iterable<Expression> actual) { + List<String> parsedExpected = new LinkedList<>(); + for (String str : expected) { + try { + parsedExpected.add(Expression.fromString(str).toString()); + } catch (ParseException e) { + fail(e.getMessage()); + } + } + for (Expression actualExp : actual) { + String str = actualExp.toString(); + assertTrue("Unexpected: " + str, parsedExpected.remove(str)); + } + assertTrue("Missing: " + parsedExpected.toString(), parsedExpected.isEmpty()); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/AssertSearchBuilder.java b/config-model/src/test/java/com/yahoo/schema/processing/AssertSearchBuilder.java new file mode 100644 index 00000000000..0b4d7c3a2b6 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/AssertSearchBuilder.java @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; + +import java.io.IOException; + +import static org.junit.Assert.*; + +/** + * @author Simon Thoresen Hult + */ +public abstract class AssertSearchBuilder { + + public static void assertBuilds(String searchDefinitionFileName) throws IOException, ParseException { + assertNotNull(ApplicationBuilder.buildFromFile(searchDefinitionFileName)); + } + + public static void assertBuildFails(String searchDefinitionFileName, String expectedException) + throws IOException, ParseException { + try { + ApplicationBuilder.buildFromFile(searchDefinitionFileName); + fail(searchDefinitionFileName); + } catch (IllegalArgumentException e) { + assertEquals(expectedException, e.getMessage()); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/AttributesExactMatchTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/AttributesExactMatchTestCase.java new file mode 100644 index 00000000000..40ebe458c74 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/AttributesExactMatchTestCase.java @@ -0,0 +1,40 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +/** + * Attributes should be implicitly exact-match in some cases + * @author vegardh + * + */ +public class AttributesExactMatchTestCase extends AbstractSchemaTestCase { + @Test + public void testAttributesExactMatch() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/attributesexactmatch.sd"); + assertEquals(schema.getConcreteField("color").getMatching().getType(), MatchType.EXACT); + assertEquals(schema.getConcreteField("artist").getMatching().getType(), MatchType.WORD); + assertEquals(schema.getConcreteField("drummer").getMatching().getType(), MatchType.WORD); + assertEquals(schema.getConcreteField("guitarist").getMatching().getType(), MatchType.TEXT); + assertEquals(schema.getConcreteField("saxophonist_arr").getMatching().getType(), MatchType.WORD); + assertEquals(schema.getConcreteField("flutist").getMatching().getType(), MatchType.TEXT); + + assertFalse(schema.getConcreteField("genre").getMatching().getType().equals(MatchType.EXACT)); + assertFalse(schema.getConcreteField("title").getMatching().getType().equals(MatchType.EXACT)); + assertFalse(schema.getConcreteField("trumpetist").getMatching().getType().equals(MatchType.EXACT)); + assertFalse(schema.getConcreteField("genre").getMatching().getType().equals(MatchType.WORD)); + assertFalse(schema.getConcreteField("title").getMatching().getType().equals(MatchType.WORD)); + assertFalse(schema.getConcreteField("trumpetist").getMatching().getType().equals(MatchType.WORD)); + + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/BoldingTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/BoldingTestCase.java new file mode 100644 index 00000000000..c37bc8085c7 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/BoldingTestCase.java @@ -0,0 +1,65 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author bratseth + */ +public class BoldingTestCase extends AbstractSchemaTestCase { + + private final String boldonnonstring = + "search boldnonstring {\n" + + " document boldnonstring {\n" + + " field title type string {\n" + + " indexing: summary | index\n" + + " }\n" + + "\n" + + " field year4 type int {\n" + + " indexing: summary | attribute\n" + + " bolding: on\n" + + " }\n" + + " }\n" + + "}\n"; + + @Test + public void testBoldOnNonString() throws ParseException { + try { + ApplicationBuilder.createFromString(boldonnonstring); + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertEquals("'bolding: on' for non-text field 'year4' (datatype int (code: 0)) is not allowed", + e.getMessage()); + } + } + + private final String boldonarray = + "search boldonarray {\n" + + " document boldonarray {\n" + + " field myarray type array<string> {\n" + + " indexing: summary | index\n" + + " bolding: on\n" + + " }\n" + + " }\n" + + "}\n"; + + @Test + public void testBoldOnArray() throws ParseException { + try { + ApplicationBuilder.createFromString(boldonarray); + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertEquals("'bolding: on' for non-text field 'myarray' (datatype Array<string> (code: -1486737430)) is not allowed", + e.getMessage()); + } + } + +} + + diff --git a/config-model/src/test/java/com/yahoo/schema/processing/BoolAttributeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/BoolAttributeValidatorTestCase.java new file mode 100644 index 00000000000..287cc6559d1 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/BoolAttributeValidatorTestCase.java @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import static com.yahoo.schema.ApplicationBuilder.createFromString; +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author geirst + */ +public class BoolAttributeValidatorTestCase { + + @Test + public void array_of_bool_attribute_is_not_supported() throws ParseException { + try { + createFromString(getSd("field b type array<bool> { indexing: attribute }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'b': Only single value bool attribute fields are supported", + e.getMessage()); + } + } + + @Test + public void weigtedset_of_bool_attribute_is_not_supported() throws ParseException { + try { + createFromString(getSd("field b type weightedset<bool> { indexing: attribute }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'b': Only single value bool attribute fields are supported", + e.getMessage()); + } + } + + private String getSd(String field) { + return joinLines( + "schema test {", + " document test {", + " " + field, + " }", + "}"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/DictionaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/DictionaryTestCase.java new file mode 100644 index 00000000000..1956b87a689 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/DictionaryTestCase.java @@ -0,0 +1,250 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.schema.processing; + +import com.yahoo.config.model.test.TestUtil; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.document.Case; +import com.yahoo.schema.document.Dictionary; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.config.search.AttributesConfig; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; + +/** + * Test configuration of dictionary control. + * + * @author baldersheim + */ +public class DictionaryTestCase { + private static AttributesConfig getConfig(Schema schema) { + AttributeFields attributes = new AttributeFields(schema); + AttributesConfig.Builder builder = new AttributesConfig.Builder(); + attributes.getConfig(builder, AttributeFields.FieldSet.ALL, 130000, true); + return builder.build(); + } + private Schema createSearch(String def) throws ParseException { + ApplicationBuilder sb = ApplicationBuilder.createFromString(def); + return sb.getSchema(); + } + @Test + public void testDefaultDictionarySettings() throws ParseException { + String def = TestUtil.joinLines( + "search test {", + " document test {", + " field s1 type string {", + " indexing: attribute | summary", + " }", + " field n1 type int {", + " indexing: summary | attribute", + " }", + " }", + "}"); + Schema schema = createSearch(def); + assertNull(schema.getAttribute("s1").getDictionary()); + assertNull(schema.getAttribute("n1").getDictionary()); + assertEquals(AttributesConfig.Attribute.Dictionary.Type.BTREE, + getConfig(schema).attribute().get(0).dictionary().type()); + assertEquals(AttributesConfig.Attribute.Dictionary.Type.BTREE, + getConfig(schema).attribute().get(1).dictionary().type()); + assertEquals(AttributesConfig.Attribute.Dictionary.Match.UNCASED, + getConfig(schema).attribute().get(0).dictionary().match()); + assertEquals(AttributesConfig.Attribute.Dictionary.Match.UNCASED, + getConfig(schema).attribute().get(1).dictionary().match()); + } + + Schema verifyDictionaryControl(Dictionary.Type expected, String type, String ... cfg) throws ParseException + { + String def = TestUtil.joinLines( + "search test {", + " document test {", + " field n1 type " + type + " {", + " indexing: summary | attribute", + " attribute:fast-search", + TestUtil.joinLines(cfg), + " }", + " }", + "}"); + Schema schema = createSearch(def); + AttributesConfig.Attribute.Dictionary.Type.Enum expectedConfig = toCfg(expected); + assertEquals(expected, schema.getAttribute("n1").getDictionary().getType()); + assertEquals(expectedConfig, getConfig(schema).attribute().get(0).dictionary().type()); + return schema; + } + + AttributesConfig.Attribute.Dictionary.Type.Enum toCfg(Dictionary.Type v) { + return (v == Dictionary.Type.HASH) + ? AttributesConfig.Attribute.Dictionary.Type.Enum.HASH + : (v == Dictionary.Type.BTREE) + ? AttributesConfig.Attribute.Dictionary.Type.Enum.BTREE + : AttributesConfig.Attribute.Dictionary.Type.Enum.BTREE_AND_HASH; + } + AttributesConfig.Attribute.Dictionary.Match.Enum toCfg(Case v) { + return (v == Case.CASED) + ? AttributesConfig.Attribute.Dictionary.Match.Enum.CASED + : AttributesConfig.Attribute.Dictionary.Match.Enum.UNCASED; + } + + void verifyStringDictionaryControl(Dictionary.Type expectedType, Case expectedCase, Case matchCasing, + String ... cfg) throws ParseException + { + Schema schema = verifyDictionaryControl(expectedType, "string", cfg); + ImmutableSDField f = schema.getField("n1"); + AttributesConfig.Attribute.Dictionary.Match.Enum expectedCaseCfg = toCfg(expectedCase); + assertEquals(matchCasing, f.getMatching().getCase()); + assertEquals(expectedCase, schema.getAttribute("n1").getDictionary().getMatch()); + assertEquals(expectedCaseCfg, getConfig(schema).attribute().get(0).dictionary().match()); + } + + @Test + public void testCasedBtreeSettings() throws ParseException { + verifyDictionaryControl(Dictionary.Type.BTREE, "int", "dictionary:cased"); + } + + @Test + public void testNumericBtreeSettings() throws ParseException { + verifyDictionaryControl(Dictionary.Type.BTREE, "int", "dictionary:btree"); + } + @Test + public void testNumericHashSettings() throws ParseException { + verifyDictionaryControl(Dictionary.Type.HASH, "int", "dictionary:hash"); + } + @Test + public void testNumericBtreeAndHashSettings() throws ParseException { + verifyDictionaryControl(Dictionary.Type.BTREE_AND_HASH, "int", "dictionary:btree", "dictionary:hash"); + } + @Test + public void testNumericArrayBtreeAndHashSettings() throws ParseException { + verifyDictionaryControl(Dictionary.Type.BTREE_AND_HASH, "array<int>", "dictionary:btree", "dictionary:hash"); + } + @Test + public void testNumericWSetBtreeAndHashSettings() throws ParseException { + verifyDictionaryControl(Dictionary.Type.BTREE_AND_HASH, "weightedset<int>", "dictionary:btree", "dictionary:hash"); + } + @Test + public void testStringBtreeSettings() throws ParseException { + verifyStringDictionaryControl(Dictionary.Type.BTREE, Case.UNCASED, Case.UNCASED, "dictionary:btree"); + } + @Test + public void testStringBtreeUnCasedSettings() throws ParseException { + verifyStringDictionaryControl(Dictionary.Type.BTREE, Case.UNCASED, Case.UNCASED, "dictionary { btree\nuncased\n}"); + } + @Test + public void testStringBtreeCasedSettings() throws ParseException { + verifyStringDictionaryControl(Dictionary.Type.BTREE, Case.CASED, Case.CASED, "dictionary { btree\ncased\n}", "match:cased"); + } + @Test + public void testStringHashSettings() throws ParseException { + try { + verifyStringDictionaryControl(Dictionary.Type.HASH, Case.UNCASED, Case.UNCASED, "dictionary:hash"); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'n1': hash dictionary require cased match", e.getMessage()); + } + } + @Test + public void testStringHashUnCasedSettings() throws ParseException { + try { + verifyStringDictionaryControl(Dictionary.Type.HASH, Case.UNCASED, Case.UNCASED, "dictionary { hash\nuncased\n}"); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'n1': hash dictionary require cased match", e.getMessage()); + } + } + @Test + public void testStringHashBothCasedSettings() throws ParseException { + verifyStringDictionaryControl(Dictionary.Type.HASH, Case.CASED, Case.CASED, "dictionary { hash\ncased\n}", "match:cased"); + } + @Test + public void testStringHashCasedSettings() throws ParseException { + try { + verifyStringDictionaryControl(Dictionary.Type.HASH, Case.CASED, Case.CASED, "dictionary { hash\ncased\n}"); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'n1': Dictionary casing 'CASED' does not match field match casing 'UNCASED'", e.getMessage()); + } + } + @Test + public void testStringBtreeHashSettings() throws ParseException { + verifyStringDictionaryControl(Dictionary.Type.BTREE_AND_HASH, Case.UNCASED, Case.UNCASED, "dictionary{hash\nbtree\n}"); + } + @Test + public void testStringBtreeHashUnCasedSettings() throws ParseException { + verifyStringDictionaryControl(Dictionary.Type.BTREE_AND_HASH, Case.UNCASED, Case.UNCASED, "dictionary { hash\nbtree\nuncased\n}"); + } + @Test + public void testStringBtreeHashCasedSettings() throws ParseException { + try { + verifyStringDictionaryControl(Dictionary.Type.BTREE_AND_HASH, Case.CASED, Case.CASED, "dictionary { btree\nhash\ncased\n}"); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'n1': Dictionary casing 'CASED' does not match field match casing 'UNCASED'", e.getMessage()); + } + } + @Test + public void testNonNumericFieldsFailsDictionaryControl() throws ParseException { + String def = TestUtil.joinLines( + "schema test {", + " document test {", + " field n1 type bool {", + " indexing: summary | attribute", + " dictionary:btree", + " }", + " }", + "}"); + try { + ApplicationBuilder sb = ApplicationBuilder.createFromString(def); + fail("Controlling dictionary for non-numeric fields are not yet supported."); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'n1': You can only specify 'dictionary:' for numeric or string fields", e.getMessage()); + } + } + @Test + public void testNonFastSearchNumericFieldsFailsDictionaryControl() throws ParseException { + String def = TestUtil.joinLines( + "schema test {", + " document test {", + " field n1 type int {", + " indexing: summary | attribute", + " dictionary:btree", + " }", + " }", + "}"); + try { + ApplicationBuilder sb = ApplicationBuilder.createFromString(def); + fail("Controlling dictionary for non-fast-search fields are not allowed."); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'n1': You must specify 'attribute:fast-search' to allow dictionary control", e.getMessage()); + } + } + + @Test + public void testCasingForNonFastSearch() throws ParseException { + String def = TestUtil.joinLines( + "schema test {", + " document test {", + " field s1 type string {", + " indexing: attribute | summary", + " }", + " field s2 type string {", + " indexing: attribute | summary", + " match:uncased", + " }", + " field s3 type string {", + " indexing: attribute | summary", + " match:cased", + " }", + " }", + "}"); + Schema schema = createSearch(def); + assertEquals(Case.UNCASED, schema.getAttribute("s1").getCase()); + assertEquals(Case.UNCASED, schema.getAttribute("s2").getCase()); + assertEquals(Case.CASED, schema.getAttribute("s3").getCase()); + assertEquals(AttributesConfig.Attribute.Match.UNCASED, getConfig(schema).attribute().get(0).match()); + assertEquals(AttributesConfig.Attribute.Match.UNCASED, getConfig(schema).attribute().get(1).match()); + assertEquals(AttributesConfig.Attribute.Match.CASED, getConfig(schema).attribute().get(2).match()); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java new file mode 100644 index 00000000000..64b0a437b1d --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java @@ -0,0 +1,57 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +/** + * @author lesters + */ +public class DisallowComplexMapAndWsetKeyTypesTestCase { + + @Test(expected = IllegalArgumentException.class) + public void requireThatComplexTypesForMapKeysFail() throws ParseException { + testFieldType("map<mystruct,string>"); + } + + @Test(expected = IllegalArgumentException.class) + public void requireThatComplexTypesForWsetFail() throws ParseException { + testFieldType("weightedset<mystruct>"); + } + + @Test(expected = IllegalArgumentException.class) + public void requireThatNestedComplexTypesForMapFail() throws ParseException { + testFieldType("array<map<mystruct,string>>"); + } + + @Test + public void requireThatNestedComplexValuesForMapSucceed() throws ParseException { + testFieldType("array<map<string,mystruct>>"); + } + + @Test(expected = IllegalArgumentException.class) + public void requireThatNestedComplexTypesForWsetFail() throws ParseException { + testFieldType("array<weightedset<mystruct>>"); + } + + @Test(expected = IllegalArgumentException.class) + public void requireThatDeepNestedComplexTypesForMapFail() throws ParseException { + testFieldType("map<string,map<mystruct,string>>"); + } + + private void testFieldType(String fieldType) throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " struct mystruct {}\n" + + " field a type " + fieldType + " {}\n" + + " }\n" + + "}\n"); + builder.build(true); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/FastAccessValidatorTest.java b/config-model/src/test/java/com/yahoo/schema/processing/FastAccessValidatorTest.java new file mode 100644 index 00000000000..b249b407c7b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/FastAccessValidatorTest.java @@ -0,0 +1,61 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.model.test.TestUtil; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +/** + * @author bjorncs + */ +public class FastAccessValidatorTest { + + @SuppressWarnings("deprecation") + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void throws_exception_on_incompatible_use_of_fastaccess() throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(new RankProfileRegistry()); + builder.addSchema( + TestUtil.joinLines( + "schema parent {", + " document parent {", + " field int_field type int { indexing: attribute }", + " }", + "}")); + builder.addSchema( + TestUtil.joinLines( + "schema test {", + " document test { ", + " field int_attribute type int { ", + " indexing: attribute ", + " attribute: fast-access", + " }", + " field predicate_attribute type predicate {", + " indexing: attribute ", + " attribute: fast-access", + " }", + " field tensor_attribute type tensor(x[5]) {", + " indexing: attribute ", + " attribute: fast-access", + " }", + " field reference_attribute type reference<parent> {", + " indexing: attribute ", + " attribute: fast-access", + " }", + " }", + "}")); + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage( + "For schema 'test': The following attributes have a type that is incompatible " + + "with fast-access: predicate_attribute, tensor_attribute, reference_attribute. " + + "Predicate, tensor and reference attributes are incompatible with fast-access."); + builder.build(true); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSchemaFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSchemaFieldsTestCase.java new file mode 100644 index 00000000000..594124c9500 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSchemaFieldsTestCase.java @@ -0,0 +1,94 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class ImplicitSchemaFieldsTestCase extends AbstractSchemaTestCase { + + @Test + public void testRequireThatExtraFieldsAreIncluded() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/extrafield.sd"); + assertNotNull(schema); + + SDDocumentType docType = schema.getDocument(); + assertNotNull(docType); + assertNotNull(docType.getField("foo")); + assertNotNull(docType.getField("bar")); + assertEquals(2, docType.getFieldCount()); + } + + @Test + public void testRequireThatSummaryFieldsAreIncluded() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/summaryfield.sd"); + assertNotNull(schema); + + SDDocumentType docType = schema.getDocument(); + assertNotNull(docType); + assertNotNull(docType.getField("foo")); + assertNotNull(docType.getField("bar")); + assertNotNull(docType.getField("cox")); + assertNotNull(docType.getField("mytags")); + assertNotNull(docType.getField("alltags")); + assertEquals(5, docType.getFieldCount()); + } + + @Test + public void testRequireThatBoldedSummaryFieldsAreIncluded() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/boldedsummaryfields.sd"); + assertNotNull(schema); + + SDDocumentType docType = schema.getDocument(); + assertNotNull(docType); + assertNotNull(docType.getField("foo")); + assertNotNull(docType.getField("bar")); + assertNotNull(docType.getField("baz")); + assertNotNull(docType.getField("cox")); + assertEquals(4, docType.getFieldCount()); + } + + @Test + public void testRequireThatUntransformedSummaryFieldsAreIgnored() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/untransformedsummaryfields.sd"); + assertNotNull(schema); + + SDDocumentType docType = schema.getDocument(); + assertNotNull(docType); + assertNotNull(docType.getField("foo")); + assertNotNull(docType.getField("bar")); + assertNotNull(docType.getField("baz")); + assertEquals(3, docType.getFieldCount()); + } + + @Test + public void testRequireThatDynamicSummaryFieldsAreIgnored() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/dynamicsummaryfields.sd"); + assertNotNull(schema); + + SDDocumentType docType = schema.getDocument(); + assertNotNull(docType); + assertNotNull(docType.getField("foo")); + assertNotNull(docType.getField("bar")); + assertEquals(2, docType.getFieldCount()); + } + + @Test + public void testRequireThatDerivedConfigurationWorks() throws IOException, ParseException { + ApplicationBuilder sb = new ApplicationBuilder(); + sb.addSchemaFile("src/test/examples/nextgen/simple.sd"); + sb.build(true); + assertNotNull(sb.getSchema()); + new DerivedConfiguration(sb.getSchema(), sb.getRankProfileRegistry()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ImplicitStructTypesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitStructTypesTestCase.java new file mode 100644 index 00000000000..111ed266d74 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitStructTypesTestCase.java @@ -0,0 +1,69 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.document.*; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.*; +public class ImplicitStructTypesTestCase extends AbstractSchemaTestCase { + @Test + public void testRequireThatImplicitStructsAreCreated() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/toggleon.sd"); + assertNotNull(schema); + + SDDocumentType docType = schema.getDocument(); + assertNotNull(docType); + assertStruct(docType, PositionDataType.INSTANCE); + } + @Test + public void testRequireThatImplicitStructsAreUsed() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/implicitstructtypes.sd"); + assertNotNull(schema); + + SDDocumentType docType = schema.getDocument(); + assertNotNull(docType); + + assertField(docType, "doc_str", DataType.STRING); + assertField(docType, "doc_str_sum", DataType.STRING); + assertField(docType, "doc_uri", DataType.URI); + assertField(docType, "docsum_str", DataType.STRING); + } + + @SuppressWarnings({ "UnusedDeclaration" }) + private static void assertStruct(SDDocumentType docType, StructDataType expectedStruct) { + // TODO: When structs are refactored from a static register to a member of the owning document types, this test + // TODO: must be changed to retrieve struct type from the provided document type. + StructDataType structType = (StructDataType) docType.getType(expectedStruct.getName()).getStruct(); + assertNotNull(structType); + for (Field expectedField : expectedStruct.getFields()) { + Field field = structType.getField(expectedField.getName()); + assertNotNull(field); + assertEquals(expectedField.getDataType(), field.getDataType()); + } + assertEquals(expectedStruct.getFieldCount(), structType.getFieldCount()); + } + + private static void assertField(SDDocumentType docType, String fieldName, DataType type) { + Field field = getSecretField(docType, fieldName); // TODO: get rid of this stupidity + assertNotNull(field); + assertEquals(type, field.getDataType()); + assertTrue(field instanceof SDField); + } + + private static Field getSecretField(SDDocumentType docType, String fieldName) { + for (Field field : docType.fieldSet()) { + if (field.getName().equals(fieldName)) { + return field; + } + } + return null; + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSummariesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSummariesTestCase.java new file mode 100644 index 00000000000..50deb5d5b42 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSummariesTestCase.java @@ -0,0 +1,78 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * @author Simon Thoresen Hult + */ +public class ImplicitSummariesTestCase { + + @Test + public void requireThatSummaryFromAttributeDoesNotWarn() throws IOException, ParseException { + LogHandler log = new LogHandler(); + Logger.getLogger("").addHandler(log); + + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/implicitsummaries_attribute.sd"); + assertNotNull(schema); + assertTrue(log.records.isEmpty()); + } + + private static class LogHandler extends Handler { + + final List<LogRecord> records = new ArrayList<>(); + + @Override + public void publish(LogRecord record) { + if (record.getLevel() == Level.WARNING || + record.getLevel() == Level.SEVERE) + { + records.add(record); + } + } + + @Override + public void flush() { + + } + + @Override + public void close() throws SecurityException { + + } + } + + @Test + public void attribute_combiner_transform_is_set_on_array_of_struct_with_only_struct_field_attributes() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/array_of_struct_attribute/test.sd"); + assertEquals(SummaryTransform.ATTRIBUTECOMBINER, schema.getSummaryField("elem_array").getTransform()); + } + + @Test + public void attribute_combiner_transform_is_set_on_map_of_struct_with_only_struct_field_attributes() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); + assertEquals(SummaryTransform.ATTRIBUTECOMBINER, schema.getSummaryField("str_elem_map").getTransform()); + } + + @Test + public void attribute_combiner_transform_is_not_set_when_map_of_struct_has_some_struct_field_attributes() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); + assertEquals(SummaryTransform.NONE, schema.getSummaryField("int_elem_map").getTransform()); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSummaryFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSummaryFieldsTestCase.java new file mode 100644 index 00000000000..f32c9079d36 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSummaryFieldsTestCase.java @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class ImplicitSummaryFieldsTestCase extends AbstractSchemaTestCase { + + @Test + public void testRequireThatImplicitFieldsAreCreated() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/implicitsummaryfields.sd"); + assertNotNull(schema); + + DocumentSummary docsum = schema.getSummary("default"); + assertNotNull(docsum); + assertNotNull(docsum.getSummaryField("rankfeatures")); + assertNotNull(docsum.getSummaryField("summaryfeatures")); + assertEquals(2, docsum.getSummaryFields().size()); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ImportedFieldsResolverTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ImportedFieldsResolverTestCase.java new file mode 100644 index 00000000000..5baa64d06d4 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/ImportedFieldsResolverTestCase.java @@ -0,0 +1,152 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.document.DataType; +import com.yahoo.document.TensorDataType; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.ImmutableImportedSDField; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.ImportedField; +import com.yahoo.schema.document.ImportedFields; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.TemporarySDField; +import com.yahoo.tensor.TensorType; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; + +/** + * @author geirst + */ +public class ImportedFieldsResolverTestCase { + + @SuppressWarnings("deprecation") + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + + private void resolve_imported_field(String fieldName, String targetFieldName) { + SearchModel model = new SearchModel(); + model.addImportedField(fieldName, "ref", targetFieldName).resolve(); + + assertEquals(1, model.importedFields.fields().size()); + ImportedField myField = model.importedFields.fields().get(fieldName); + assertNotNull(myField); + assertEquals(fieldName, myField.fieldName()); + assertSame(model.childSchema.getConcreteField("ref"), myField.reference().referenceField()); + assertSame(model.parentSchema, myField.reference().targetSearch()); + ImmutableSDField targetField = model.parentSchema.getField(targetFieldName); + if (targetField instanceof SDField) { + assertSame(targetField, myField.targetField()); + } else { + assertSame(getImportedField(targetField), getImportedField(myField.targetField())); + } + } + + private static ImportedField getImportedField(ImmutableSDField field) { + return ((ImmutableImportedSDField) field).getImportedField(); + } + + @Test + public void valid_imported_fields_are_resolved() { + resolve_imported_field("my_attribute_field", "attribute_field"); + resolve_imported_field("my_tensor_field", "tensor_field"); + resolve_imported_field("my_ancient_field", "ancient_field"); + } + + @Test + public void resolver_fails_if_document_reference_is_not_found() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For schema 'child', import field 'my_attribute_field': " + + "Reference field 'not_ref' not found"); + new SearchModel().addImportedField("my_attribute_field", "not_ref", "budget").resolve(); + } + + @Test + public void resolver_fails_if_referenced_field_is_not_found() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For schema 'child', import field 'my_attribute_field': " + + "Field 'not_existing' via reference field 'ref': Not found"); + new SearchModel().addImportedField("my_attribute_field", "ref", "not_existing").resolve(); + } + + @Test + public void resolver_fails_if_imported_field_is_not_an_attribute() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For schema 'child', import field 'my_not_attribute': " + + "Field 'not_attribute' via reference field 'ref': Is not an attribute field. Only attribute fields supported"); + new SearchModel().addImportedField("my_not_attribute", "ref", "not_attribute").resolve(); + } + + @Test + public void resolver_fails_if_imported_field_is_indexing() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage( + "For schema 'child', import field 'my_attribute_and_index': " + + "Field 'attribute_and_index' via reference field 'ref': Is an index field. Not supported"); + new SearchModel() + .addImportedField("my_attribute_and_index", "ref", "attribute_and_index") + .resolve(); + } + + @Test + public void resolver_fails_if_imported_field_is_of_type_predicate() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage( + "For schema 'child', import field 'my_predicate_field': " + + "Field 'predicate_field' via reference field 'ref': Is of type 'predicate'. Not supported"); + new SearchModel().addImportedField("my_predicate_field", "ref", "predicate_field").resolve(); + } + + static class SearchModel extends ParentChildSearchModel { + + public final Schema grandParentSchema; + public ImportedFields importedFields; + + public SearchModel() { + super(); + grandParentSchema = createSearch("grandparent"); + var grandParentDoc = grandParentSchema.getDocument(); + grandParentDoc.addField(createField(grandParentDoc, "ancient_field", DataType.INT, "{ attribute }")); + var parentDoc = parentSchema.getDocument(); + parentDoc.addField(createField(parentDoc, "attribute_field", DataType.INT, "{ attribute }")); + parentDoc.addField(createField(parentDoc, "attribute_and_index", DataType.INT, "{ attribute | index }")); + parentDoc.addField(new TemporarySDField(parentDoc, "not_attribute", DataType.INT)); + parentDoc.addField(createField(parentDoc, "tensor_field", new TensorDataType(TensorType.fromSpec("tensor(x[5])")), "{ attribute }")); + parentDoc.addField(createField(parentDoc, "predicate_field", DataType.PREDICATE, "{ attribute }")); + addRefField(parentSchema, grandParentSchema, "ref"); + addImportedField(parentSchema, "ancient_field", "ref", "ancient_field"); + + addRefField(childSchema, parentSchema, "ref"); + } + + + protected SearchModel addImportedField(String fieldName, String referenceFieldName, String targetFieldName) { + return addImportedField(childSchema, fieldName, referenceFieldName, targetFieldName); + } + + protected SearchModel addImportedField(Schema schema, String fieldName, String referenceFieldName, String targetFieldName) { + super.addImportedField(schema, fieldName, referenceFieldName, targetFieldName); + return this; + } + + public void resolve() { + resolve(grandParentSchema); + resolve(parentSchema); + importedFields = resolve(childSchema); + } + + private static ImportedFields resolve(Schema schema) { + assertNotNull(schema.temporaryImportedFields().get()); + assertFalse(schema.importedFields().isPresent()); + new ImportedFieldsResolver(schema, null, null, null).process(true, false); + assertNotNull(schema.importedFields().get()); + return schema.importedFields().get(); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ImportedFieldsTestCase.java new file mode 100644 index 00000000000..ab702154527 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/ImportedFieldsTestCase.java @@ -0,0 +1,529 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.document.ImportedComplexField; +import com.yahoo.schema.document.ImportedField; +import com.yahoo.schema.parser.ParseException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static org.junit.Assert.assertEquals; +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +/** + * @author geirst + */ +public class ImportedFieldsTestCase { + + @Test + public void fields_can_be_imported_from_referenced_document_types() throws ParseException { + Schema schema = buildAdSearch(joinLines( + "search ad {", + " document ad {", + " field campaign_ref type reference<campaign> { indexing: attribute }", + " field person_ref type reference<person> { indexing: attribute }", + " }", + " import field campaign_ref.budget as my_budget {}", + " import field person_ref.name as my_name {}", + "}")); + assertEquals(2, schema.importedFields().get().fields().size()); + assertSearchContainsImportedField("my_budget", "campaign_ref", "campaign", "budget", schema); + assertSearchContainsImportedField("my_name", "person_ref", "person", "name", schema); + } + + @SuppressWarnings("deprecation") + @Rule + public ExpectedException exception = ExpectedException.none(); + + @Test + public void field_reference_spec_must_include_dot() throws ParseException { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Illegal field reference spec 'campaignrefbudget': Does not include a single '.'"); + buildAdSearch(joinLines( + "search ad {", + " document ad {}", + " import field campaignrefbudget as budget {}", + "}")); + } + + @Test + public void fail_duplicate_import() throws ParseException { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For schema 'ad', import field as 'my_budget': Field already imported"); + Schema schema = buildAdSearch(joinLines( + "schema ad {", + " document ad {", + " field campaign_ref type reference<campaign> { indexing: attribute }", + " }", + " import field campaign_ref.budget as my_budget {}", + " import field campaign_ref.budget as my_budget {}", + "}")); + } + + private static Schema buildAdSearch(String sdContent) throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( + "schema campaign {", + " document campaign {", + " field budget type int { indexing: attribute }", + " }", + "}")); + builder.addSchema(joinLines( + "schema person {", + " document person {", + " field name type string { indexing: attribute }", + " }", + "}")); + builder.addSchema(sdContent); + builder.build(true); + return builder.getSchema("ad"); + } + + private static void checkStructImport(AncestorStructSdBuilder parentBuilder) throws ParseException { + Schema schema = buildChildSearch(parentBuilder.build(), new ChildStructSdBuilder().build()); + checkImportedStructFields(schema, parentBuilder); + } + + private static void checkNestedStructImport(AncestorStructSdBuilder grandParentBuilder) throws ParseException { + Schema schema = buildChildSearch(grandParentBuilder.build(), + new IntermediateParentStructSdBuilder().build(), + new ChildStructSdBuilder().build()); + checkImportedStructFields(schema, grandParentBuilder); + } + + private static void checkImportedStructFields(Schema schema, AncestorStructSdBuilder ancestorBuilder) { + assertEquals(3, schema.importedFields().get().fields().size()); + checkImportedField("my_elem_array.name", "parent_ref", "parent", "elem_array.name", schema, ancestorBuilder.elem_array_name_attr); + checkImportedField("my_elem_array.weight", "parent_ref", "parent", "elem_array.weight", schema, ancestorBuilder.elem_array_weight_attr); + checkImportedField("my_elem_map.key", "parent_ref", "parent", "elem_map.key", schema, ancestorBuilder.elem_map_key_attr); + checkImportedField("my_elem_map.value.name", "parent_ref", "parent", "elem_map.value.name", schema, ancestorBuilder.elem_map_value_name_attr); + checkImportedField("my_elem_map.value.weight", "parent_ref", "parent", "elem_map.value.weight", schema, ancestorBuilder.elem_map_value_weight_attr); + checkImportedField("my_str_int_map.key", "parent_ref", "parent", "str_int_map.key", schema, ancestorBuilder.str_int_map_key_attr); + checkImportedField("my_str_int_map.value", "parent_ref", "parent", "str_int_map.value", schema, ancestorBuilder.str_int_map_value_attr); + checkImportedField("my_elem_array", "parent_ref", "parent", "elem_array", schema, true); + checkImportedField("my_elem_map", "parent_ref", "parent", "elem_map", schema, true); + checkImportedField("my_str_int_map", "parent_ref", "parent", "str_int_map", schema, true); + } + + @Test + public void check_struct_import() throws ParseException { + checkStructImport(new ParentStructSdBuilder()); + checkStructImport(new ParentStructSdBuilder().elem_array_weight_attr(false).elem_map_value_weight_attr(false)); + checkStructImport(new ParentStructSdBuilder().elem_array_name_attr(false).elem_map_value_name_attr(false)); + } + + @Test + public void check_nested_struct_import() throws ParseException { + checkNestedStructImport(new GrandParentStructSdBuilder()); + checkNestedStructImport(new GrandParentStructSdBuilder().elem_array_weight_attr(false).elem_map_value_weight_attr(false)); + checkNestedStructImport(new GrandParentStructSdBuilder().elem_array_name_attr(false).elem_map_value_name_attr(false)); + } + + @Test + public void check_illegal_struct_import_missing_array_of_struct_attributes() throws ParseException { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For schema 'child', import field 'my_elem_array': Field 'elem_array' via reference field 'parent_ref': Is not a struct containing an attribute field."); + checkStructImport(new ParentStructSdBuilder().elem_array_name_attr(false).elem_array_weight_attr(false)); + } + + @Test + public void check_illegal_struct_import_missing_map_of_struct_key_attribute() throws ParseException { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For schema 'child', import field 'my_elem_map' (nested to 'my_elem_map.key'): Field 'elem_map.key' via reference field 'parent_ref': Is not an attribute field. Only attribute fields supported"); + checkStructImport(new ParentStructSdBuilder().elem_map_key_attr(false)); + } + + @Test + public void check_illegal_struct_import_missing_map_of_struct_value_attributes() throws ParseException { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For schema 'child', import field 'my_elem_map' (nested to 'my_elem_map.value'): Field 'elem_map.value' via reference field 'parent_ref': Is not a struct containing an attribute field."); + checkStructImport(new ParentStructSdBuilder().elem_map_value_name_attr(false).elem_map_value_weight_attr(false)); + } + + @Test + public void check_illegal_struct_import_missing_map_of_primitive_key_attribute() throws ParseException { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For schema 'child', import field 'my_str_int_map' (nested to 'my_str_int_map.key'): Field 'str_int_map.key' via reference field 'parent_ref': Is not an attribute field. Only attribute fields supported"); + checkStructImport(new ParentStructSdBuilder().str_int_map_key_attr(false)); + } + + @Test + public void check_illegal_struct_import_missing_map_of_primitive_value_attribute() throws ParseException { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For schema 'child', import field 'my_str_int_map' (nested to 'my_str_int_map.value'): Field 'str_int_map.value' via reference field 'parent_ref': Is not an attribute field. Only attribute fields supported"); + checkStructImport(new ParentStructSdBuilder().str_int_map_value_attr(false)); + } + + private static class NamedSdBuilder { + protected String name; + private String fieldPrefix; + + public NamedSdBuilder(String name, String fieldPrefix) { + this.name = name; + this.fieldPrefix = fieldPrefix; + } + + protected String prefixedFieldName(String name) { + return fieldPrefix + name; + } + } + + private static class AncestorStructSdBuilder extends NamedSdBuilder { + private boolean elem_array_name_attr; + private boolean elem_array_weight_attr; + private boolean elem_map_key_attr; + private boolean elem_map_value_name_attr; + private boolean elem_map_value_weight_attr; + private boolean str_int_map_key_attr; + private boolean str_int_map_value_attr; + + public AncestorStructSdBuilder(String name, String fieldPrefix) { + super(name, fieldPrefix); + elem_array_name_attr = true; + elem_array_weight_attr = true; + elem_map_key_attr = true; + elem_map_value_name_attr = true; + elem_map_value_weight_attr = true; + str_int_map_key_attr = true; + str_int_map_value_attr = true; + } + + public AncestorStructSdBuilder elem_array_name_attr(boolean v) { elem_array_name_attr = v; return this; } + public AncestorStructSdBuilder elem_array_weight_attr(boolean v) { elem_array_weight_attr = v; return this; } + public AncestorStructSdBuilder elem_map_key_attr(boolean v) { elem_map_key_attr = v; return this; } + public AncestorStructSdBuilder elem_map_value_name_attr(boolean v) { elem_map_value_name_attr = v; return this; } + public AncestorStructSdBuilder elem_map_value_weight_attr(boolean v) { elem_map_value_weight_attr = v; return this; } + public AncestorStructSdBuilder str_int_map_key_attr(boolean v) { str_int_map_key_attr = v; return this; } + public AncestorStructSdBuilder str_int_map_value_attr(boolean v) { str_int_map_value_attr = v; return this; } + + public String build() { + return joinLines("search " + name + " {", + " document " + name + " {", + " struct elem {", + " field name type string {}", + " field weight type int {}", + " }", + " field " + prefixedFieldName("elem_array") + " type array<elem> {", + " indexing: summary", + " struct-field name {", + structFieldSpec(elem_array_name_attr), + " }", + " struct-field weight {", + structFieldSpec(elem_array_weight_attr), + " }", + " }", + " field " + prefixedFieldName("elem_map") + " type map<string, elem> {", + " indexing: summary", + " struct-field key {", + structFieldSpec(elem_map_key_attr), + " }", + " struct-field value.name {", + structFieldSpec(elem_map_value_name_attr), + " }", + " struct-field value.weight {", + structFieldSpec(elem_map_value_weight_attr), + " }", + " }", + " field " + prefixedFieldName("str_int_map") + " type map<string, int> {", + " indexing: summary", + " struct-field key {", + structFieldSpec(str_int_map_key_attr), + " }", + " struct-field value {", + structFieldSpec(str_int_map_value_attr), + " }", + " }", + " }", + "}"); + } + + private static String structFieldSpec(boolean isAttribute) { + return isAttribute ? " indexing: attribute" : ""; + } + } + + private static class ParentStructSdBuilder extends AncestorStructSdBuilder { + ParentStructSdBuilder() { + super("parent", ""); + } + } + + private static class GrandParentStructSdBuilder extends AncestorStructSdBuilder { + GrandParentStructSdBuilder() { + super("grandparent", "gp_"); + } + } + + private static class DescendantSdBuilder extends NamedSdBuilder { + protected String parentName; + private String parentFieldPrefix; + + public DescendantSdBuilder(String name, String fieldPrefix, String parentName, String parentFieldPrefix) { + super(name, fieldPrefix); + this.parentName = parentName; + this.parentFieldPrefix = parentFieldPrefix; + } + + protected String parentRef() { + return parentName + "_ref"; + } + + protected String importParentField(String fieldName) { + return " import field " + parentRef() + "." + parentFieldPrefix + fieldName + " as " + prefixedFieldName(fieldName) + " {}"; + } + } + + private static class DescendantStructSdBuilder extends DescendantSdBuilder { + public DescendantStructSdBuilder(String name, String fieldPrefix, String parentName, String parentFieldPrefix) { + super(name, fieldPrefix, parentName, parentFieldPrefix); + } + + public String build() { + return joinLines("search " + name + " {", + " document " + name + " {", + " field " + parentRef() + " type reference<" + parentName + "> {", + " indexing: attribute | summary", + " }", + " }", + importParentField("elem_array"), + importParentField("elem_map"), + importParentField("str_int_map"), + "}"); + } + } + + private static class ChildStructSdBuilder extends DescendantStructSdBuilder { + public ChildStructSdBuilder() { + super("child", "my_", "parent", ""); + } + } + + private static class IntermediateParentStructSdBuilder extends DescendantStructSdBuilder { + public IntermediateParentStructSdBuilder() { + super("parent", "", "grandparent", "gp_"); + } + } + + private static Schema buildChildSearch(String parentSdContent, String sdContent) throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(parentSdContent); + builder.addSchema(sdContent); + builder.build(true); + return builder.getSchema("child"); + } + + private static Schema buildChildSearch(String grandParentSdContent, String parentSdContent, String sdContent) throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(grandParentSdContent); + builder.addSchema(parentSdContent); + builder.addSchema(sdContent); + builder.build(true); + return builder.getSchema("child"); + } + + private static class AncestorPosSdBuilder extends NamedSdBuilder { + public AncestorPosSdBuilder(String name, String fieldPrefix) { + super(name, fieldPrefix); + } + + public String build() { + return joinLines("search " + name + " {", + " document " + name + " {", + "field " + prefixedFieldName("pos") + " type position {", + "indexing: attribute | summary", + " }", + " }", + "}"); + } + } + + private static class ParentPosSdBuilder extends AncestorPosSdBuilder { + public ParentPosSdBuilder() { super("parent", ""); } + } + + private static class GrandParentPosSdBuilder extends AncestorPosSdBuilder { + public GrandParentPosSdBuilder() { super("grandparent", "gp_"); } + } + + private static class DescendantPosSdBuilder extends DescendantSdBuilder { + private boolean import_pos_zcurve_before; + + public DescendantPosSdBuilder(String name, String fieldPrefix, String parentName, String parentFieldPrefix) { + super(name, fieldPrefix, parentName, parentFieldPrefix); + import_pos_zcurve_before = false; + } + + DescendantPosSdBuilder import_pos_zcurve_before(boolean v) { import_pos_zcurve_before = v; return this; } + + public String build() { + return joinLines("search " + name + " {", + " document " + name + " {", + " field " + parentRef() + " type reference<" + parentName + "> {", + " indexing: attribute | summary", + " }", + " }", + importPosZCurve(import_pos_zcurve_before), + importParentField("pos"), + "}"); + } + + private static String importPosZCurve(boolean doImport) { + return doImport ? "import field parent_ref.pos_zcurve as my_pos_zcurve {}" : ""; + } + } + + private static class ChildPosSdBuilder extends DescendantPosSdBuilder { + public ChildPosSdBuilder() { + super("child", "my_", "parent", ""); + } + } + + private static class IntermediateParentPosSdBuilder extends DescendantPosSdBuilder { + public IntermediateParentPosSdBuilder() { + super("parent", "", "grandparent", "gp_"); + } + } + + private static void checkPosImport(ParentPosSdBuilder parentBuilder, DescendantPosSdBuilder childBuilder) throws ParseException { + Schema schema = buildChildSearch(parentBuilder.build(), childBuilder.build()); + checkImportedPosFields(schema); + } + + private static void checkNestedPosImport(GrandParentPosSdBuilder grandParentBuilder, DescendantPosSdBuilder childBuilder) throws ParseException { + Schema schema = buildChildSearch(grandParentBuilder.build(), new IntermediateParentPosSdBuilder().build(), childBuilder.build()); + checkImportedPosFields(schema); + } + + private static void checkImportedPosFields(Schema schema) { + assertEquals(2, schema.importedFields().get().fields().size()); + assertSearchContainsImportedField("my_pos_zcurve", "parent_ref", "parent", "pos_zcurve", schema); + assertSearchContainsImportedField("my_pos", "parent_ref", "parent", "pos", schema); + } + + @Test + public void check_pos_import() throws ParseException { + checkPosImport(new ParentPosSdBuilder(), new ChildPosSdBuilder()); + } + + @Test + public void check_nested_pos_import() throws ParseException { + checkNestedPosImport(new GrandParentPosSdBuilder(), new ChildPosSdBuilder()); + } + + @Test + public void check_pos_import_after_pos_zcurve_import() throws ParseException { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For schema 'child', import field 'my_pos_zcurve': Field 'pos_zcurve' via reference field 'parent_ref': Field already imported"); + checkPosImport(new ParentPosSdBuilder(), new ChildPosSdBuilder().import_pos_zcurve_before(true)); + } + + private static ImportedField getImportedField(String name, Schema schema) { + if (name.contains(".")) { + assertNull(schema.importedFields().get().fields().get(name)); + String superFieldName = name.substring(0,name.indexOf(".")); + String subFieldName = name.substring(name.indexOf(".")+1); + ImportedField superField = schema.importedFields().get().fields().get(superFieldName); + if (superField != null && superField instanceof ImportedComplexField) { + return ((ImportedComplexField)superField).getNestedField(subFieldName); + } + return null; + } + return schema.importedFields().get().fields().get(name); + } + + private static void assertSearchNotContainsImportedField(String fieldName, Schema schema) { + ImportedField importedField = getImportedField(fieldName, schema); + assertNull(importedField); + } + + private static void assertSearchContainsImportedField(String fieldName, + String referenceFieldName, + String referenceDocType, + String targetFieldName, + Schema schema) { + ImportedField importedField = getImportedField(fieldName, schema); + assertNotNull(importedField); + assertEquals(fieldName, importedField.fieldName()); + assertEquals(referenceFieldName, importedField.reference().referenceField().getName()); + assertEquals(referenceDocType, importedField.reference().targetSearch().getName()); + assertEquals(targetFieldName, importedField.targetField().getName()); + } + + private static void checkImportedField(String fieldName, String referenceFieldName, String referenceDocType, + String targetFieldName, Schema schema, boolean present) { + if (present) { + assertSearchContainsImportedField(fieldName, referenceFieldName, referenceDocType, targetFieldName, schema); + } else { + assertSearchNotContainsImportedField(fieldName, schema); + } + } + + @Test + public void field_with_struct_field_attributes_can_be_imported_from_parents_that_use_inheritance() throws ParseException { + var builder = buildParentsUsingInheritance(); + + assertParentContainsEntriesAttributes(builder.getSchema("parent_a")); + assertParentContainsEntriesAttributes(builder.getSchema("parent_b")); + + var child = builder.getSchema("child"); + checkImportedField("entries_from_a", "ref_parent_a", "parent_a", "entries", child, true); + checkImportedField("entries_from_a.key", "ref_parent_a", "parent_a", "entries.key", child, true); + checkImportedField("entries_from_a.value", "ref_parent_a", "parent_a", "entries.value", child, true); + + checkImportedField("entries_from_b", "ref_parent_b", "parent_b", "entries", child, true); + checkImportedField("entries_from_b.key", "ref_parent_b", "parent_b", "entries.key", child, true); + checkImportedField("entries_from_b.value", "ref_parent_b", "parent_b", "entries.value", child, true); + } + + private void assertParentContainsEntriesAttributes(Schema parent) { + var attrs = new AttributeFields(parent); + assertTrue(attrs.containsAttribute("entries.key")); + assertTrue(attrs.containsAttribute("entries.value")); + } + + private ApplicationBuilder buildParentsUsingInheritance() throws ParseException { + var builder = new ApplicationBuilder(); + builder.addSchema(joinLines("schema parent_a {", + "document parent_a {", + " struct Entry {", + " field key type string {}", + " field value type string {}", + " }", + " field entries type array<Entry> {", + " indexing: summary", + " struct-field key { indexing: attribute }", + " struct-field value { indexing: attribute }", + " }", + "}", + "}")); + + builder.addSchema(joinLines("schema parent_b {", + "document parent_b inherits parent_a {", + "}", + "}")); + + builder.addSchema(joinLines("schema child {", + "document child {", + " field ref_parent_a type reference<parent_a> {", + " indexing: attribute", + " }", + " field ref_parent_b type reference<parent_b> {", + " indexing: attribute", + " }", + "}", + "import field ref_parent_a.entries as entries_from_a {}", + "import field ref_parent_b.entries as entries_from_b {}", + "}")); + + builder.build(true); + return builder; + } + + } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingInputsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingInputsTestCase.java new file mode 100644 index 00000000000..71c79feedc1 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingInputsTestCase.java @@ -0,0 +1,45 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; + +/** + * @author Simon Thoresen Hult + */ +public class IndexingInputsTestCase { + + @Test + public void requireThatExtraFieldInputExtraFieldThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_extra_field_input_extra_field.sd", + "For schema 'indexing_extra_field_input_extra_field', field 'bar': Indexing script refers " + + "to field 'bar' which does not exist in document type " + + "'indexing_extra_field_input_extra_field', and is not a mutable attribute."); + } + + @Test + public void requireThatExtraFieldInputImplicitThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_extra_field_input_implicit.sd", + "For schema 'indexing_extra_field_input_implicit', field 'foo': Indexing script refers to " + + "field 'foo' which does not exist in document type 'indexing_extra_field_input_implicit', and is not a mutable attribute."); + } + + @Test + public void requireThatExtraFieldInputNullThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_extra_field_input_null.sd", + "For schema 'indexing_extra_field_input_null', field 'foo': Indexing script refers to field " + + "'foo' which does not exist in document type 'indexing_extra_field_input_null', and is not a mutable attribute."); + } + + @Test + public void requireThatExtraFieldInputSelfThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_extra_field_input_self.sd", + "For schema 'indexing_extra_field_input_self', field 'foo': Indexing script refers to field " + + "'foo' which does not exist in document type 'indexing_extra_field_input_self', and is not a mutable attribute."); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java new file mode 100644 index 00000000000..687549f920e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; + + +/** + * @author Simon Thoresen Hult + */ +public class IndexingOutputsTestCase { + + @Test + public void requireThatOutputOtherFieldThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_output_other_field.sd", + "For schema 'indexing_output_other_field', field 'foo': Indexing expression 'index bar' " + + "attempts to write to a field other than 'foo'."); + } + + @Test + public void requireThatOutputConflictThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_output_conflict.sd", + "For schema 'indexing_output_confict', field 'bar': For expression 'index bar': Attempting " + + "to assign conflicting values to field 'bar'."); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java new file mode 100644 index 00000000000..76cb6a5505c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java @@ -0,0 +1,200 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.document.DataType; +import com.yahoo.schema.Index; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.BooleanIndexDefinition; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.util.Arrays; +import java.util.OptionalDouble; +import java.util.OptionalInt; +import java.util.OptionalLong; +import java.util.Set; + +import static com.yahoo.schema.processing.AssertIndexingScript.assertIndexing; +import static org.junit.Assert.assertEquals; + +/** + * @author Simon Thoresen Hult + */ +public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { + + @Test + public void testSetLanguageRewriting() { + assertIndexingScript("{ input test | set_language; }", + createField("test", DataType.STRING, "{ set_language }")); + } + + @Test + public void testSummaryRewriting() { + assertIndexingScript("{ input test | summary test; }", + createField("test", DataType.STRING, "{ summary }")); + } + + @Test + public void testDynamicSummaryRewriting() { + SDField field = createField("test", DataType.STRING, "{ summary }"); + field.addSummaryField(createDynamicSummaryField(field, "dyn")); + assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | summary dyn; }", field); + } + + @Test + public void testSummaryRewritingWithIndexing() { + assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | summary test | index test; }", + createField("test", DataType.STRING, "{ summary | index }")); + } + + @Test + public void testDynamicAndStaticSummariesRewritingWithIndexing() { + SDField field = createField("test", DataType.STRING, "{ summary | index }"); + field.addSummaryField(createDynamicSummaryField(field, "dyn")); + field.addSummaryField(createStaticSummaryField(field, "test")); + field.addSummaryField(createStaticSummaryField(field, "other")); + field.addSummaryField(createDynamicSummaryField(field, "dyn2")); + assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | summary dyn | summary dyn2 | summary other | " + + "summary test | index test; }", field); + } + + @Test + public void testIntSummaryRewriting() { + assertIndexingScript("{ input test | summary test | attribute test; }", + createField("test", DataType.INT, "{ summary | index }")); + } + + @Test + public void testStringAttributeSummaryRewriting() { + assertIndexingScript("{ input test | summary test | attribute test; }", + createField("test", DataType.STRING, "{ summary | attribute }")); + } + + @Test + public void testMultiblockTokenize() { + SDField field = createField("test", DataType.STRING, + "{ input test | tokenize | { summary test; }; }"); + assertIndexingScript("{ input test | tokenize | { summary test; }; }", field); + } + + @Test + public void requireThatOutputDefaultsToCurrentField() { + assertIndexingScript("{ input test | attribute test; }", + createField("test", DataType.STRING, "{ attribute; }")); + assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | index test; }", + createField("test", DataType.STRING, "{ index; }")); + assertIndexingScript("{ input test | summary test; }", + createField("test", DataType.STRING, "{ summary; }")); + } + + @Test + public void testTokenizeComparisonDisregardsConfig() { + assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | summary test | index test; }", + createField("test", DataType.STRING, "{ summary | tokenize | index; }")); + } + + @Test + public void testDerivingFromSimple() throws Exception { + assertIndexing(Arrays.asList("clear_state | guard { input access | attribute access; }", + "clear_state | guard { input category | split \";\" | attribute category_arr; }", + "clear_state | guard { input category | tokenize | index category; }", + "clear_state | guard { input categories_src | lowercase | normalize | tokenize normalize stem:\"BEST\" | index categories; }", + "clear_state | guard { input categoriesagain_src | lowercase | normalize | tokenize normalize stem:\"BEST\" | index categoriesagain; }", + "clear_state | guard { input chatter | tokenize normalize stem:\"BEST\" | index chatter; }", + "clear_state | guard { input description | tokenize normalize stem:\"BEST\" | summary description | summary dyndesc | index description; }", + "clear_state | guard { input exactemento_src | lowercase | tokenize normalize stem:\"BEST\" | index exactemento | summary exactemento; }", + "clear_state | guard { input longdesc | tokenize normalize stem:\"BEST\" | summary dyndesc2 | summary dynlong | summary longdesc | summary longstat; }", + "clear_state | guard { input measurement | attribute measurement | summary measurement; }", + "clear_state | guard { input measurement | to_array | attribute measurement_arr; }", + "clear_state | guard { input popularity | attribute popularity; }", + "clear_state | guard { input popularity * input measurement | attribute popsiness; }", + "clear_state | guard { input smallattribute | attribute smallattribute; }", + "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }", + "clear_state | guard { input title . \" \" . input category | tokenize | summary exact | index exact; }"), + ApplicationBuilder.buildFromFile("src/test/examples/simple.sd")); + } + + @Test + public void testIndexRewrite() throws Exception { + assertIndexing( + Arrays.asList("clear_state | guard { input title_src | lowercase | normalize | " + + " tokenize | index title; }", + "clear_state | guard { input title_src | summary title_s; }"), + ApplicationBuilder.buildFromFile("src/test/examples/indexrewrite.sd")); + } + + @Test + public void requireThatPredicateFieldsGetOptimization() { + assertIndexingScript("{ 10 | set_var arity | { input test | optimize_predicate | attribute test; }; }", + createPredicateField( + "test", DataType.PREDICATE, "{ attribute; }", 10, OptionalLong.empty(), OptionalLong.empty())); + assertIndexingScript("{ 10 | set_var arity | { input test | optimize_predicate | summary test | attribute test; }; }", + createPredicateField( + "test", DataType.PREDICATE, "{ summary | attribute ; }", 10, OptionalLong.empty(), OptionalLong.empty())); + assertIndexingScript( + "{ 2 | set_var arity | 0L | set_var lower_bound | 1023L | set_var upper_bound | " + + "{ input test | optimize_predicate | attribute test; }; }", + createPredicateField("test", DataType.PREDICATE, "{ attribute; }", 2, OptionalLong.of(0L), OptionalLong.of(1023L))); + } + + private static void assertIndexingScript(String expectedScript, SDField unprocessedField) { + assertEquals(expectedScript, + processField(unprocessedField).toString()); + } + + private static ScriptExpression processField(SDField unprocessedField) { + SDDocumentType sdoc = new SDDocumentType("test"); + sdoc.addField(unprocessedField); + Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); + schema.addDocument(sdoc); + new Processing().process(schema, new BaseDeployLogger(), new RankProfileRegistry(), + new QueryProfiles(), true, false, Set.of()); + return unprocessedField.getIndexingScript(); + } + + private static SDField createField(String name, DataType type, String script) { + SDField field = new SDField(null, name, type); + field.parseIndexingScript(script); + return field; + } + + private static SDField createPredicateField( + String name, DataType type, String script, int arity, OptionalLong lower_bound, OptionalLong upper_bound) { + SDField field = new SDField(null, name, type); + field.parseIndexingScript(script); + Index index = new Index("foo"); + index.setBooleanIndexDefiniton(new BooleanIndexDefinition( + OptionalInt.of(arity), lower_bound, upper_bound, OptionalDouble.empty())); + field.addIndex(index); + return field; + } + + private static SummaryField createDynamicSummaryField(SDField field, String name) { + return createSummaryField(field, name, true); + } + + private static SummaryField createStaticSummaryField(SDField field, String name) { + return createSummaryField(field, name, false); + } + + private static SummaryField createSummaryField(SDField field, String name, boolean dynamic) { + SummaryField summaryField = new SummaryField(name, field.getDataType()); + if (dynamic) { + summaryField.setTransform(SummaryTransform.DYNAMICTEASER); + } + summaryField.addDestination("default"); + summaryField.addSource(field.getName()); + return summaryField; + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java new file mode 100644 index 00000000000..4da6880aa26 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java @@ -0,0 +1,76 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.derived.AbstractExportingTestCase; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; + +import static com.yahoo.schema.processing.AssertIndexingScript.assertIndexing; +import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; + +/** + * @author Simon Thoresen Hult + */ +public class IndexingValidationTestCase extends AbstractExportingTestCase { + + @Test + public void testAttributeChanged() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_attribute_changed.sd", + "For schema 'indexing_attribute_changed', field 'foo': For expression 'attribute foo': " + + "Attempting to assign conflicting values to field 'foo'."); + } + + @Test + public void testAttributeOther() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_attribute_other.sd", + "For schema 'indexing_attribute_other', field 'foo': Indexing expression 'attribute bar' " + + "attempts to write to a field other than 'foo'."); + } + + @Test + public void testIndexChanged() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_index_changed.sd", + "For schema 'indexing_index_changed', field 'foo': For expression 'index foo': " + + "Attempting to assign conflicting values to field 'foo'."); + } + + @Test + public void testIndexOther() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_index_other.sd", + "For schema 'indexing_index_other', field 'foo': Indexing expression 'index bar' " + + "attempts to write to a field other than 'foo'."); + } + + @Test + public void testSummaryChanged() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_summary_changed.sd", + "For schema 'indexing_summary_fail', field 'foo': For expression 'summary foo': Attempting " + + "to assign conflicting values to field 'foo'."); + } + + @Test + public void testSummaryOther() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_summary_other.sd", + "For schema 'indexing_summary_other', field 'foo': Indexing expression 'summary bar' " + + "attempts to write to a field other than 'foo'."); + } + + @Test + public void testExtraField() throws IOException, ParseException { + assertIndexing( + Arrays.asList("clear_state | guard { input my_index | tokenize normalize stem:\"BEST\" | index my_index | summary my_index }", + "clear_state | guard { input my_input | tokenize normalize stem:\"BEST\" | index my_extra | summary my_extra }"), + ApplicationBuilder.buildFromFile("src/test/examples/indexing_extra.sd")); + } + + @Test + public void requireThatMultilineOutputConflictThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_multiline_output_conflict.sd", + "For schema 'indexing_multiline_output_confict', field 'cox': For expression 'index cox': " + + "Attempting to assign conflicting values to field 'cox'."); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingValuesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValuesTestCase.java new file mode 100644 index 00000000000..2784fe69b28 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValuesTestCase.java @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; +import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuilds; + +/** + * @author Simon Thoresen Hult + */ +public class IndexingValuesTestCase { + + @Test + public void requireThatModifyFieldNoOutputDoesNotThrow() throws IOException, ParseException { + assertBuilds("src/test/examples/indexing_modify_field_no_output.sd"); + } + + @Test + public void requireThatInputOtherFieldThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_input_other_field.sd", + "For schema 'indexing_input_other_field', field 'bar': Indexing expression 'input foo' " + + "attempts to modify the value of the document field 'bar'. " + + "Use a field outside the document block instead."); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IntegerIndex2AttributeTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IntegerIndex2AttributeTestCase.java new file mode 100644 index 00000000000..f36effab146 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/IntegerIndex2AttributeTestCase.java @@ -0,0 +1,61 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author baldersheim + */ +public class IntegerIndex2AttributeTestCase extends AbstractSchemaTestCase { + + @Test + public void testIntegerIndex2Attribute() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/integerindex2attribute.sd"); + new IntegerIndex2Attribute(schema, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); + + SDField f; + f = schema.getConcreteField("s1"); + assertTrue(f.getAttributes().isEmpty()); + assertTrue(f.existsIndex("s1")); + f = schema.getConcreteField("s2"); + assertEquals(f.getAttributes().size(), 1); + assertTrue(f.existsIndex("s2")); + + f = schema.getConcreteField("as1"); + assertTrue(f.getAttributes().isEmpty()); + assertTrue(f.existsIndex("as1")); + f = schema.getConcreteField("as2"); + assertEquals(f.getAttributes().size(), 1); + assertTrue(f.existsIndex("as2")); + + f = schema.getConcreteField("i1"); + assertEquals(f.getAttributes().size(), 1); + assertFalse(f.existsIndex("i1")); + + f = schema.getConcreteField("i2"); + assertEquals(f.getAttributes().size(), 1); + assertFalse(f.existsIndex("i2")); + + f = schema.getConcreteField("ai1"); + assertEquals(schema.getConcreteField("ai1").getAttributes().size(), 1); + assertFalse(schema.getConcreteField("ai1").existsIndex("ai1")); + f = schema.getConcreteField("ai2"); + assertEquals(f.getAttributes().size(), 1); + assertFalse(f.existsIndex("ai2")); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/MatchPhaseSettingsValidatorTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/MatchPhaseSettingsValidatorTestCase.java new file mode 100644 index 00000000000..530b6a95ce8 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/MatchPhaseSettingsValidatorTestCase.java @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import org.junit.Test; + +import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; + +public class MatchPhaseSettingsValidatorTestCase { + + private static String getMessagePrefix() { + return "In search definition 'test', rank-profile 'default': match-phase attribute 'foo' "; + } + + @Test + public void requireThatAttributeMustExists() throws Exception { + assertBuildFails("src/test/examples/matchphase/non_existing_attribute.sd", + getMessagePrefix() + "does not exists"); + } + + @Test + public void requireThatAttributeMustBeNumeric() throws Exception { + assertBuildFails("src/test/examples/matchphase/wrong_data_type_attribute.sd", + getMessagePrefix() + "must be single value numeric, but it is 'string'"); + } + + @Test + public void requireThatAttributeMustBeSingleValue() throws Exception { + assertBuildFails("src/test/examples/matchphase/wrong_collection_type_attribute.sd", + getMessagePrefix() + "must be single value numeric, but it is 'Array<int>'"); + } + + @Test + public void requireThatAttributeMustHaveFastSearch() throws Exception { + assertBuildFails("src/test/examples/matchphase/non_fast_search_attribute.sd", + getMessagePrefix() + "must be fast-search, but it is not"); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/MatchedElementsOnlyResolverTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/MatchedElementsOnlyResolverTestCase.java new file mode 100644 index 00000000000..c401376ac3a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/MatchedElementsOnlyResolverTestCase.java @@ -0,0 +1,192 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.Assert.assertEquals; + +/** + * @author geirst + */ +public class MatchedElementsOnlyResolverTestCase { + + @SuppressWarnings("deprecation") + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void complex_field_with_some_struct_field_attributes_gets_default_transform() throws ParseException { + assertSummaryField(joinLines("field my_field type map<string, string> {", + " indexing: summary", + " summary: matched-elements-only", + " struct-field key { indexing: attribute }", + "}"), + "my_field", SummaryTransform.MATCHED_ELEMENTS_FILTER); + + assertSummaryField(joinLines("field my_field type map<string, elem> {", + " indexing: summary", + " summary: matched-elements-only", + " struct-field key { indexing: attribute }", + "}"), + "my_field", SummaryTransform.MATCHED_ELEMENTS_FILTER); + + assertSummaryField(joinLines("field my_field type array<elem> {", + " indexing: summary", + " summary: matched-elements-only", + " struct-field name { indexing: attribute }", + "}"), + "my_field", SummaryTransform.MATCHED_ELEMENTS_FILTER); + } + + @Test + public void complex_field_with_only_struct_field_attributes_gets_attribute_transform() throws ParseException { + assertSummaryField(joinLines("field my_field type map<string, string> {", + " indexing: summary", + " summary: matched-elements-only", + " struct-field key { indexing: attribute }", + " struct-field value { indexing: attribute }", + "}"), + "my_field", SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); + + assertSummaryField(joinLines("field my_field type map<string, elem> {", + " indexing: summary", + " summary: matched-elements-only", + " struct-field key { indexing: attribute }", + " struct-field value.name { indexing: attribute }", + " struct-field value.weight { indexing: attribute }", + "}"), + "my_field", SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); + + assertSummaryField(joinLines("field my_field type array<elem> {", + " indexing: summary", + " summary: matched-elements-only", + " struct-field name { indexing: attribute }", + " struct-field weight { indexing: attribute }", + "}"), + "my_field", SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); + } + + @Test + public void explicit_complex_summary_field_can_use_filter_transform_with_reference_to_source_field() throws ParseException { + String documentSummary = joinLines("document-summary my_summary {", + " summary my_filter_field type map<string, string> {", + " source: my_field", + " matched-elements-only", + " }", + "}"); + { + var search = buildSearch(joinLines("field my_field type map<string, string> {", + " indexing: summary", + " struct-field key { indexing: attribute }", + "}"), + documentSummary); + assertSummaryField(search.getSummaryField("my_filter_field"), + SummaryTransform.MATCHED_ELEMENTS_FILTER, "my_field"); + assertSummaryField(search.getSummaryField("my_field"), + SummaryTransform.NONE, "my_field"); + } + { + var search = buildSearch(joinLines("field my_field type map<string, string> {", + " indexing: summary", + " struct-field key { indexing: attribute }", + " struct-field value { indexing: attribute }", + "}"), + documentSummary); + assertSummaryField(search.getSummaryField("my_filter_field"), + SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER, "my_field"); + assertSummaryField(search.getSummaryField("my_field"), + SummaryTransform.ATTRIBUTECOMBINER, "my_field"); + } + } + + @Test + public void primitive_array_attribute_field_gets_attribute_transform() throws ParseException { + assertSummaryField(joinLines("field my_field type array<string> {", + " indexing: attribute | summary", + " summary: matched-elements-only", + "}"), + "my_field", SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); + } + + @Test + public void primitive_weighted_set_attribute_field_gets_attribute_transform() throws ParseException { + assertSummaryField(joinLines("field my_field type weightedset<string> {", + " indexing: attribute | summary", + " summary: matched-elements-only", + "}"), + "my_field", SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); + } + + @Test + public void explicit_summary_field_can_use_filter_transform_with_reference_to_attribute_source_field() throws ParseException { + String documentSummary = joinLines("document-summary my_summary {", + " summary my_filter_field type array<string> {", + " source: my_field", + " matched-elements-only", + " }", + "}"); + + var search = buildSearch(joinLines( + "field my_field type array<string> {", + " indexing: attribute | summary", + "}"), + documentSummary); + assertSummaryField(search.getSummaryField("my_filter_field"), + SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER, "my_field"); + assertSummaryField(search.getSummaryField("my_field"), + SummaryTransform.ATTRIBUTE, "my_field"); + } + + @Test + public void unsupported_field_type_throws() throws ParseException { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For schema 'test', document summary 'default', summary field 'my_field': " + + "'matched-elements-only' is not supported for this field type. " + + "Supported field types are: array of primitive, weighted set of primitive, " + + "array of simple struct, map of primitive type to simple struct, " + + "and map of primitive type to primitive type"); + buildSearch(joinLines("field my_field type string {", + " indexing: summary", + " summary: matched-elements-only", + "}")); + } + + private void assertSummaryField(String fieldContent, String fieldName, SummaryTransform expTransform) throws ParseException { + var search = buildSearch(fieldContent); + assertSummaryField(search.getSummaryField(fieldName), expTransform, fieldName); + } + + private void assertSummaryField(SummaryField field, SummaryTransform expTransform, String expSourceField) { + assertEquals(expTransform, field.getTransform()); + assertEquals(expSourceField, field.getSingleSource()); + } + + private Schema buildSearch(String field) throws ParseException { + return buildSearch(field, ""); + } + + private Schema buildSearch(String field, String summary) throws ParseException { + var builder = new ApplicationBuilder(new RankProfileRegistry()); + builder.addSchema(joinLines("search test {", + " document test {", + " struct elem {", + " field name type string {}", + " field weight type int {}", + " }", + field, + " }", + summary, + "}")); + builder.build(true); + return builder.getSchema(); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java new file mode 100644 index 00000000000..912e6fcf030 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java @@ -0,0 +1,88 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; + +/** + * @author bratseth + */ +public class NGramTestCase extends AbstractSchemaTestCase { + + @Test + public void testNGram() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/ngram.sd"); + assertNotNull(schema); + + SDField gram1 = schema.getConcreteField("gram_1"); + assertEquals(MatchType.GRAM, gram1.getMatching().getType()); + assertEquals(1, gram1.getMatching().getGramSize()); + + SDField gram2 = schema.getConcreteField("gram_2"); + assertEquals(MatchType.GRAM, gram2.getMatching().getType()); + assertEquals(-1, gram2.getMatching().getGramSize()); // Not set explicitly + + SDField gram3= schema.getConcreteField("gram_3"); + assertEquals(MatchType.GRAM,gram3.getMatching().getType()); + assertEquals(3, gram3.getMatching().getGramSize()); + + assertEquals("input gram_1 | ngram 1 | index gram_1 | summary gram_1", gram1.getIndexingScript().iterator().next().toString()); + assertEquals("input gram_2 | ngram 2 | attribute gram_2 | index gram_2", gram2.getIndexingScript().iterator().next().toString()); + assertEquals("input gram_3 | ngram 3 | index gram_3", gram3.getIndexingScript().iterator().next().toString()); + + assertFalse(gram1.getNormalizing().doRemoveAccents()); + assertEquals(Stemming.NONE, gram1.getStemming()); + + List<String> queryCommands = gram1.getQueryCommands(); + assertEquals(2, queryCommands.size()); + assertEquals("ngram 1", queryCommands.get(1)); + } + + @Test + public void testInvalidNGramSetting1() throws IOException, ParseException { + try { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/invalidngram1.sd"); + fail("Should cause an exception"); + } + catch (IllegalArgumentException e) { + assertEquals("gram-size can only be set when the matching mode is 'gram'", e.getMessage()); + } + } + + @Test + public void testInvalidNGramSetting2() throws IOException, ParseException { + try { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/invalidngram2.sd"); + fail("Should cause an exception"); + } + catch (IllegalArgumentException e) { + assertEquals("gram-size can only be set when the matching mode is 'gram'", e.getMessage()); + } + } + + @Test + public void testInvalidNGramSetting3() throws IOException, ParseException { + try { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/invalidngram3.sd"); + fail("Should cause an exception"); + } + catch (IllegalArgumentException e) { + assertEquals("gram matching is not supported with attributes, use 'index' in indexing", e.getMessage()); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java new file mode 100644 index 00000000000..a291dda24b9 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java @@ -0,0 +1,119 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.util.Optional; + +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static com.yahoo.schema.ApplicationBuilder.createFromString; +import static com.yahoo.schema.ApplicationBuilder.createFromStrings; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class PagedAttributeValidatorTestCase { + + @Test + public void dense_tensor_attribute_supports_paged_setting() throws ParseException { + assertPagedSupported("tensor(x[2],y[2])"); + } + + @Test + public void primitive_attribute_types_support_paged_setting() throws ParseException { + assertPagedSupported("int"); + assertPagedSupported("array<int>"); + assertPagedSupported("weightedset<int>"); + + assertPagedSupported("string"); + assertPagedSupported("array<string>"); + assertPagedSupported("weightedset<string>"); + } + + @Test + public void struct_field_attributes_support_paged_setting() throws ParseException { + var sd = joinLines("schema test {", + " document test {", + " struct elem {", + " field first type int {}", + " field second type string {}", + " }", + " field foo type array<elem> {", + " indexing: summary", + " struct-field first {", + " indexing: attribute", + " attribute: paged", + " }", + " struct-field second {", + " indexing: attribute", + " attribute: paged", + " }", + " }", + " }", + "}"); + + var appBuilder = createFromString(sd); + var field = appBuilder.getSchema().getField("foo"); + assertTrue(field.getStructField("first").getAttribute().isPaged()); + assertTrue(field.getStructField("second").getAttribute().isPaged()); + } + + private void assertPagedSupported(String fieldType) throws ParseException { + var appBuilder = createFromString(getSd(fieldType)); + var attribute = appBuilder.getSchema().getAttribute("foo"); + assertTrue(attribute.isPaged()); + } + + @Test + public void non_dense_tensor_attribute_does_not_support_paged_setting() throws ParseException { + assertPagedSettingNotSupported("tensor(x{},y[2])"); + } + + @Test + public void predicate_attribute_does_not_support_paged_setting() throws ParseException { + assertPagedSettingNotSupported("predicate"); + } + + @Test + public void reference_attribute_does_not_support_paged_setting() throws ParseException { + assertPagedSettingNotSupported("reference<parent>", Optional.of(getSd("parent", "int"))); + } + + private void assertPagedSettingNotSupported(String fieldType) throws ParseException { + assertPagedSettingNotSupported(fieldType, Optional.empty()); + } + + private void assertPagedSettingNotSupported(String fieldType, Optional<String> parentSd) throws ParseException { + try { + if (parentSd.isPresent()) { + createFromStrings(new BaseDeployLogger(), parentSd.get(), getSd(fieldType)); + } else { + createFromString(getSd(fieldType)); + } + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'foo': The 'paged' attribute setting is not supported for non-dense tensor, predicate and reference types", + e.getMessage()); + } + } + + private String getSd(String fieldType) { + return getSd("test", fieldType); + } + + private String getSd(String docType, String fieldType) { + return joinLines( + "schema " + docType + " {", + " document " + docType + " {", + " field foo type " + fieldType + "{", + " indexing: attribute", + " attribute: paged", + " }", + " }", + "}"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ParentChildSearchModel.java b/config-model/src/test/java/com/yahoo/schema/processing/ParentChildSearchModel.java new file mode 100644 index 00000000000..e5636da57a0 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/ParentChildSearchModel.java @@ -0,0 +1,64 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.document.DataType; +import com.yahoo.documentmodel.NewDocumentReferenceDataType; +import com.yahoo.schema.DocumentReference; +import com.yahoo.schema.DocumentReferences; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.TestableDeployLogger; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.TemporaryImportedField; +import com.yahoo.schema.document.TemporarySDField; + +/* + * Fixture class used for ImportedFieldsResolverTestCase and AdjustPositionSummaryFieldsTestCase. + */ +public class ParentChildSearchModel { + + public Schema parentSchema; + public Schema childSchema; + + ParentChildSearchModel() { + parentSchema = createSearch("parent"); + childSchema = createSearch("child"); + } + + protected Schema createSearch(String name) { + Schema result = new Schema(name, MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); + result.addDocument(new SDDocumentType(name)); + return result; + } + + protected static TemporarySDField createField(SDDocumentType repo, String name, DataType dataType, String indexingScript) { + TemporarySDField result = new TemporarySDField(repo, name, dataType); + result.parseIndexingScript(indexingScript); + return result; + } + + @SuppressWarnings("deprecation") + protected static SDField createRefField(SDDocumentType repo, String parentType, String fieldName) { + return new TemporarySDField(repo, fieldName, NewDocumentReferenceDataType.forDocumentName(parentType)); + } + + protected static void addRefField(Schema child, Schema parent, String fieldName) { + SDField refField = createRefField(child.getDocument(), parent.getName(), fieldName); + child.getDocument().addField(refField); + child.getDocument().setDocumentReferences(new DocumentReferences(ImmutableMap.of(refField.getName(), + new DocumentReference(refField, parent)))); + } + + protected ParentChildSearchModel addImportedField(String fieldName, String referenceFieldName, String targetFieldName) { + return addImportedField(childSchema, fieldName, referenceFieldName, targetFieldName); + } + + protected ParentChildSearchModel addImportedField(Schema schema, String fieldName, String referenceFieldName, String targetFieldName) { + schema.temporaryImportedFields().get().add(new TemporaryImportedField(fieldName, referenceFieldName, targetFieldName)); + return this; + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/PositionTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/PositionTestCase.java new file mode 100644 index 00000000000..6f0facf9541 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/PositionTestCase.java @@ -0,0 +1,130 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.PositionDataType; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.FieldSet; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; + +import org.junit.Test; + +import java.util.Arrays; +import java.util.Iterator; + +import static org.junit.Assert.*; + +/** + * Test Position processor. + * + * @author hmusum + */ +public class PositionTestCase { + + @Test + public void inherited_position_zcurve_field_is_not_added_to_document_fieldset() throws Exception { + ApplicationBuilder sb = ApplicationBuilder.createFromFiles(Arrays.asList( + "src/test/examples/position_base.sd", + "src/test/examples/position_inherited.sd")); + + Schema schema = sb.getSchema("position_inherited"); + FieldSet fieldSet = schema.getDocument().getFieldSets().builtInFieldSets().get(DocumentType.DOCUMENT); + assertFalse(fieldSet.getFieldNames().contains(PositionDataType.getZCurveFieldName("pos"))); + } + + @Test + public void requireThatPositionCanBeAttribute() throws Exception { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/position_attribute.sd"); + assertNull(schema.getAttribute("pos")); + assertNull(schema.getAttribute("pos.x")); + assertNull(schema.getAttribute("pos.y")); + + assertPositionAttribute(schema, "pos", Attribute.CollectionType.SINGLE); + assertPositionSummary(schema, "pos", false); + } + + @Test + public void requireThatPositionCanNotBeIndex() throws Exception { + try { + ApplicationBuilder.buildFromFile("src/test/examples/position_index.sd"); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'position_index', field 'pos': Indexing of data type 'position' is not " + + "supported, replace 'index' statement with 'attribute'.", e.getMessage()); + } + } + + @Test + public void requireThatSummaryAloneDoesNotCreateZCurve() throws Exception { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/position_summary.sd"); + assertNull(schema.getAttribute("pos")); + assertNull(schema.getAttribute("pos.x")); + assertNull(schema.getAttribute("pos.y")); + assertNull(schema.getAttribute("pos.zcurve")); + + SummaryField summary = schema.getSummaryField("pos"); + assertNotNull(summary); + assertEquals(2, summary.getSourceCount()); + Iterator<SummaryField.Source> it = summary.getSources().iterator(); + assertEquals("pos.x", it.next().getName()); + assertEquals("pos.y", it.next().getName()); + assertEquals(SummaryTransform.NONE, summary.getTransform()); + + assertNull(schema.getSummaryField("pos_ext.distance")); + } + + @Test + public void requireThatExtraFieldCanBePositionAttribute() throws Exception { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/position_extra.sd"); + assertNull(schema.getAttribute("pos_ext")); + assertNull(schema.getAttribute("pos_ext.x")); + assertNull(schema.getAttribute("pos_ext.y")); + + assertPositionAttribute(schema, "pos_ext", Attribute.CollectionType.SINGLE); + assertPositionSummary(schema, "pos_ext", false); + } + + @Test + public void requireThatPositionArrayIsSupported() throws Exception { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/position_array.sd"); + assertNull(schema.getAttribute("pos")); + assertNull(schema.getAttribute("pos.x")); + assertNull(schema.getAttribute("pos.y")); + + assertPositionAttribute(schema, "pos", Attribute.CollectionType.ARRAY); + assertPositionSummary(schema, "pos", true); + } + + private static void assertPositionAttribute(Schema schema, String fieldName, Attribute.CollectionType type) { + Attribute attribute = schema.getAttribute(PositionDataType.getZCurveFieldName(fieldName)); + assertNotNull(attribute); + assertTrue(attribute.isPosition()); + assertEquals(attribute.getCollectionType(), type); + assertEquals(attribute.getType(), Attribute.Type.LONG); + } + + private static void assertPositionSummary(Schema schema, String fieldName, boolean isArray) { + assertSummaryField(schema, + fieldName, + PositionDataType.getZCurveFieldName(fieldName), + (isArray ? DataType.getArray(PositionDataType.INSTANCE) : PositionDataType.INSTANCE), + SummaryTransform.GEOPOS); + assertNull(schema.getSummaryField(PositionDataType.getDistanceSummaryFieldName(fieldName))); + assertNull(schema.getSummaryField(PositionDataType.getPositionSummaryFieldName(fieldName))); + } + + private static void assertSummaryField(Schema schema, String fieldName, String sourceName, DataType dataType, + SummaryTransform transform) + { + SummaryField summary = schema.getSummaryField(fieldName); + assertNotNull(summary); + assertEquals(1, summary.getSourceCount()); + assertEquals(sourceName, summary.getSingleSource()); + assertEquals(dataType, summary.getDataType()); + assertEquals(transform, summary.getTransform()); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankModifierTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankModifierTestCase.java new file mode 100644 index 00000000000..69bf62be84b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankModifierTestCase.java @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * Tests for the field "rank {" shortcut + * @author vegardh + * + */ +public class RankModifierTestCase extends AbstractSchemaTestCase { + @Test + public void testLiteral() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/rankmodifier/literal.sd"); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/schema/processing/RankProfileSearchFixture.java new file mode 100644 index 00000000000..e380b1ab9af --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankProfileSearchFixture.java @@ -0,0 +1,128 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.google.common.collect.ImmutableList; +import com.yahoo.config.application.api.ApplicationPackage; +import ai.vespa.rankingexpression.importer.configmodelview.MlModelImporter; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.path.Path; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; +import ai.vespa.rankingexpression.importer.onnx.OnnxImporter; +import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter; +import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter; +import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static org.junit.Assert.assertEquals; + +/** + * Helper class for setting up and asserting over a Search instance with a rank profile given literally + * in the search definition language. + * + * @author geirst + */ +class RankProfileSearchFixture { + + private final ImmutableList<MlModelImporter> importers = ImmutableList.of(new TensorFlowImporter(), + new OnnxImporter(), + new LightGBMImporter(), + new XGBoostImporter()); + private final RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + private final QueryProfileRegistry queryProfileRegistry; + private final Schema schema; + private final Map<String, RankProfile> compiledRankProfiles = new HashMap<>(); + private final ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + + public RankProfileRegistry getRankProfileRegistry() { + return rankProfileRegistry; + } + + public QueryProfileRegistry getQueryProfileRegistry() { + return queryProfileRegistry; + } + + RankProfileSearchFixture(String rankProfiles) throws ParseException { + this(MockApplicationPackage.createEmpty(), new QueryProfileRegistry(), rankProfiles); + } + + RankProfileSearchFixture(ApplicationPackage applicationpackage, QueryProfileRegistry queryProfileRegistry, + String rankProfiles) throws ParseException { + this(applicationpackage, queryProfileRegistry, rankProfiles, null, null); + } + + RankProfileSearchFixture(ApplicationPackage applicationpackage, QueryProfileRegistry queryProfileRegistry, + String rankProfiles, String constant, String field) + throws ParseException { + this.queryProfileRegistry = queryProfileRegistry; + ApplicationBuilder builder = new ApplicationBuilder(applicationpackage, new MockFileRegistry(), new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfileRegistry); + String sdContent = "search test {\n" + + " " + (constant != null ? constant : "") + "\n" + + " document test {\n" + + " " + (field != null ? field : "") + "\n" + + " }\n" + + rankProfiles + + "\n" + + "}"; + builder.addSchema(sdContent); + builder.build(true); + schema = builder.getSchema(); + } + + public void assertFirstPhaseExpression(String expExpression, String rankProfile) { + assertEquals(expExpression, compiledRankProfile(rankProfile).getFirstPhaseRanking().getRoot().toString()); + } + + public void assertSecondPhaseExpression(String expExpression, String rankProfile) { + assertEquals(expExpression, compiledRankProfile(rankProfile).getSecondPhaseRanking().getRoot().toString()); + } + + public void assertRankProperty(String expValue, String name, String rankProfile) { + List<RankProfile.RankProperty> rankPropertyList = compiledRankProfile(rankProfile).getRankPropertyMap().get(name); + assertEquals(1, rankPropertyList.size()); + assertEquals(expValue, rankPropertyList.get(0).getValue()); + } + + public void assertFunction(String expexctedExpression, String functionName, String rankProfile) { + assertEquals(expexctedExpression, + compiledRankProfile(rankProfile).getFunctions().get(functionName).function().getBody().getRoot().toString()); + } + + public RankProfile compileRankProfile(String rankProfile) { + return compileRankProfile(rankProfile, Path.fromString("nonexistinng")); + } + + public RankProfile compileRankProfile(String rankProfile, Path applicationDir) { + RankProfile compiled = rankProfileRegistry.get(schema, rankProfile) + .compile(queryProfileRegistry, + new ImportedMlModels(applicationDir.toFile(), executor, importers)); + compiledRankProfiles.put(rankProfile, compiled); + return compiled; + } + + /** Returns the given uncompiled profile */ + public RankProfile rankProfile(String rankProfile) { + return rankProfileRegistry.get(schema, rankProfile); + } + + /** Returns the given compiled profile, or null if not compiled yet or not present at all */ + public RankProfile compiledRankProfile(String rankProfile) { + return compiledRankProfiles.get(rankProfile); + } + + public Schema search() { return schema; } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankPropertyVariablesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankPropertyVariablesTestCase.java new file mode 100644 index 00000000000..dab1d9e6e95 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankPropertyVariablesTestCase.java @@ -0,0 +1,47 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.RankProfile.RankProperty; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.fail; + +public class RankPropertyVariablesTestCase extends AbstractSchemaTestCase { + + @Test + public void testRankPropVariables() throws IOException, ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/rankpropvars.sd", + new BaseDeployLogger(), + rankProfileRegistry, + new QueryProfileRegistry()); + assertRankPropEquals(rankProfileRegistry.get(schema, "other").getRankProperties(), "$testvar1", "foo"); + assertRankPropEquals(rankProfileRegistry.get(schema, "other").getRankProperties(), "$testvar_2", "bar"); + assertRankPropEquals(rankProfileRegistry.get(schema, "other").getRankProperties(), "$testvarOne23", "baz"); + assertRankPropEquals(rankProfileRegistry.get(schema, "another").getRankProperties(), "$Testvar1", "1"); + assertRankPropEquals(rankProfileRegistry.get(schema, "another").getRankProperties(), "$Testvar_4", "4"); + assertRankPropEquals(rankProfileRegistry.get(schema, "another").getRankProperties(), "$testvarFour23", "234234.234"); + } + + private void assertRankPropEquals(List<RankProperty> props, String key, String val) { + for (RankProperty prop : props) { + if (prop.getName().equals(key)) { + if (prop.getValue().equals(val)) { + return; + } + } + } + fail(key+":"+val+ " not found in rank properties."); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionTypeResolverTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionTypeResolverTestCase.java new file mode 100644 index 00000000000..4b6a22fc81a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionTypeResolverTestCase.java @@ -0,0 +1,521 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.TensorFieldType; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.tensor.TensorType; +import com.yahoo.yolean.Exceptions; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.stream.Collectors; + +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; + +/** + * @author bratseth + */ +public class RankingExpressionTypeResolverTestCase { + + @Test + public void tensorFirstPhaseMustProduceDouble() throws Exception { + try { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( + "search test {", + " document test { ", + " field a type tensor(x[10],y[3]) {", + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " first-phase {", + " expression: attribute(a)", + " }", + " }", + "}" + )); + builder.build(true); + fail("Expected exception"); + } + catch (IllegalArgumentException expected) { + assertEquals("In schema 'test', rank profile 'my_rank_profile': The first-phase expression must produce a double (a tensor with no dimensions), but produces tensor(x[10],y[3])", + Exceptions.toMessageString(expected)); + } + } + + + @Test + public void tensorFirstPhaseFromConstantMustProduceDouble() throws Exception { + try { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( + "schema test {", + " document test { ", + " field a type tensor(d0[3]) {", + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " function my_func() {", + " expression: x_tensor*2.0", + " }", + " function inline other_func() {", + " expression: z_tensor+3.0", + " }", + " first-phase {", + " expression: reduce(attribute(a),sum,d0)+y_tensor+my_func+other_func", + " }", + " constants {", + " x_tensor {", // legacy form + " type: tensor(x{})", + " value: { {x:bar}:17 }", + " }", + " y_tensor tensor(y{}):{{y:foo}:42 }", + " z_tensor tensor(z{}):{qux:666}", + " }", + " }", + "}" + )); + builder.build(true); + fail("Expected exception"); + } + catch (IllegalArgumentException expected) { + assertEquals("In schema 'test', rank profile 'my_rank_profile': The first-phase expression must produce a double (a tensor with no dimensions), but produces tensor(x{},y{},z{})", + Exceptions.toMessageString(expected)); + } + } + + + + @Test + public void tensorSecondPhaseMustProduceDouble() throws Exception { + try { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( + "search test {", + " document test { ", + " field a type tensor(x[10],y[3]) {", + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " first-phase {", + " expression: sum(attribute(a))", + " }", + " second-phase {", + " expression: attribute(a)", + " }", + " }", + "}" + )); + builder.build(true); + fail("Expected exception"); + } + catch (IllegalArgumentException expected) { + assertEquals("In schema 'test', rank profile 'my_rank_profile': The second-phase expression must produce a double (a tensor with no dimensions), but produces tensor(x[10],y[3])", + Exceptions.toMessageString(expected)); + } + } + + @Test + public void tensorConditionsMustHaveTypeCompatibleBranches() throws Exception { + try { + ApplicationBuilder schemaBuilder = new ApplicationBuilder(); + schemaBuilder.addSchema(joinLines( + "search test {", + " document test { ", + " field a type tensor(x[10],y[5]) {", + " indexing: attribute", + " }", + " field b type tensor(z[10]) {", + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " first-phase {", + " expression: sum(if(1>0, attribute(a), attribute(b)))", + " }", + " }", + "}" + )); + schemaBuilder.build(true); + fail("Expected exception"); + } + catch (IllegalArgumentException expected) { + assertEquals("In schema 'test', rank profile 'my_rank_profile': The first-phase expression is invalid: An if expression must produce compatible types in both alternatives, but the 'true' type is tensor(x[10],y[5]) while the 'false' type is tensor(z[10])" + + "\n'true' branch: attribute(a)" + + "\n'false' branch: attribute(b)", + Exceptions.toMessageString(expected)); + } + } + + @Test + public void testFunctionInvocationTypes() throws Exception { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema(joinLines( + "search test {", + " document test { ", + " field a type tensor(x[10],y[3]) {", + " indexing: attribute", + " }", + " field b type tensor(z[10]) {", + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " function macro1(attribute_to_use) {", + " expression: attribute(attribute_to_use)", + " }", + " summary-features {", + " macro1(a)", + " macro1(b)", + " }", + " }", + "}" + )); + builder.build(true); + RankProfile profile = + builder.getRankProfileRegistry().get(builder.getSchema(), "my_rank_profile"); + assertEquals(TensorType.fromSpec("tensor(x[10],y[3])"), + summaryFeatures(profile).get("macro1(a)").type(profile.typeContext(builder.getQueryProfileRegistry()))); + assertEquals(TensorType.fromSpec("tensor(z[10])"), + summaryFeatures(profile).get("macro1(b)").type(profile.typeContext(builder.getQueryProfileRegistry()))); + } + + @Test + public void testTensorFunctionInvocationTypes_Nested() throws Exception { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( + "search test {", + " document test { ", + " field a type tensor(x[10],y[1]) {", + " indexing: attribute", + " }", + " field b type tensor(z[10]) {", + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " function return_a() {", + " expression: return_first(attribute(a), attribute(b))", + " }", + " function return_b() {", + " expression: return_second(attribute(a), attribute(b))", + " }", + " function return_first(e1, e2) {", + " expression: e1", + " }", + " function return_second(e1, e2) {", + " expression: return_first(e2, e1)", + " }", + " summary-features {", + " return_a", + " return_b", + " }", + " }", + "}" + )); + builder.build(true); + RankProfile profile = + builder.getRankProfileRegistry().get(builder.getSchema(), "my_rank_profile"); + assertEquals(TensorType.fromSpec("tensor(x[10],y[1])"), + summaryFeatures(profile).get("return_a").type(profile.typeContext(builder.getQueryProfileRegistry()))); + assertEquals(TensorType.fromSpec("tensor(z[10])"), + summaryFeatures(profile).get("return_b").type(profile.typeContext(builder.getQueryProfileRegistry()))); + } + + @Test + public void testAttributeInvocationViaBoundIdentifier() throws Exception { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( + "search newsarticle {", + " document newsarticle {", + " field title type string {", + " indexing {", + " input title | index", + " }", + " weight: 30", + " }", + " field usstaticrank type int {", + " indexing: summary | attribute", + " }", + " field eustaticrank type int {", + " indexing: summary | attribute", + " }", + " }", + " rank-profile default {", + " macro newsboost() { ", + " expression: 200 * matches(title)", + " }", + " macro commonboost(mystaticrank) { ", + " expression: attribute(mystaticrank) + newsboost", + " }", + " macro commonfirstphase(mystaticrank) { ", + " expression: nativeFieldMatch(title) + commonboost(mystaticrank) ", + " }", + " first-phase { expression: commonfirstphase(usstaticrank) }", + " }", + " rank-profile eurank inherits default {", + " first-phase { expression: commonfirstphase(eustaticrank) }", + " }", + "}")); + builder.build(true); + RankProfile profile = builder.getRankProfileRegistry().get(builder.getSchema(), "eurank"); + } + + @Test + public void testTensorFunctionInvocationTypes_NestedSameName() throws Exception { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( + "search test {", + " document test { ", + " field a type tensor(x[10],y[1]) {", + " indexing: attribute", + " }", + " field b type tensor(z[10]) {", + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " function return_a() {", + " expression: return_first(attribute(a), attribute(b))", + " }", + " function return_b() {", + " expression: return_second(attribute(a), attribute(b))", + " }", + " function return_first(e1, e2) {", + " expression: just_return(e1)", + " }", + " function just_return(e1) {", + " expression: e1", + " }", + " function return_second(e1, e2) {", + " expression: return_first(e2+0, e1)", + " }", + " summary-features {", + " return_a", + " return_b", + " }", + " }", + "}" + )); + builder.build(true); + RankProfile profile = + builder.getRankProfileRegistry().get(builder.getSchema(), "my_rank_profile"); + assertEquals(TensorType.fromSpec("tensor(x[10],y[1])"), + summaryFeatures(profile).get("return_a").type(profile.typeContext(builder.getQueryProfileRegistry()))); + assertEquals(TensorType.fromSpec("tensor(z[10])"), + summaryFeatures(profile).get("return_b").type(profile.typeContext(builder.getQueryProfileRegistry()))); + } + + @Test + public void testTensorFunctionInvocationTypes_viaFuncWithExpr() throws Exception { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( + "search test {", + " document test {", + " field t1 type tensor<float>(y{}) { indexing: attribute | summary }", + " field t2 type tensor<float>(x{}) { indexing: attribute | summary }", + " }", + " rank-profile test {", + " function my_func(t) { expression: sum(t, x) + 1 }", + " function test_func_via_func_with_expr() { expression: call_func_with_expr( attribute(t1), attribute(t2) ) }", + " function call_func_with_expr(a, b) { expression: my_func( a * b ) }", + " summary-features { test_func_via_func_with_expr }", + " }", + "}")); + builder.build(true); + RankProfile profile = builder.getRankProfileRegistry().get(builder.getSchema(), "test"); + assertEquals(TensorType.fromSpec("tensor<float>(y{})"), + summaryFeatures(profile).get("test_func_via_func_with_expr").type(profile.typeContext(builder.getQueryProfileRegistry()))); + } + + @Test + public void importedFieldsAreAvailable() throws Exception { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( + "search parent {", + " document parent {", + " field a type tensor(x[5],y[1000]) {", + " indexing: attribute", + " }", + " }", + "}" + )); + builder.addSchema(joinLines( + "search child {", + " document child { ", + " field ref type reference<parent> {", + "indexing: attribute | summary", + " }", + " }", + " import field ref.a as imported_a {}", + " rank-profile my_rank_profile {", + " first-phase {", + " expression: sum(attribute(imported_a))", + " }", + " }", + "}" + )); + builder.build(true); + } + + @Test + public void undeclaredQueryFeaturesAreAccepted() throws Exception { + InspectableDeployLogger logger = new InspectableDeployLogger(); + ApplicationBuilder builder = new ApplicationBuilder(logger); + builder.addSchema(joinLines( + "search test {", + " document test { ", + " field anyfield type double {" + + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " first-phase {", + " expression: query(foo) + f() + sum(attribute(anyfield))", + " }", + " function f() {", + " expression: query(bar) + query(baz)", + " }", + " }", + "}" + )); + builder.build(true); + String message = logger.findMessage("The following query features"); + assertNull(message); + } + + @Test + public void undeclaredQueryFeaturesAreNotAcceptedWhenStrict() throws Exception { + try { + InspectableDeployLogger logger = new InspectableDeployLogger(); + ApplicationBuilder builder = new ApplicationBuilder(logger); + builder.addSchema(joinLines( + "search test {", + " document test { ", + " field anyfield type double {" + + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " strict: true" + + " first-phase {", + " expression: query(foo) + f() + sum(attribute(anyfield))", + " }", + " function f() {", + " expression: query(bar) + query(baz)", + " }", + " }", + "}" + )); + builder.build(true); + } + catch (IllegalArgumentException e) { + assertEquals("In schema 'test', rank profile 'my_rank_profile': rank profile 'my_rank_profile' is strict but is missing a query profile type declaration of features [query(bar), query(baz), query(foo)]", + Exceptions.toMessageString(e)); + } + } + + @Test + public void undeclaredQueryFeaturesAreAcceptedWithWarningWhenUsingTensors() throws Exception { + InspectableDeployLogger logger = new InspectableDeployLogger(); + ApplicationBuilder builder = new ApplicationBuilder(logger); + builder.addSchema(joinLines( + "search test {", + " document test { ", + " field anyfield type tensor(d[2]) {", + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " first-phase {", + " expression: query(foo) + f() + sum(attribute(anyfield))", + " }", + " function f() {", + " expression: query(bar) + query(baz)", + " }", + " }", + "}" + )); + builder.build(true); + String message = logger.findMessage("The following query features"); + assertNotNull(message); + assertEquals("WARNING: The following query features used in rank profile 'my_rank_profile' are not declared in query profile types and " + + "will be interpreted as scalars, not tensors: [query(bar), query(baz), query(foo)]", + message); + } + + @Test + public void noWarningWhenUsingTensorsWhenQueryFeaturesAreDeclared() throws Exception { + InspectableDeployLogger logger = new InspectableDeployLogger(); + ApplicationBuilder builder = new ApplicationBuilder(logger); + QueryProfileType myType = new QueryProfileType("mytype"); + myType.addField(new FieldDescription("rank.feature.query(foo)", + new TensorFieldType(TensorType.fromSpec("tensor(d[2])"))), + builder.getQueryProfileRegistry().getTypeRegistry()); + myType.addField(new FieldDescription("rank.feature.query(bar)", + new TensorFieldType(TensorType.fromSpec("tensor(d[2])"))), + builder.getQueryProfileRegistry().getTypeRegistry()); + myType.addField(new FieldDescription("rank.feature.query(baz)", + new TensorFieldType(TensorType.fromSpec("tensor(d[2])"))), + builder.getQueryProfileRegistry().getTypeRegistry()); + builder.getQueryProfileRegistry().getTypeRegistry().register(myType); + builder.addSchema(joinLines( + "search test {", + " document test { ", + " field anyfield type tensor(d[2]) {", + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " first-phase {", + " expression: sum(query(foo) + f() + sum(attribute(anyfield)))", + " }", + " function f() {", + " expression: query(bar) + query(baz)", + " }", + " }", + "}" + )); + builder.build(true); + String message = logger.findMessage("The following query features"); + assertNull(message); + } + + private Map<String, ReferenceNode> summaryFeatures(RankProfile profile) { + return profile.getSummaryFeatures().stream().collect(Collectors.toMap(f -> f.toString(), f -> f)); + } + + private static class InspectableDeployLogger implements DeployLogger { + + private List<String> messages = new ArrayList<>(); + + @Override + public void log(Level level, String message) { + messages.add(level + ": " + message); + } + + /** Returns the first message containing the given string, or null if none */ + public String findMessage(String substring) { + return messages.stream().filter(message -> message.contains(substring)).findFirst().orElse(null); + } + + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithLightGBMTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithLightGBMTestCase.java new file mode 100644 index 00000000000..4df0a09ec2e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithLightGBMTestCase.java @@ -0,0 +1,88 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.schema.parser.ParseException; +import org.junit.After; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author lesters + */ +public class RankingExpressionWithLightGBMTestCase { + + private final Path applicationDir = Path.fromString("src/test/integration/lightgbm/"); + + private final static String lightGBMExpression = + "if (!(numerical_2 >= 0.46643291586559305), 2.1594397038037663, if (categorical_2 in [\"k\", \"l\", \"m\"], 2.235297305276056, 2.1792953471546546)) + if (categorical_1 in [\"d\", \"e\"], 0.03070842919354316, if (!(numerical_1 >= 0.5102250691730842), -0.04439151147520909, 0.005117411709368601)) + if (!(numerical_2 >= 0.668665477622446), if (!(numerical_2 >= 0.008118820676863816), -0.15361238490967524, -0.01192330846157292), 0.03499044894987518) + if (!(numerical_1 >= 0.5201391072644542), -0.02141000620783247, if (categorical_1 in [\"a\", \"b\"], -0.004121485787596721, 0.04534090904886873)) + if (categorical_2 in [\"k\", \"l\", \"m\"], if (!(numerical_2 >= 0.27283279016959255), -0.01924803254356527, 0.03643772842347651), -0.02701711918923075)"; + + @After + public void removeGeneratedModelFiles() { + IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + } + + @Test + public void testLightGBMReference() { + RankProfileSearchFixture search = fixtureWith("lightgbm('regression.json')"); + search.assertFirstPhaseExpression(lightGBMExpression, "my_profile"); + } + + @Test + public void testNestedLightGBMReference() { + RankProfileSearchFixture search = fixtureWith("5 + sum(lightgbm('regression.json'))"); + search.assertFirstPhaseExpression("5 + reduce(" + lightGBMExpression + ", sum)", "my_profile"); + } + + @Test + public void testImportingFromStoredExpressions() throws IOException { + RankProfileSearchFixture search = fixtureWith("lightgbm('regression.json')"); + search.assertFirstPhaseExpression(lightGBMExpression, "my_profile"); + + // At this point the expression is stored - copy application to another location which do not have a models dir + Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); + try { + storedApplicationDirectory.toFile().mkdirs(); + IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), + storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + RankingExpressionWithOnnxTestCase.StoringApplicationPackage storedApplication = new RankingExpressionWithOnnxTestCase.StoringApplicationPackage(storedApplicationDirectory); + RankProfileSearchFixture searchFromStored = fixtureWith("lightgbm('regression.json')"); + searchFromStored.assertFirstPhaseExpression(lightGBMExpression, "my_profile"); + } + finally { + IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); + } + } + + private RankProfileSearchFixture fixtureWith(String firstPhaseExpression) { + return fixtureWith(firstPhaseExpression, null, null, + new RankingExpressionWithOnnxTestCase.StoringApplicationPackage(applicationDir)); + } + + private RankProfileSearchFixture fixtureWith(String firstPhaseExpression, + String constant, + String field, + RankingExpressionWithOnnxTestCase.StoringApplicationPackage application) { + try { + RankProfileSearchFixture fixture = new RankProfileSearchFixture( + application, + application.getQueryProfiles(), + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: " + firstPhaseExpression + + " }\n" + + " }", + constant, + field); + fixture.compileRankProfile("my_profile", applicationDir.append("models")); + return fixture; + } catch (ParseException e) { + throw new IllegalArgumentException(e); + } + } + +} + diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxModelTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxModelTestCase.java new file mode 100644 index 00000000000..713e11fd608 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxModelTestCase.java @@ -0,0 +1,184 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.search.DocumentDatabase; +import com.yahoo.vespa.model.search.IndexedSearchCluster; +import org.junit.After; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class RankingExpressionWithOnnxModelTestCase { + + private final Path applicationDir = Path.fromString("src/test/integration/onnx-model/"); + + @After + public void removeGeneratedModelFiles() { + IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + } + + @Test + public void testOnnxModelFeature() throws Exception { + VespaModel model = loadModel(applicationDir); + assertTransformedFeature(model); + assertGeneratedConfig(model); + + Path storedApplicationDir = applicationDir.append("copy"); + try { + storedApplicationDir.toFile().mkdirs(); + IOUtils.copy(applicationDir.append("services.xml").toString(), storedApplicationDir.append("services.xml").toString()); + IOUtils.copyDirectory(applicationDir.append("schemas").toFile(), storedApplicationDir.append("schemas").toFile()); + IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), + storedApplicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + + VespaModel storedModel = loadModel(storedApplicationDir); + assertTransformedFeature(storedModel); + assertGeneratedConfig(storedModel); + } + finally { + IOUtils.recursiveDeleteDir(storedApplicationDir.toFile()); + } + } + + private VespaModel loadModel(Path path) throws Exception { + FilesApplicationPackage applicationPackage = FilesApplicationPackage.fromFile(path.toFile()); + DeployState state = new DeployState.Builder().applicationPackage(applicationPackage).build(); + return new VespaModel(state); + } + + private void assertGeneratedConfig(VespaModel vespaModel) { + DocumentDatabase db = ((IndexedSearchCluster)vespaModel.getSearchClusters().get(0)).getDocumentDbs().get(0); + + RankingConstantsConfig.Builder rankingConstantsConfigBuilder = new RankingConstantsConfig.Builder(); + db.getConfig(rankingConstantsConfigBuilder); + var rankingConstantsConfig = rankingConstantsConfigBuilder.build(); + assertEquals(1, rankingConstantsConfig.constant().size()); + assertEquals("my_constant", rankingConstantsConfig.constant(0).name()); + assertEquals("tensor(d0[2])", rankingConstantsConfig.constant(0).type()); + assertEquals("files/constant.json", rankingConstantsConfig.constant(0).fileref().value()); + + OnnxModelsConfig.Builder builder = new OnnxModelsConfig.Builder(); + ((OnnxModelsConfig.Producer) db).getConfig(builder); + OnnxModelsConfig config = new OnnxModelsConfig(builder); + assertEquals(6, config.model().size()); + for (OnnxModelsConfig.Model model : config.model()) { + assertTrue(model.dry_run_on_setup()); + } + + OnnxModelsConfig.Model model = config.model(0); + assertEquals("my_model", model.name()); + assertEquals(3, model.input().size()); + assertEquals("second/input:0", model.input(0).name()); + assertEquals("constant(my_constant)", model.input(0).source()); + assertEquals("first_input", model.input(1).name()); + assertEquals("attribute(document_field)", model.input(1).source()); + assertEquals("third_input", model.input(2).name()); + assertEquals("rankingExpression(my_function)", model.input(2).source()); + assertEquals(3, model.output().size()); + assertEquals("path/to/output:0", model.output(0).name()); + assertEquals("out", model.output(0).as()); + assertEquals("path/to/output:1", model.output(1).name()); + assertEquals("path_to_output_1", model.output(1).as()); + assertEquals("path/to/output:2", model.output(2).name()); + assertEquals("path_to_output_2", model.output(2).as()); + + model = config.model(1); + assertEquals("dynamic_model", model.name()); + assertEquals(1, model.input().size()); + assertEquals(1, model.output().size()); + assertEquals("rankingExpression(my_function)", model.input(0).source()); + + model = config.model(2); + assertEquals("unbound_model", model.name()); + assertEquals(1, model.input().size()); + assertEquals(1, model.output().size()); + assertEquals("rankingExpression(my_function)", model.input(0).source()); + + model = config.model(3); + assertEquals("files_model_onnx", model.name()); + assertEquals(3, model.input().size()); + assertEquals(3, model.output().size()); + assertEquals("path/to/output:0", model.output(0).name()); + assertEquals("path_to_output_0", model.output(0).as()); + assertEquals("path/to/output:1", model.output(1).name()); + assertEquals("path_to_output_1", model.output(1).as()); + assertEquals("path/to/output:2", model.output(2).name()); + assertEquals("path_to_output_2", model.output(2).as()); + assertEquals("files_model_onnx", model.name()); + + model = config.model(4); + assertEquals("another_model", model.name()); + assertEquals("third_input", model.input(2).name()); + assertEquals("rankingExpression(another_function)", model.input(2).source()); + + model = config.model(5); + assertEquals("files_summary_model_onnx", model.name()); + assertEquals(3, model.input().size()); + assertEquals(3, model.output().size()); + } + + private void assertTransformedFeature(VespaModel model) { + DocumentDatabase db = ((IndexedSearchCluster)model.getSearchClusters().get(0)).getDocumentDbs().get(0); + RankProfilesConfig.Builder builder = new RankProfilesConfig.Builder(); + ((RankProfilesConfig.Producer) db).getConfig(builder); + RankProfilesConfig config = new RankProfilesConfig(builder); + assertEquals(10, config.rankprofile().size()); + + assertEquals("test_model_config", config.rankprofile(2).name()); + assertEquals("rankingExpression(my_function).rankingScript", config.rankprofile(2).fef().property(0).name()); + assertEquals("vespa.rank.firstphase", config.rankprofile(2).fef().property(2).name()); + assertEquals("rankingExpression(firstphase)", config.rankprofile(2).fef().property(2).value()); + assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(2).fef().property(3).name()); + assertEquals("onnxModel(my_model).out{d0:1}", config.rankprofile(2).fef().property(3).value()); + + assertEquals("test_generated_model_config", config.rankprofile(3).name()); + assertEquals("rankingExpression(my_function).rankingScript", config.rankprofile(3).fef().property(0).name()); + assertEquals("rankingExpression(first_input).rankingScript", config.rankprofile(3).fef().property(2).name()); + assertEquals("rankingExpression(second_input).rankingScript", config.rankprofile(3).fef().property(4).name()); + assertEquals("rankingExpression(third_input).rankingScript", config.rankprofile(3).fef().property(6).name()); + assertEquals("vespa.rank.firstphase", config.rankprofile(3).fef().property(8).name()); + assertEquals("rankingExpression(firstphase)", config.rankprofile(3).fef().property(8).value()); + assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(3).fef().property(9).name()); + assertEquals("onnxModel(files_model_onnx).path_to_output_1{d0:1}", config.rankprofile(3).fef().property(9).value()); + + assertEquals("test_summary_features", config.rankprofile(4).name()); + assertEquals("rankingExpression(another_function).rankingScript", config.rankprofile(4).fef().property(0).name()); + assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(4).fef().property(3).name()); + assertEquals("1", config.rankprofile(4).fef().property(3).value()); + assertEquals("vespa.summary.feature", config.rankprofile(4).fef().property(4).name()); + assertEquals("onnxModel(files_summary_model_onnx).path_to_output_2", config.rankprofile(4).fef().property(4).value()); + assertEquals("vespa.summary.feature", config.rankprofile(4).fef().property(5).name()); + assertEquals("onnxModel(another_model).out", config.rankprofile(4).fef().property(5).value()); + + assertEquals("test_dynamic_model", config.rankprofile(5).name()); + assertEquals("rankingExpression(my_function).rankingScript", config.rankprofile(5).fef().property(0).name()); + assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(5).fef().property(3).name()); + assertEquals("onnxModel(dynamic_model).my_output{d0:0, d1:1}", config.rankprofile(5).fef().property(3).value()); + + assertEquals("test_dynamic_model_2", config.rankprofile(6).name()); + assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(6).fef().property(5).name()); + assertEquals("onnxModel(dynamic_model).my_output{d0:0, d1:2}", config.rankprofile(6).fef().property(5).value()); + + assertEquals("test_dynamic_model_with_transformer_tokens", config.rankprofile(7).name()); + assertEquals("rankingExpression(my_function).rankingScript", config.rankprofile(7).fef().property(1).name()); + assertEquals("tensor<float>(d0[1],d1[10])((if (d1 < 1.0 + rankingExpression(__token_length@-1993461420) + 1.0, 0.0, if (d1 < 1.0 + rankingExpression(__token_length@-1993461420) + 1.0 + rankingExpression(__token_length@-1993461420) + 1.0, 1.0, 0.0))))", config.rankprofile(7).fef().property(1).value()); + + assertEquals("test_unbound_model", config.rankprofile(8).name()); + assertEquals("rankingExpression(my_function).rankingScript", config.rankprofile(8).fef().property(0).name()); + assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(8).fef().property(3).name()); + assertEquals("onnxModel(unbound_model).my_output{d0:0, d1:1}", config.rankprofile(8).fef().property(3).value()); + + + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxTestCase.java new file mode 100644 index 00000000000..94a51d25717 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxTestCase.java @@ -0,0 +1,417 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.io.IOUtils; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.path.Path; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.FeatureNames; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.tensor.TensorType; +import com.yahoo.yolean.Exceptions; +import org.junit.After; +import org.junit.Test; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; + +public class RankingExpressionWithOnnxTestCase { + + private final Path applicationDir = Path.fromString("src/test/integration/onnx/"); + + private final static String name = "mnist_softmax"; + private final static String vespaExpression = "join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), constant(mnist_softmax_layer_Variable), f(a,b)(a * b)), sum, d2) * 1.0, constant(mnist_softmax_layer_Variable_1) * 1.0, f(a,b)(a + b))"; + + @After + public void removeGeneratedModelFiles() { + IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + } + + @Test + public void testOnnxReferenceWithConstantFeature() { + RankProfileSearchFixture search = fixtureWith("constant(mytensor)", + "onnx_vespa('mnist_softmax.onnx')", + "constant mytensor { file: ignored\ntype: tensor<float>(d0[1],d1[784]) }", + null); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + } + + @Test + public void testOnnxReferenceWithQueryFeature() { + String queryProfile = "<query-profile id='default' type='root'/>"; + String queryProfileType = + "<query-profile-type id='root'>" + + " <field name='query(mytensor)' type='tensor<float>(d0[1],d1[784])'/>" + + "</query-profile-type>"; + StoringApplicationPackage application = new StoringApplicationPackage(applicationDir, + queryProfile, + queryProfileType); + RankProfileSearchFixture search = fixtureWith("query(mytensor)", + "onnx_vespa('mnist_softmax.onnx')", + null, + null, + "Placeholder", + application); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + } + + @Test + public void testOnnxReferenceWithDocumentFeature() { + StoringApplicationPackage application = new StoringApplicationPackage(applicationDir); + RankProfileSearchFixture search = fixtureWith("attribute(mytensor)", + "onnx_vespa('mnist_softmax.onnx')", + null, + "field mytensor type tensor<float>(d0[1],d1[784]) { indexing: attribute }", + "Placeholder", + application); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + } + + + @Test + public void testOnnxReferenceWithFeatureCombination() { + String queryProfile = "<query-profile id='default' type='root'/>"; + String queryProfileType = + "<query-profile-type id='root'>" + + " <field name='query(mytensor)' type='tensor<float>(d0[1],d1[784],d2[10])'/>" + + "</query-profile-type>"; + StoringApplicationPackage application = new StoringApplicationPackage(applicationDir, queryProfile, queryProfileType); + RankProfileSearchFixture search = fixtureWith("sum(query(mytensor) * attribute(mytensor) * constant(mytensor),d2)", + "onnx_vespa('mnist_softmax.onnx')", + "constant mytensor { file: ignored\ntype: tensor<float>(d0[1],d1[784]) }", + "field mytensor type tensor<float>(d0[1],d1[784]) { indexing: attribute }", + "Placeholder", + application); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + } + + + @Test + public void testNestedOnnxReference() { + RankProfileSearchFixture search = fixtureWith("tensor<float>(d0[1],d1[784])(0.0)", + "5 + sum(onnx_vespa('mnist_softmax.onnx'))"); + search.assertFirstPhaseExpression("5 + reduce(" + vespaExpression + ", sum)", "my_profile"); + } + + @Test + public void testOnnxReferenceWithSpecifiedOutput() { + RankProfileSearchFixture search = fixtureWith("tensor<float>(d0[1],d1[784])(0.0)", + "onnx_vespa('mnist_softmax.onnx', 'layer_add')"); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + } + + @Test + public void testOnnxReferenceWithSpecifiedOutputAndSignature() { + RankProfileSearchFixture search = fixtureWith("tensor<float>(d0[1],d1[784])(0.0)", + "onnx_vespa('mnist_softmax.onnx', 'default.layer_add')"); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + } + + @Test + public void testOnnxReferenceMissingFunction() throws ParseException { + try { + RankProfileSearchFixture search = new RankProfileSearchFixture( + new StoringApplicationPackage(applicationDir), + new QueryProfileRegistry(), + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: onnx_vespa('mnist_softmax.onnx')" + + " }\n" + + " }"); + search.compileRankProfile("my_profile", applicationDir.append("models")); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + fail("Expecting exception"); + } + catch (IllegalArgumentException expected) { + assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " + + "onnx_vespa(\"mnist_softmax.onnx\"): " + + "Model refers input 'Placeholder' of type tensor<float>(d0[1],d1[784]) but this function is " + + "not present in rank profile 'my_profile'", + Exceptions.toMessageString(expected)); + } + } + + @Test + public void testOnnxReferenceWithWrongFunctionType() { + try { + RankProfileSearchFixture search = fixtureWith("tensor(d0[1],d5[10])(0.0)", + "onnx_vespa('mnist_softmax.onnx')"); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + fail("Expecting exception"); + } + catch (IllegalArgumentException expected) { + assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " + + "onnx_vespa(\"mnist_softmax.onnx\"): " + + "Model refers input 'Placeholder'. The required type of this is tensor<float>(d0[1],d1[784]), " + + "but this function returns tensor(d0[1],d5[10])", + Exceptions.toMessageString(expected)); + } + } + + @Test + public void testOnnxReferenceSpecifyingNonExistingOutput() { + try { + RankProfileSearchFixture search = fixtureWith("tensor<float>(d0[2],d1[784])(0.0)", + "onnx_vespa('mnist_softmax.onnx', 'y')"); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + fail("Expecting exception"); + } + catch (IllegalArgumentException expected) { + assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " + + "onnx_vespa(\"mnist_softmax.onnx\",\"y\"): " + + "No expressions named 'y' in model 'mnist_softmax.onnx'. Available expressions: default.layer_add", + Exceptions.toMessageString(expected)); + } + } + + @Test + public void testImportingFromStoredExpressions() throws IOException { + RankProfileSearchFixture search = fixtureWith("tensor<float>(d0[1],d1[784])(0.0)", + "onnx_vespa(\"mnist_softmax.onnx\")"); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + + // At this point the expression is stored - copy application to another location which do not have a models dir + Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); + try { + storedApplicationDirectory.toFile().mkdirs(); + IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), + storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + StoringApplicationPackage storedApplication = new StoringApplicationPackage(storedApplicationDirectory); + RankProfileSearchFixture searchFromStored = fixtureWith("tensor<float>(d0[2],d1[784])(0.0)", + "onnx_vespa('mnist_softmax.onnx')", + null, + null, + "Placeholder", + storedApplication); + searchFromStored.assertFirstPhaseExpression(vespaExpression, "my_profile"); + // Verify that the constants exists, but don't verify the content as we are not + // simulating file distribution in this test + } + finally { + IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); + } + } + + @Test + public void testImportingFromStoredExpressionsWithFunctionOverridingConstantAndInheritance() throws IOException { + String rankProfile = + " rank-profile my_profile {\n" + + " function Placeholder() {\n" + + " expression: tensor<float>(d0[1],d1[784])(0.0)\n" + + " }\n" + + " function " + name + "_layer_Variable() {\n" + + " expression: tensor<float>(d1[10],d2[784])(0.0)\n" + + " }\n" + + " first-phase {\n" + + " expression: onnx_vespa('mnist_softmax.onnx')" + + " }\n" + + " }" + + " rank-profile my_profile_child inherits my_profile {\n" + + " }"; + + String vespaExpressionWithoutConstant = + "join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), " + name + "_layer_Variable, f(a,b)(a * b)), sum, d2) * 1.0, constant(" + name + "_layer_Variable_1) * 1.0, f(a,b)(a + b))"; + RankProfileSearchFixture search = uncompiledFixtureWith(rankProfile, new StoringApplicationPackage(applicationDir)); + search.compileRankProfile("my_profile", applicationDir.append("models")); + search.compileRankProfile("my_profile_child", applicationDir.append("models")); + search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); + search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile_child"); + + assertNull("Constant overridden by function is not added", + search.search().constants().get(name + "_Variable")); + + // At this point the expression is stored - copy application to another location which do not have a models dir + Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); + try { + storedApplicationDirectory.toFile().mkdirs(); + IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), + storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + StoringApplicationPackage storedApplication = new StoringApplicationPackage(storedApplicationDirectory); + RankProfileSearchFixture searchFromStored = uncompiledFixtureWith(rankProfile, storedApplication); + searchFromStored.compileRankProfile("my_profile", applicationDir.append("models")); + searchFromStored.compileRankProfile("my_profile_child", applicationDir.append("models")); + searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); + searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile_child"); + assertNull("Constant overridden by function is not added", + searchFromStored.search().constants().get(name + "_Variable")); + } finally { + IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); + } + } + + @Test + public void testFunctionGeneration() { + final String name = "small_constants_and_functions"; + final String rankProfiles = + " rank-profile my_profile {\n" + + " function input() {\n" + + " expression: tensor<float>(d0[3])(0.0)\n" + + " }\n" + + " first-phase {\n" + + " expression: onnx_vespa('" + name + ".onnx')" + + " }\n" + + " }"; + final String functionName = "imported_ml_function_" + name + "_exp_output"; + final String expression = "join(" + functionName + ", reduce(join(join(reduce(" + functionName + ", sum, d0), tensor<float>(d0[1])(1.0), f(a,b)(a * b)), constant(" + name + "_epsilon), f(a,b)(a + b)), sum, d0), f(a,b)(a / b))"; + final String functionExpression = "map(input, f(a)(exp(a)))"; + + RankProfileSearchFixture search = uncompiledFixtureWith(rankProfiles, new StoringApplicationPackage(applicationDir)); + search.compileRankProfile("my_profile", applicationDir.append("models")); + search.assertFirstPhaseExpression(expression, "my_profile"); + search.assertFunction(functionExpression, functionName, "my_profile"); + } + + @Test + public void testImportingFromStoredExpressionsWithSmallConstantsAndInheritance() throws IOException { + final String name = "small_constants_and_functions"; + final String rankProfiles = + " rank-profile my_profile {\n" + + " function input() {\n" + + " expression: tensor<float>(d0[3])(0.0)\n" + + " }\n" + + " first-phase {\n" + + " expression: onnx_vespa('" + name + ".onnx')" + + " }\n" + + " }" + + " rank-profile my_profile_child inherits my_profile {\n" + + " }"; + final String functionName = "imported_ml_function_" + name + "_exp_output"; + final String expression = "join(" + functionName + ", reduce(join(join(reduce(" + functionName + ", sum, d0), tensor<float>(d0[1])(1.0), f(a,b)(a * b)), constant(" + name + "_epsilon), f(a,b)(a + b)), sum, d0), f(a,b)(a / b))"; + final String functionExpression = "map(input, f(a)(exp(a)))"; + + RankProfileSearchFixture search = uncompiledFixtureWith(rankProfiles, new StoringApplicationPackage(applicationDir)); + search.compileRankProfile("my_profile", applicationDir.append("models")); + search.compileRankProfile("my_profile_child", applicationDir.append("models")); + search.assertFirstPhaseExpression(expression, "my_profile"); + search.assertFirstPhaseExpression(expression, "my_profile_child"); + assertSmallConstant(name + "_epsilon", TensorType.fromSpec("tensor()"), search); + search.assertFunction(functionExpression, functionName, "my_profile"); + search.assertFunction(functionExpression, functionName, "my_profile_child"); + + // At this point the expression is stored - copy application to another location which do not have a models dir + Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); + try { + storedApplicationDirectory.toFile().mkdirs(); + IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), + storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + StoringApplicationPackage storedApplication = new StoringApplicationPackage(storedApplicationDirectory); + RankProfileSearchFixture searchFromStored = uncompiledFixtureWith(rankProfiles, storedApplication); + searchFromStored.compileRankProfile("my_profile", applicationDir.append("models")); + searchFromStored.compileRankProfile("my_profile_child", applicationDir.append("models")); + searchFromStored.assertFirstPhaseExpression(expression, "my_profile"); + searchFromStored.assertFirstPhaseExpression(expression, "my_profile_child"); + assertSmallConstant(name + "_epsilon", TensorType.fromSpec("tensor()"), search); + searchFromStored.assertFunction(functionExpression, functionName, "my_profile"); + searchFromStored.assertFunction(functionExpression, functionName, "my_profile_child"); + } + finally { + IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); + } + } + + private void assertSmallConstant(String name, TensorType type, RankProfileSearchFixture search) { + var value = search.compiledRankProfile("my_profile").constants().get(FeatureNames.asConstantFeature(name)); + assertNotNull(value); + assertEquals(type, value.type()); + } + + private RankProfileSearchFixture fixtureWith(String placeholderExpression, String firstPhaseExpression) { + return fixtureWith(placeholderExpression, firstPhaseExpression, null, null, "Placeholder", + new StoringApplicationPackage(applicationDir)); + } + + private RankProfileSearchFixture fixtureWith(String placeholderExpression, String firstPhaseExpression, + String constant, String field) { + return fixtureWith(placeholderExpression, firstPhaseExpression, constant, field, "Placeholder", + new StoringApplicationPackage(applicationDir)); + } + + private RankProfileSearchFixture uncompiledFixtureWith(String rankProfile, StoringApplicationPackage application) { + try { + return new RankProfileSearchFixture(application, application.getQueryProfiles(), + rankProfile, null, null); + } + catch (ParseException e) { + throw new IllegalArgumentException(e); + } + } + + private RankProfileSearchFixture fixtureWith(String functionExpression, + String firstPhaseExpression, + String constant, + String field, + String functionName, + StoringApplicationPackage application) { + try { + RankProfileSearchFixture fixture = new RankProfileSearchFixture( + application, + application.getQueryProfiles(), + " rank-profile my_profile {\n" + + " function " + functionName + "() {\n" + + " expression: " + functionExpression + + " }\n" + + " first-phase {\n" + + " expression: " + firstPhaseExpression + + " }\n" + + " }", + constant, + field); + fixture.compileRankProfile("my_profile", applicationDir.append("models")); + return fixture; + } + catch (ParseException e) { + throw new IllegalArgumentException(e); + } + } + + static class StoringApplicationPackage extends MockApplicationPackage { + + StoringApplicationPackage(Path applicationPackageWritableRoot) { + this(applicationPackageWritableRoot, null, null); + } + + StoringApplicationPackage(Path applicationPackageWritableRoot, String queryProfile, String queryProfileType) { + super(new File(applicationPackageWritableRoot.toString()), + null, null, List.of(), Map.of(), null, + null, null, false, queryProfile, queryProfileType); + } + + @Override + public ApplicationFile getFile(Path file) { + return new MockApplicationFile(file, Path.fromString(root().toString())); + } + + @Override + public List<NamedReader> getFiles(Path path, String suffix) { + File[] files = getFileReference(path).listFiles(); + if (files == null) return List.of(); + List<NamedReader> readers = new ArrayList<>(); + for (File file : files) { + if ( ! file.getName().endsWith(suffix)) continue; + try { + readers.add(new NamedReader(file.getName(), new FileReader(file))); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + return readers; + } + + } + + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithTensorTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithTensorTestCase.java new file mode 100644 index 00000000000..1f065bc7a20 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithTensorTestCase.java @@ -0,0 +1,202 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author geirst + */ +public class RankingExpressionWithTensorTestCase { + + @Test + public void requireThatSingleLineConstantMappedTensorCanBeParsed() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: sum(my_tensor)\n" + + " }\n" + + " constants {\n" + + " my_tensor tensor(x{},y{}):{ {x:1,y:2}:1, {x:2,y:1}:2 }\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); + f.assertRankProperty("tensor(x{},y{}):{{x:1,y:2}:1.0, {x:2,y:1}:2.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{},y{})", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireThatSingleLineConstantIndexedTensorCanBeParsed() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: sum(my_tensor)\n" + + " }\n" + + " constants {\n" + + " my_tensor tensor(x[3]):{ {x:0}:1, {x:1}:2, {x:2}:3 }\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); + f.assertRankProperty("tensor(x[3]):[1.0, 2.0, 3.0]", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x[3])", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireThatSingleLineConstantIndexedTensorShortFormCanBeParsed() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: sum(my_tensor)\n" + + " }\n" + + " constants {\n" + + " my_tensor tensor(x[3]):[1, 2, 3]\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); + f.assertRankProperty("tensor(x[3]):[1.0, 2.0, 3.0]", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x[3])", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireConstantTensorCanBeReferredViaConstantFeature() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: sum(constant(my_tensor))\n" + + " }\n" + + " constants {\n" + + " my_tensor tensor(x{},y{}):{{x:1,y:2}:1, {x:2,y:1}:2}\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); + f.assertRankProperty("tensor(x{},y{}):{{x:1,y:2}:1.0, {x:2,y:1}:2.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{},y{})", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireThatMultiLineConstantTensorAndTypeCanBeParsed() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: sum(my_tensor)\n" + + " }\n" + + " constants {\n" + + " my_tensor tensor(x{},y{}):\n" + + " { {x:1,y:2}:1,\n" + + " {x:2,y:1}:2 }\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); + f.assertRankProperty("tensor(x{},y{}):{{x:1,y:2}:1.0, {x:2,y:1}:2.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{},y{})", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireThatConstantTensorsCanBeUsedInSecondPhaseExpression() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " second-phase {\n" + + " expression: sum(my_tensor)\n" + + " }\n" + + " constants {\n" + + " my_tensor tensor(x{}):{ {x:1}:1 }\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertSecondPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); + f.assertRankProperty("tensor(x{}):{1:1.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireThatConstantTensorsCanBeUsedInInheritedRankProfile() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile parent {\n" + + " constants {\n" + + " my_tensor {\n" + + " value: { {x:1}:1 }\n" + + " }\n" + + " }\n" + + " }\n" + + " rank-profile my_profile inherits parent {\n" + + " first-phase {\n" + + " expression: sum(my_tensor)\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); + f.assertRankProperty("tensor(x{}):{1:1.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireThatConstantTensorsCanBeUsedInFunction() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " function my_macro() {\n" + + " expression: sum(my_tensor)\n" + + " }\n" + + " first-phase {\n" + + " expression: 5.0 + my_macro\n" + + " }\n" + + " constants {\n" + + " my_tensor tensor(x{}):{ {x:1}:1 }\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertFirstPhaseExpression("5.0 + my_macro", "my_profile"); + f.assertFunction("reduce(constant(my_tensor), sum)", "my_macro", "my_profile"); + f.assertRankProperty("tensor(x{}):{1:1.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireThatCombinationOfConstantTensorsAndConstantValuesCanBeUsed() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: my_number_1 + sum(my_tensor) + my_number_2\n" + + " }\n" + + " constants {\n" + + " my_number_1 double: 3.0\n" + + " my_tensor tensor(x{}):{ {x:1}:1 }\n" + + " my_number_2 double: 5.0\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertFirstPhaseExpression("3.0 + reduce(constant(my_tensor), sum) + 5.0", "my_profile"); + f.assertRankProperty("tensor(x{}):{1:1.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireThatInvalidTensorTypeSpecThrowsException() throws ParseException { + try { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " constants {\n" + + " my_tensor tensor(x):{ {x:1}:1 }\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertStartsWith("Type of constant(my_tensor): Illegal tensor type spec: A tensor type spec must be on the form", + e.getMessage()); + } + } + + private void assertStartsWith(String prefix, String string) { + assertEquals(prefix, string.substring(0, Math.min(prefix.length(), string.length()))); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithTransformerTokensTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithTransformerTokensTestCase.java new file mode 100644 index 00000000000..f8086fb3bc6 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithTransformerTokensTestCase.java @@ -0,0 +1,98 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.expressiontransforms.RankProfileTransformContext; +import com.yahoo.schema.expressiontransforms.TokenTransformer; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.evaluation.MapContext; +import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.tensor.Tensor; +import org.junit.Test; + +import java.util.Collections; + +import static org.junit.Assert.assertEquals; + +public class RankingExpressionWithTransformerTokensTestCase { + + @Test + public void testTokenInputIds() throws Exception { + String expected = "tensor(d0[1],d1[12]):[101,1,2,102,3,4,5,102,6,7,102,0]"; + String a = "tensor(d0[2]):[1,2]"; + String b = "tensor(d0[3]):[3,4,5]"; + String c = "tensor(d0[2]):[6,7]"; + String expression = "tokenInputIds(12, a, b, c)"; + Tensor result = evaluateExpression(expression, a, b, c); + assertEquals(Tensor.from(expected), result); + } + + @Test + public void testTokenTypeIds() throws Exception { + String expected = "tensor(d0[1],d1[10]):[0,0,0,0,1,1,1,1,0,0]"; + String a = "tensor(d0[2]):[1,2]"; + String b = "tensor(d0[3]):[3,4,5]"; + String expression = "tokenTypeIds(10, a, b)"; + Tensor result = evaluateExpression(expression, a, b); + assertEquals(Tensor.from(expected), result); + } + + @Test + public void testAttentionMask() throws Exception { + String expected = "tensor(d0[1],d1[10]):[1,1,1,1,1,1,1,1,0,0]"; + String a = "tensor(d0[2]):[1,2]"; + String b = "tensor(d0[3]):[3,4,5]"; + String expression = "tokenAttentionMask(10, a, b)"; + Tensor result = evaluateExpression(expression, a, b); + assertEquals(Tensor.from(expected), result); + } + + private Tensor evaluateExpression(String expression, String a, String b) throws Exception { + return evaluateExpression(expression, a, b, null, null); + } + + private Tensor evaluateExpression(String expression, String a, String b, String c) throws Exception { + return evaluateExpression(expression, a, b, c, null); + } + + private Tensor evaluateExpression(String expression, String a, String b, String c, String d) throws Exception { + MapContext context = new MapContext(); + if (a != null) context.put("a", new TensorValue(Tensor.from(a))); + if (b != null) context.put("b", new TensorValue(Tensor.from(b))); + if (c != null) context.put("c", new TensorValue(Tensor.from(c))); + if (d != null) context.put("d", new TensorValue(Tensor.from(d))); + var transformContext = createTransformContext(); + var rankingExpression = new RankingExpression(expression); + var transformed = new TokenTransformer().transform(rankingExpression, transformContext); + for (var entry : transformContext.rankProfile().getFunctions().entrySet()) { + context.put(entry.getKey(), entry.getValue().function().getBody().evaluate(context).asDouble()); + } + return transformed.evaluate(context).asTensor(); + } + + private RankProfileTransformContext createTransformContext() throws ParseException { + MockApplicationPackage application = (MockApplicationPackage) MockApplicationPackage.createEmpty(); + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + QueryProfileRegistry queryProfileRegistry = application.getQueryProfiles(); + String sdContent = "search test {\n" + + " document test {}\n" + + " rank-profile my_profile inherits default {}\n" + + "}"; + ApplicationBuilder schemaBuilder = new ApplicationBuilder(application, new MockFileRegistry(), new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfileRegistry); + schemaBuilder.addSchema(sdContent); + schemaBuilder.build(true); + Schema schema = schemaBuilder.getSchema(); + RankProfile rp = rankProfileRegistry.get(schema, "my_profile"); + return new RankProfileTransformContext(rp, queryProfileRegistry, Collections.emptyMap(), null, Collections.emptyMap(), Collections.emptyMap()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithXGBoostTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithXGBoostTestCase.java new file mode 100644 index 00000000000..e1b1473a59a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithXGBoostTestCase.java @@ -0,0 +1,90 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.schema.parser.ParseException; +import org.junit.After; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author grace-lam + * @author bratseth + */ +public class RankingExpressionWithXGBoostTestCase { + + private final Path applicationDir = Path.fromString("src/test/integration/xgboost/"); + + private final static String vespaExpression = + "if (f29 < -0.1234567, if (!(f56 >= -0.242398), 1.71218, -1.70044), if (f109 < 0.8723473, -1.94071, 1.85965)) + " + + "if (!(f60 >= -0.482947), if (f29 < -4.2387498, 0.784718, -0.96853), -6.23624)"; + + @After + public void removeGeneratedModelFiles() { + IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + } + + @Test + public void testXGBoostReference() { + RankProfileSearchFixture search = fixtureWith("xgboost('xgboost.2.2.json')"); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + } + + @Test + public void testNestedXGBoostReference() { + RankProfileSearchFixture search = fixtureWith("5 + sum(xgboost('xgboost.2.2.json'))"); + search.assertFirstPhaseExpression("5 + reduce(" + vespaExpression + ", sum)", "my_profile"); + } + + @Test + public void testImportingFromStoredExpressions() throws IOException { + RankProfileSearchFixture search = fixtureWith("xgboost('xgboost.2.2.json')"); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + + // At this point the expression is stored - copy application to another location which do not have a models dir + Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); + try { + storedApplicationDirectory.toFile().mkdirs(); + IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), + storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + RankingExpressionWithOnnxTestCase.StoringApplicationPackage storedApplication = new RankingExpressionWithOnnxTestCase.StoringApplicationPackage(storedApplicationDirectory); + RankProfileSearchFixture searchFromStored = fixtureWith("xgboost('xgboost.2.2.json')"); + searchFromStored.assertFirstPhaseExpression(vespaExpression, "my_profile"); + } + finally { + IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); + } + } + + private RankProfileSearchFixture fixtureWith(String firstPhaseExpression) { + return fixtureWith(firstPhaseExpression, null, null, + new RankingExpressionWithOnnxTestCase.StoringApplicationPackage(applicationDir)); + } + + private RankProfileSearchFixture fixtureWith(String firstPhaseExpression, + String constant, + String field, + RankingExpressionWithOnnxTestCase.StoringApplicationPackage application) { + try { + RankProfileSearchFixture fixture = new RankProfileSearchFixture( + application, + application.getQueryProfiles(), + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: " + firstPhaseExpression + + " }\n" + + " }", + constant, + field); + fixture.compileRankProfile("my_profile", applicationDir.append("models")); + return fixture; + } catch (ParseException e) { + throw new IllegalArgumentException(e); + } + } + +} + diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionsTestCase.java new file mode 100644 index 00000000000..ace3788e49a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionsTestCase.java @@ -0,0 +1,128 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.collections.Pair; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.LargeRankExpressions; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.RawRankProfile; +import com.yahoo.schema.derived.TestableDeployLogger; +import com.yahoo.schema.parser.ParseException; +import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class RankingExpressionsTestCase extends AbstractSchemaTestCase { + + private static Schema createSearch(String dir, ModelContext.Properties deployProperties, RankProfileRegistry rankProfileRegistry) throws IOException, ParseException { + return ApplicationBuilder.createFromDirectory(dir, new MockFileRegistry(), new TestableDeployLogger(), deployProperties, rankProfileRegistry).getSchema(); + } + + @Test + public void testFunctions() throws IOException, ParseException { + ModelContext.Properties deployProperties = new TestProperties(); + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + Schema schema = createSearch("src/test/examples/rankingexpressionfunction", deployProperties, rankProfileRegistry); + RankProfile functionsRankProfile = rankProfileRegistry.get(schema, "macros"); + Map<String, RankProfile.RankingExpressionFunction> functions = functionsRankProfile.getFunctions(); + assertEquals(2, functions.get("titlematch$").function().arguments().size()); + assertEquals("var1", functions.get("titlematch$").function().arguments().get(0)); + assertEquals("var2", functions.get("titlematch$").function().arguments().get(1)); + assertEquals("var1 * var2 + 890", functions.get("titlematch$").function().getBody().getRoot().toString()); + assertEquals("0.8 + 0.2 * titlematch$(4,5) + 0.8 * titlematch$(7,8) * closeness(distance)", + functionsRankProfile.getFirstPhaseRanking().getRoot().toString()); + assertEquals("78 + closeness(distance)", + functions.get("artistmatch").function().getBody().getRoot().toString()); + assertEquals(0, functions.get("artistmatch").function().arguments().size()); + + RawRankProfile rawRankProfile = new RawRankProfile(functionsRankProfile, new LargeRankExpressions(new MockFileRegistry()), new QueryProfileRegistry(), + new ImportedMlModels(), new AttributeFields(schema), deployProperties); + List<Pair<String, String>> rankProperties = rawRankProfile.configProperties(); + assertEquals(6, rankProperties.size()); + + assertEquals("rankingExpression(titlematch$).rankingScript", rankProperties.get(2).getFirst()); + assertEquals("var1 * var2 + 890", rankProperties.get(2).getSecond()); + + assertEquals("rankingExpression(artistmatch).rankingScript", rankProperties.get(3).getFirst()); + assertEquals("78 + closeness(distance)", rankProperties.get(3).getSecond()); + + assertEquals("rankingExpression(firstphase).rankingScript", rankProperties.get(5).getFirst()); + assertEquals("0.8 + 0.2 * rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c) + 0.8 * rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6) * closeness(distance)", rankProperties.get(5).getSecond()); + + assertEquals("rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6).rankingScript", rankProperties.get(1).getFirst()); + assertEquals("7 * 8 + 890", rankProperties.get(1).getSecond()); + + assertEquals("rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c).rankingScript", rankProperties.get(0).getFirst()); + assertEquals("4 * 5 + 890", rankProperties.get(0).getSecond()); + } + + @Test(expected = IllegalArgumentException.class) + public void testThatIncludingFileInSubdirFails() throws IOException, ParseException { + RankProfileRegistry registry = new RankProfileRegistry(); + Schema schema = createSearch("src/test/examples/rankingexpressioninfile", new TestProperties(), registry); + new DerivedConfiguration(schema, registry); // rank profile parsing happens during deriving + } + + private void verifyProfile(RankProfile profile, List<String> expectedFunctions, List<Pair<String, String>> rankProperties, + LargeRankExpressions largeExpressions, QueryProfileRegistry queryProfiles, ImportedMlModels models, + AttributeFields attributes, ModelContext.Properties properties) { + var functions = profile.getFunctions(); + assertEquals(expectedFunctions.size(), functions.size()); + for (String func : expectedFunctions) { + assertTrue(functions.containsKey(func)); + } + + RawRankProfile raw = new RawRankProfile(profile, largeExpressions, queryProfiles, models, attributes, properties); + assertEquals(rankProperties.size(), raw.configProperties().size()); + for (int i = 0; i < rankProperties.size(); i++) { + assertEquals(rankProperties.get(i).getFirst(), raw.configProperties().get(i).getFirst()); + assertEquals(rankProperties.get(i).getSecond(), raw.configProperties().get(i).getSecond()); + } + } + + private void verifySearch(Schema schema, RankProfileRegistry rankProfileRegistry, LargeRankExpressions largeExpressions, + QueryProfileRegistry queryProfiles, ImportedMlModels models, ModelContext.Properties properties) + { + AttributeFields attributes = new AttributeFields(schema); + + verifyProfile(rankProfileRegistry.get(schema, "base"), Arrays.asList("large_f", "large_m"), + Arrays.asList(new Pair<>("rankingExpression(large_f).expressionName", "base.large_f"), new Pair<>("rankingExpression(large_m).expressionName", "base.large_m")), + largeExpressions, queryProfiles, models, attributes, properties); + for (String child : Arrays.asList("child_a", "child_b")) { + verifyProfile(rankProfileRegistry.get(schema, child), Arrays.asList("large_f", "large_m", "large_local_f", "large_local_m"), + Arrays.asList(new Pair<>("rankingExpression(large_f).expressionName", child + ".large_f"), new Pair<>("rankingExpression(large_m).expressionName", child + ".large_m"), + new Pair<>("rankingExpression(large_local_f).expressionName", child + ".large_local_f"), new Pair<>("rankingExpression(large_local_m).expressionName", child + ".large_local_m"), + new Pair<>("vespa.rank.firstphase", "rankingExpression(firstphase)"), new Pair<>("rankingExpression(firstphase).expressionName", child + ".firstphase")), + largeExpressions, queryProfiles, models, attributes, properties); + } + } + + @Test + public void testLargeInheritedFunctions() throws IOException, ParseException { + ModelContext.Properties properties = new TestProperties(); + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + LargeRankExpressions largeExpressions = new LargeRankExpressions(new MockFileRegistry(), 50); + QueryProfileRegistry queryProfiles = new QueryProfileRegistry(); + ImportedMlModels models = new ImportedMlModels(); + Schema schema = createSearch("src/test/examples/largerankingexpressions", properties, rankProfileRegistry); + verifySearch(schema, rankProfileRegistry, largeExpressions, queryProfiles, models, properties); + // Need to verify that second derivation works as that will happen if same sd is used in multiple content clusters + verifySearch(schema, rankProfileRegistry, largeExpressions, queryProfiles, models, properties); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ReferenceFieldTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ReferenceFieldTestCase.java new file mode 100644 index 00000000000..57b4d928a52 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/ReferenceFieldTestCase.java @@ -0,0 +1,92 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.documentmodel.NewDocumentReferenceDataType; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.parser.ParseException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * @author bjorncs + */ +public class ReferenceFieldTestCase { + + @SuppressWarnings("deprecation") + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void reference_fields_are_parsed_from_search_definition() throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(); + String campaignSdContent = + "schema campaign {\n" + + " document campaign {\n" + + " }\n" + + "}"; + String salespersonSdContent = + "schema salesperson {\n" + + " document salesperson {\n" + + " }\n" + + "}"; + String adSdContent = + "schema ad {\n" + + " document ad {\n" + + " field campaign_ref type reference<campaign> { indexing: attribute }\n" + + " field salesperson_ref type reference<salesperson> { indexing: attribute }\n" + + " }\n" + + "}"; + builder.addSchema(campaignSdContent); + builder.addSchema(salespersonSdContent); + builder.addSchema(adSdContent); + builder.build(true); + Schema schema = builder.getSchema("ad"); + assertSearchContainsReferenceField("campaign_ref", "campaign", schema.getDocument()); + assertSearchContainsReferenceField("salesperson_ref", "salesperson", schema.getDocument()); + } + + @Test + public void cyclic_document_dependencies_are_detected() throws ParseException { + var builder = new ApplicationBuilder(new TestProperties()); + String campaignSdContent = + "schema campaign {\n" + + " document campaign {\n" + + " field ad_ref type reference<ad> { indexing: attribute }\n" + + " }\n" + + "}"; + String adSdContent = + "schema ad {\n" + + " document ad {\n" + + " field campaign_ref type reference<campaign> { indexing: attribute }\n" + + " }\n" + + "}"; + builder.addSchema(campaignSdContent); + builder.addSchema(adSdContent); + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("reference cycle for documents"); + builder.build(true); + } + + private static void assertSearchContainsReferenceField(String expectedFieldname, + String referencedDocType, + SDDocumentType documentType) { + Field field = documentType.getDocumentType().getField(expectedFieldname); + assertNotNull("Field does not exist in document type: " + expectedFieldname, field); + DataType dataType = field.getDataType(); + assertTrue(dataType instanceof NewDocumentReferenceDataType); + NewDocumentReferenceDataType refField = (NewDocumentReferenceDataType) dataType; + assertEquals(referencedDocType, refField.getTargetTypeName()); + assertTrue(! refField.isTemporary()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ReservedDocumentNamesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ReservedDocumentNamesTestCase.java new file mode 100644 index 00000000000..974d8c261ca --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/ReservedDocumentNamesTestCase.java @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.derived.AbstractExportingTestCase; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author Simon Thoresen Hult + */ +public class ReservedDocumentNamesTestCase extends AbstractExportingTestCase { + + @Test + public void requireThatPositionIsAReservedDocumentName() throws IOException, ParseException { + try { + assertCorrectDeriving("reserved_position"); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'position': Document name 'position' is reserved.", e.getMessage()); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ReservedRankingExpressionFunctionNamesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ReservedRankingExpressionFunctionNamesTestCase.java new file mode 100644 index 00000000000..e405a105f3c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/ReservedRankingExpressionFunctionNamesTestCase.java @@ -0,0 +1,71 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.util.logging.Level; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author lesters + */ +public class ReservedRankingExpressionFunctionNamesTestCase { + + @Test + public void requireThatFunctionsWithReservedNamesIssueAWarning() throws ParseException { + TestDeployLogger deployLogger = new TestDeployLogger(); + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(deployLogger, rankProfileRegistry); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile test_rank_profile {\n" + + " function not_a_reserved_name(x) {\n" + + " expression: x + x\n" + + " }\n" + + " function sigmoid(x) {\n" + + " expression: x * x\n" + + " }\n" + + " first-phase {\n" + + " expression: sigmoid(2) + not_a_reserved_name(1)\n" + + " }\n" + + " }\n" + + " rank-profile test_rank_profile_2 inherits test_rank_profile {\n" + + " function sin(x) {\n" + + " expression: x * x\n" + + " }\n" + + " first-phase {\n" + + " expression: sigmoid(2) + sin(1)\n" + + " }\n" + + " }\n" + + "}\n"); + builder.build(true); + + assertTrue(deployLogger.log.contains("sigmoid") && deployLogger.log.contains("test_rank_profile")); + assertTrue(deployLogger.log.contains("sigmoid") && deployLogger.log.contains("test_rank_profile_2")); + assertTrue(deployLogger.log.contains("sin") && deployLogger.log.contains("test_rank_profile_2")); + assertFalse(deployLogger.log.contains("not_a_reserved_name") && deployLogger.log.contains("test_rank_profile")); + assertFalse(deployLogger.log.contains("not_a_reserved_name") && deployLogger.log.contains("test_rank_profile_2")); + + } + + public static class TestDeployLogger implements DeployLogger { + public String log = ""; + @Override + public void log(Level level, String message) { + log += message; + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/SchemaMustHaveDocumentTest.java b/config-model/src/test/java/com/yahoo/schema/processing/SchemaMustHaveDocumentTest.java new file mode 100644 index 00000000000..03f9d7c5960 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/SchemaMustHaveDocumentTest.java @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.fail; + +/** + * @author hmusum + */ +public class SchemaMustHaveDocumentTest { + + @Test + public void requireErrorWhenMissingDocument() throws IOException, ParseException { + try { + ApplicationBuilder.buildFromFile("src/test/examples/invalid_sd_missing_document.sd"); + fail("SD without document"); + } catch (IllegalArgumentException e) { + if (!e.getMessage() + .contains("For schema 'imageconfig': A search specification must have an equally named document inside of it.")) { + throw e; + } + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java new file mode 100644 index 00000000000..76132a4d09f --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java @@ -0,0 +1,45 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import org.junit.Test; + +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.Assert.assertEquals; + +public class SummaryConsistencyTestCase { + + @Test + public void attribute_combiner_transform_is_set_when_source_is_array_of_struct_with_only_struct_field_attributes() throws ParseException { + String sd = joinLines( + "search structmemorysummary {", + " document structmemorysummary {", + " struct elem {", + " field name type string {}", + " field weight type int {}\n", + " }", + " field elem_array type array<elem> {", + " indexing: summary", + " struct-field name {", + " indexing: attribute", + " }", + " struct-field weight {", + " indexing: attribute", + " }", + " }", + " }", + " document-summary unfiltered {", + " summary elem_array_unfiltered type array<elem> {", + " source: elem_array", + " }", + " }", + "", + "}" + ); + Schema schema = ApplicationBuilder.createFromString(sd).getSchema(); + assertEquals(SummaryTransform.ATTRIBUTECOMBINER, schema.getSummaryField("elem_array_unfiltered").getTransform()); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/SummaryFieldsMustHaveValidSourceTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/SummaryFieldsMustHaveValidSourceTestCase.java new file mode 100644 index 00000000000..d94815015d7 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/SummaryFieldsMustHaveValidSourceTestCase.java @@ -0,0 +1,60 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; + +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class SummaryFieldsMustHaveValidSourceTestCase extends AbstractSchemaTestCase { + + @Test + public void requireThatInvalidSourceIsCaught() throws IOException, ParseException { + try { + ApplicationBuilder.buildFromFile("src/test/examples/invalidsummarysource.sd"); + fail("This should throw and never get here"); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'nonexistingfield'.", e.getMessage()); + } + } + + @Test + public void requireThatInvalidImplicitSourceIsCaught() throws IOException, ParseException { + try { + ApplicationBuilder.buildFromFile("src/test/examples/invalidimplicitsummarysource.sd"); + fail("This should throw and never get here"); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'cox'.", e.getMessage()); + } + } + + @Test + public void requireThatInvalidSelfReferingSingleSource() throws IOException, ParseException { + try { + ApplicationBuilder.buildFromFile("src/test/examples/invalidselfreferringsummary.sd"); + fail("This should throw and never get here"); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'invalidselfreferringsummary', summary class 'withid', summary field 'w': there is no valid source 'w'.", e.getMessage()); + } + } + + @Test + public void requireThatDocumentIdIsAllowedToPass() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/documentidinsummary.sd"); + BaseDeployLogger deployLogger = new BaseDeployLogger(); + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + new SummaryFieldsMustHaveValidSource(schema, deployLogger, rankProfileRegistry, new QueryProfiles()).process(true, false); + assertEquals("documentid", schema.getSummary("withid").getSummaryField("w").getSingleSource()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/TensorFieldTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/TensorFieldTestCase.java new file mode 100644 index 00000000000..67c77508e3b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/TensorFieldTestCase.java @@ -0,0 +1,172 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + + +import static com.yahoo.schema.ApplicationBuilder.createFromString; +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author geirst + */ +public class TensorFieldTestCase { + + @Test + public void requireThatTensorFieldCannotBeOfCollectionType() throws ParseException { + try { + createFromString(getSd("field f1 type array<tensor(x{})> {}")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'f1': A field with collection type of tensor is not supported. Use simple type 'tensor' instead.", + e.getMessage()); + } + } + + @Test + public void requireThatTensorFieldCannotBeIndexField() throws ParseException { + try { + createFromString(getSd("field f1 type tensor(x{}) { indexing: index }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'f1': A tensor of type 'tensor(x{})' does not support having an 'index'. " + + "Currently, only tensors with 1 indexed dimension supports that.", + e.getMessage()); + } + } + + @Test + public void requireThatIndexedTensorAttributeCannotBeFastSearch() throws ParseException { + try { + createFromString(getSd("field f1 type tensor(x[3]) { indexing: attribute \n attribute: fast-search }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'f1': An attribute of type 'tensor' cannot be 'fast-search'.", e.getMessage()); + } + } + + @Test + public void requireThatIllegalTensorTypeSpecThrowsException() throws ParseException { + try { + createFromString(getSd("field f1 type tensor(invalid) { indexing: attribute }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertStartsWith("Field type: Illegal tensor type spec:", e.getMessage()); + } + } + + @Test + public void hnsw_index_is_default_turned_off() throws ParseException { + var attr = getAttributeFromSd("field t1 type tensor(x[64]) { indexing: attribute }", "t1"); + assertFalse(attr.hnswIndexParams().isPresent()); + } + + @Test + public void hnsw_index_gets_default_parameters_if_not_specified() throws ParseException { + assertHnswIndexParams("", 16, 200); + assertHnswIndexParams("index: hnsw", 16, 200); + } + + @Test + public void hnsw_index_parameters_can_be_specified() throws ParseException { + assertHnswIndexParams("index { hnsw { max-links-per-node: 32 } }", 32, 200); + assertHnswIndexParams("index { hnsw { neighbors-to-explore-at-insert: 300 } }", 16, 300); + assertHnswIndexParams(joinLines("index {", + " hnsw {", + " max-links-per-node: 32", + " neighbors-to-explore-at-insert: 300", + " }", + "}"), + 32, 300); + } + + @Test + public void tensor_with_hnsw_index_must_be_an_attribute() throws ParseException { + try { + createFromString(getSd("field t1 type tensor(x[64]) { indexing: index }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 't1': A tensor that has an index must also be an attribute.", e.getMessage()); + } + } + + @Test + public void tensor_with_hnsw_index_parameters_must_be_an_index() throws ParseException { + try { + createFromString(getSd(joinLines( + "field t1 type tensor(x[64]) {", + " indexing: attribute ", + " index {", + " hnsw { max-links-per-node: 32 }", + " }", + "}"))); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 't1': " + + "A tensor that specifies hnsw index parameters must also specify 'index' in 'indexing'", + e.getMessage()); + } + } + + @Test + public void tensors_with_at_least_one_mapped_dimension_can_be_direct() throws ParseException { + assertTrue(getAttributeFromSd( + "field t1 type tensor(x{}) { indexing: attribute \n attribute: fast-search }", "t1").isFastSearch()); + assertTrue(getAttributeFromSd( + "field t1 type tensor(x{},y{},z[4]) { indexing: attribute \n attribute: fast-search }", "t1").isFastSearch()); + } + + @Test + public void tensors_with_at_least_one_mapped_dimension_can_be_fast_rank() throws ParseException { + assertTrue(getAttributeFromSd( + "field t1 type tensor(x{}) { indexing: attribute \n attribute: fast-rank }", "t1").isFastRank()); + assertTrue(getAttributeFromSd( + "field t1 type tensor(x{},y{},z[4]) { indexing: attribute \n attribute: fast-rank }", "t1").isFastRank()); + } + + private static String getSd(String field) { + return joinLines("search test {", + " document test {", + " " + field, + " }", + "}"); + } + + private Attribute getAttributeFromSd(String fieldSpec, String attrName) throws ParseException { + return createFromString(getSd(fieldSpec)).getSchema().getAttribute(attrName); + } + + private void assertHnswIndexParams(String indexSpec, int maxLinksPerNode, int neighborsToExploreAtInsert) throws ParseException { + var sd = getSdWithIndexSpec(indexSpec); + var search = createFromString(sd).getSchema(); + var attr = search.getAttribute("t1"); + var params = attr.hnswIndexParams(); + assertTrue(params.isPresent()); + assertEquals(maxLinksPerNode, params.get().maxLinksPerNode()); + assertEquals(neighborsToExploreAtInsert, params.get().neighborsToExploreAtInsert()); + } + + private String getSdWithIndexSpec(String indexSpec) { + return getSd(joinLines("field t1 type tensor(x[64]) {", + " indexing: attribute | index", + " " + indexSpec, + "}")); + } + + private void assertStartsWith(String prefix, String string) { + assertEquals(prefix, string.substring(0, Math.min(prefix.length(), string.length()))); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/TensorTransformTestCase.java new file mode 100644 index 00000000000..aaf5f381c62 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/TensorTransformTestCase.java @@ -0,0 +1,234 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.collections.Pair; +import com.yahoo.component.ComponentId; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; +import com.yahoo.schema.LargeRankExpressions; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.RawRankProfile; +import com.yahoo.schema.parser.ParseException; +import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class TensorTransformTestCase extends AbstractSchemaTestCase { + + @Test + public void requireThatNormalMaxAndMinAreNotReplaced() throws ParseException { + assertTransformedExpression("max(1.0,2.0)", + "max(1.0,2.0)"); + assertTransformedExpression("min(attribute(double_field),x)", + "min(attribute(double_field),x)"); + assertTransformedExpression("max(attribute(double_field),attribute(double_array_field))", + "max(attribute(double_field),attribute(double_array_field))"); + assertTransformedExpression("min(attribute(tensor_field_1),attribute(double_field))", + "min(attribute(tensor_field_1),attribute(double_field))"); + assertTransformedExpression("reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),sum)", + "reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),sum)"); + assertTransformedExpression("min(constant(test_constant_tensor),1.0)", + "min(test_constant_tensor,1.0)"); + assertTransformedExpression("max(constant(base_constant_tensor),1.0)", + "max(base_constant_tensor,1.0)"); + assertTransformedExpression("min(constant(file_constant_tensor),1.0)", + "min(constant(file_constant_tensor),1.0)"); + assertTransformedExpression("max(query(q),1.0)", + "max(query(q),1.0)"); + assertTransformedExpression("max(query(n),1.0)", + "max(query(n),1.0)"); + } + + @Test + public void requireThatMaxAndMinWithTensorAttributesAreReplaced() throws ParseException { + assertTransformedExpression("reduce(attribute(tensor_field_1),max,x)", + "max(attribute(tensor_field_1),x)"); + assertTransformedExpression("1+reduce(attribute(tensor_field_1),max,x)", + "1 + max(attribute(tensor_field_1),x)"); + assertTransformedExpression("if(attribute(double_field),1+reduce(attribute(tensor_field_1),max,x),reduce(attribute(tensor_field_1),sum,x))", + "if(attribute(double_field),1 + max(attribute(tensor_field_1),x),reduce(attribute(tensor_field_1), sum, x))"); + assertTransformedExpression("reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),max,x)", + "max(max(attribute(tensor_field_1),attribute(tensor_field_2)),x)"); + assertTransformedExpression("reduce(if(attribute(double_field),attribute(tensor_field_2),attribute(tensor_field_2)),max,x)", + "max(if(attribute(double_field),attribute(tensor_field_2),attribute(tensor_field_2)),x)"); + assertTransformedExpression("max(reduce(attribute(tensor_field_1),max,x),x)", + "max(max(attribute(tensor_field_1),x),x)"); // will result in deploy error. + assertTransformedExpression("reduce(reduce(attribute(tensor_field_2),max,x),max,y)", + "max(max(attribute(tensor_field_2),x),y)"); + } + + @Test + public void requireThatMaxAndMinWithConstantTensorsAreReplaced() throws ParseException { + assertTransformedExpression("reduce(constant(test_constant_tensor),max,x)", + "max(test_constant_tensor,x)"); + assertTransformedExpression("reduce(constant(base_constant_tensor),max,x)", + "max(base_constant_tensor,x)"); + assertTransformedExpression("reduce(constant(file_constant_tensor),min,x)", + "min(constant(file_constant_tensor),x)"); + } + + @Test + public void requireThatMaxAndMinWithTensorExpressionsAreReplaced() throws ParseException { + assertTransformedExpression("reduce(attribute(double_field)+attribute(tensor_field_1),min,x)", + "min(attribute(double_field) + attribute(tensor_field_1),x)"); + assertTransformedExpression("reduce(attribute(tensor_field_1)*attribute(tensor_field_2),min,x)", + "min(attribute(tensor_field_1) * attribute(tensor_field_2),x)"); + assertTransformedExpression("reduce(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),min,x)", + "min(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),x)"); + assertTransformedExpression("min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)", + "min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)"); // because tensor fields are not in attribute(...) + assertTransformedExpression("reduce(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),min,x)", + "min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)"); + } + + @Test + public void requireThatMaxAndMinWithTensorFromIsReplaced() throws ParseException { + assertTransformedExpression("reduce(tensorFromLabels(attribute(double_array_field)),max,double_array_field)", + "max(tensorFromLabels(attribute(double_array_field)),double_array_field)"); + assertTransformedExpression("reduce(tensorFromLabels(attribute(double_array_field),x),max,x)", + "max(tensorFromLabels(attribute(double_array_field),x),x)"); + assertTransformedExpression("reduce(tensorFromWeightedSet(attribute(weightedset_field)),max,weightedset_field)", + "max(tensorFromWeightedSet(attribute(weightedset_field)),weightedset_field)"); + assertTransformedExpression("reduce(tensorFromWeightedSet(attribute(weightedset_field),x),max,x)", + "max(tensorFromWeightedSet(attribute(weightedset_field),x),x)"); + } + + @Test + public void requireThatMaxAndMinWithTensorInQueryIsReplaced() throws ParseException { + assertTransformedExpression("reduce(query(q),max,x)", "max(query(q),x)"); + assertTransformedExpression("max(query(n),x)", "max(query(n),x)"); + } + + @Test + public void requireThatMaxAndMinWithTensorsReturnedFromFunctionsAreReplaced() throws ParseException { + assertTransformedExpression("reduce(rankingExpression(returns_tensor),max,x)", + "max(returns_tensor,x)"); + assertTransformedExpression("reduce(rankingExpression(wraps_returns_tensor),max,x)", + "max(wraps_returns_tensor,x)"); + assertTransformedExpression("reduce(rankingExpression(tensor_inheriting),max,x)", + "max(tensor_inheriting,x)"); + assertTransformedExpression("reduce(rankingExpression(returns_tensor_with_arg@),max,x)", + "max(returns_tensor_with_arg(attribute(tensor_field_1)),x)"); + } + + private void assertTransformedExpression(String expected, String original) throws ParseException { + for (Pair<String, String> rankPropertyExpression : buildSearch(original)) { + String rankProperty = rankPropertyExpression.getFirst(); + if (rankProperty.equals("rankingExpression(testexpression).rankingScript")) { + String rankExpression = censorBindingHash(rankPropertyExpression.getSecond().replace(" ","")); + assertEquals(expected, rankExpression); + return; + } + } + fail("No 'rankingExpression(testexpression).rankingScript' property produced"); + } + + private List<Pair<String, String>> buildSearch(String expression) throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + QueryProfileRegistry queryProfiles = setupQueryProfileTypes(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry, queryProfiles); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " field double_field type double { \n" + + " indexing: summary | attribute \n" + + " }\n" + + " field double_array_field type array<double> { \n" + + " indexing: summary | attribute \n" + + " }\n" + + " field weightedset_field type weightedset<int> { \n" + + " indexing: summary | attribute \n" + + " }\n" + + " field tensor_field_1 type tensor(x{}) { \n" + + " indexing: summary | attribute \n" + + " }\n" + + " field tensor_field_2 type tensor(x[3],y[3]) { \n" + + " indexing: summary | attribute \n" + + " }\n" + + " }\n" + + " constant file_constant_tensor {\n" + + " file: constants/tensor.json\n" + + " type: tensor(x{})\n" + + " }\n" + + " rank-profile base {\n" + + " constants {\n" + + " base_constant_tensor tensor(x[1]):[0]\n"+ + " }\n" + + " function base_tensor() {\n" + + " expression: constant(base_constant_tensor)\n" + + " }\n" + + " }\n" + + " rank-profile test inherits base {\n" + + " constants {\n" + + " test_constant_tensor tensor(x[1]):[1]" + + " }\n" + + " function returns_tensor_with_arg(arg1) {\n" + + " expression: 2.0 * arg1\n" + + " }\n" + + " function wraps_returns_tensor() {\n" + + " expression: returns_tensor\n" + + " }\n" + + " function returns_tensor() {\n" + + " expression: attribute(tensor_field_2)\n" + + " }\n" + + " function tensor_inheriting() {\n" + + " expression: base_tensor\n" + + " }\n" + + " function testexpression() {\n" + + " expression: " + expression + "\n" + + " }\n" + + " }\n" + + "}\n"); + builder.build(true); + Schema s = builder.getSchema(); + RankProfile test = rankProfileRegistry.get(s, "test").compile(queryProfiles, new ImportedMlModels()); + List<Pair<String, String>> testRankProperties = new RawRankProfile(test, + new LargeRankExpressions(new MockFileRegistry()), + queryProfiles, + new ImportedMlModels(), + new AttributeFields(s), new TestProperties()).configProperties(); + return testRankProperties; + } + + private static QueryProfileRegistry setupQueryProfileTypes() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfileTypeRegistry typeRegistry = registry.getTypeRegistry(); + QueryProfileType type = new QueryProfileType(new ComponentId("testtype")); + type.addField(new FieldDescription("ranking.features.query(q)", + FieldType.fromString("tensor(x{})", typeRegistry)), typeRegistry); + type.addField(new FieldDescription("ranking.features.query(n)", + FieldType.fromString("integer", typeRegistry)), typeRegistry); + typeRegistry.register(type); + return registry; + } + + private String censorBindingHash(String s) { + StringBuilder b = new StringBuilder(); + boolean areInHash = false; + for (int i = 0; i < s.length() ; i++) { + char current = s.charAt(i); + if ( ! Character.isLetterOrDigit(current)) // end of hash + areInHash = false; + if ( ! areInHash) + b.append(current); + if (current == '@') // start of hash + areInHash = true; + } + return b.toString(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ValidateFieldTypesTest.java b/config-model/src/test/java/com/yahoo/schema/processing/ValidateFieldTypesTest.java new file mode 100644 index 00000000000..87bb2e96042 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/ValidateFieldTypesTest.java @@ -0,0 +1,80 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.schema.DocumentReference; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.TestableDeployLogger; +import com.yahoo.schema.document.ImportedField; +import com.yahoo.schema.document.ImportedFields; +import com.yahoo.schema.document.ImportedSimpleField; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.Collections; + +/** + * @author bjorncs + */ +public class ValidateFieldTypesTest { + + private static final String IMPORTED_FIELD_NAME = "imported_myfield"; + private static final String DOCUMENT_NAME = "my_doc"; + + @SuppressWarnings("deprecation") + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void throws_exception_if_type_of_document_field_does_not_match_summary_field() { + Schema schema = createSearchWithDocument(DOCUMENT_NAME); + schema.setImportedFields(createSingleImportedField(IMPORTED_FIELD_NAME, DataType.INT)); + schema.addSummary(createDocumentSummary(IMPORTED_FIELD_NAME, DataType.STRING, schema)); + + ValidateFieldTypes validator = new ValidateFieldTypes(schema, null, null, null); + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage( + "For schema '" + DOCUMENT_NAME + "', field '" + IMPORTED_FIELD_NAME + "': Incompatible types. " + + "Expected int for summary field '" + IMPORTED_FIELD_NAME + "', got string."); + validator.process(true, false); + } + + private static Schema createSearch(String documentType) { + return new Schema(documentType, + MockApplicationPackage.createEmpty(), + new MockFileRegistry(), + new TestableDeployLogger(), + new TestProperties()); + } + + private static Schema createSearchWithDocument(String documentName) { + Schema schema = createSearch(documentName); + SDDocumentType document = new SDDocumentType(documentName, schema); + schema.addDocument(document); + return schema; + } + + private static ImportedFields createSingleImportedField(String fieldName, DataType dataType) { + Schema targetSchema = createSearchWithDocument("target_doc"); + SDField targetField = new SDField(targetSchema.getDocument(), "target_field", dataType); + DocumentReference documentReference = new DocumentReference(new Field("reference_field"), targetSchema); + ImportedField importedField = new ImportedSimpleField(fieldName, documentReference, targetField); + return new ImportedFields(Collections.singletonMap(fieldName, importedField)); + } + + private static DocumentSummary createDocumentSummary(String fieldName, DataType dataType, Schema schema) { + DocumentSummary summary = new DocumentSummary("mysummary", schema); + summary.add(new SummaryField(fieldName, dataType)); + return summary; + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/VespaMlModelTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/VespaMlModelTestCase.java new file mode 100644 index 00000000000..016e30e80af --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/VespaMlModelTestCase.java @@ -0,0 +1,77 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.schema.derived.RawRankProfile; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.ml.ImportedModelTester; +import org.junit.After; +import org.junit.Test; + +import java.io.IOException; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; + +/** + * Tests adding Vespa ranking expression based models in the models/ dir + * + * @author bratseth + */ +public class VespaMlModelTestCase { + + private final Path applicationDir = Path.fromString("src/test/integration/vespa/"); + + private final String expectedRankConfig = + "constant(constant1).type : tensor(x[3])\n" + + "constant(constant1).value : tensor(x[3]):[0.5, 1.5, 2.5]\n" + + "rankingExpression(foo1).rankingScript : reduce(reduce(input1 * input2, sum, name) * constant(constant1), max, x) * 3.0\n" + + "rankingExpression(foo1).input2.type : tensor(x[3])\n" + + "rankingExpression(foo1).input1.type : tensor(name{},x[3])\n" + + "rankingExpression(foo2).rankingScript : reduce(reduce(input1 * input2, sum, name) * constant(constant1asLarge), max, x) * 3.0\n" + + "rankingExpression(foo2).input2.type : tensor(x[3])\n" + + "rankingExpression(foo2).input1.type : tensor(name{},x[3])\n"; + + /** The model name */ + private final String name = "example"; + + @After + public void removeGeneratedModelFiles() { + IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + } + + @Test + public void testGlobalVespaModel() throws IOException { + ImportedModelTester tester = new ImportedModelTester(name, applicationDir); + VespaModel model = tester.createVespaModel(); + tester.assertLargeConstant("constant1asLarge", model, Optional.of(3L)); + assertEquals(expectedRankConfig, rankConfigOf("example", model)); + + // At this point the expression is stored - copy application to another location which do not have a models dir + Path storedAppDir = applicationDir.append("copy"); + try { + storedAppDir.toFile().mkdirs(); + IOUtils.copy(applicationDir.append("services.xml").toString(), storedAppDir.append("services.xml").toString()); + IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), + storedAppDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + ImportedModelTester storedTester = new ImportedModelTester(name, storedAppDir); + VespaModel storedModel = storedTester.createVespaModel(); + storedTester.assertLargeConstant("constant1asLarge", model, Optional.of(3L)); + assertEquals(expectedRankConfig, rankConfigOf("example", storedModel)); + } + finally { + IOUtils.recursiveDeleteDir(storedAppDir.toFile()); + } + } + + private String rankConfigOf(String rankProfileName, VespaModel model) { + StringBuilder b = new StringBuilder(); + RawRankProfile profile = model.rankProfileList().getRankProfiles().get(rankProfileName); + for (var property : profile.configProperties()) + b.append(property.getFirst()).append(" : ").append(property.getSecond()).append("\n"); + return b.toString(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/WeightedSetSummaryToTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/WeightedSetSummaryToTestCase.java new file mode 100644 index 00000000000..2f62228cc3f --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/WeightedSetSummaryToTestCase.java @@ -0,0 +1,23 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertNotNull; + +/** @author bratseth */ +public class WeightedSetSummaryToTestCase extends AbstractSchemaTestCase { + + @Test + public void testRequireThatImplicitFieldsAreCreated() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/weightedset-summaryto.sd"); + assertNotNull(schema); + } + +} |