diff options
author | Tor Egge <Tor.Egge@broadpark.no> | 2018-11-29 16:35:06 +0100 |
---|---|---|
committer | Tor Egge <Tor.Egge@broadpark.no> | 2018-11-30 12:06:01 +0100 |
commit | ca699faa10770c397a82923c29cd7247d3dfde60 (patch) | |
tree | 63d2e7a5d8d809018bb725e5d2acc8ecb92fc6b3 | |
parent | 1b2ccec2d8f5ad05a7a718b62f256b4fd19bda74 (diff) |
Derive config for imported position field.
12 files changed, 262 insertions, 7 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java index 9ddb18ecc8d..4f71bd90830 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java @@ -88,7 +88,14 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { private void deriveImportedComplexField(ImportedField field) { String fieldName = field.fieldName(); - addIndexCommand(fieldName, CMD_MULTIVALUE); + if (isPositionField(field.targetField())) { + addIndexCommand(fieldName, CMD_DEFAULT_POSITION); + if (isPositionArrayField(field.targetField())) { + addIndexCommand(fieldName, CMD_MULTIVALUE); + } + } else { + addIndexCommand(fieldName, CMD_MULTIVALUE); + } addIndexCommand(fieldName, CMD_INDEX); } @@ -103,6 +110,14 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { return b.toString(); } + private static boolean isPositionArrayField(ImmutableSDField field) { + return field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE)); + } + + private static boolean isPositionField(ImmutableSDField field) { + return field.getDataType().equals(PositionDataType.INSTANCE) || isPositionArrayField(field); + } + @Override protected void derive(ImmutableSDField field, Search search) { derive(field, search, false); @@ -127,8 +142,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { String name = e.getValue(); addIndexAlias(alias, name); } - boolean isPosition = field.getDataType().equals(PositionDataType.INSTANCE) || - field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE)); + boolean isPosition = isPositionField(field); if (field.usesStructOrMap()) { for (ImmutableSDField structField : field.getStructFields()) { derive(structField, search, isPosition); // Recursion diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java index bfb2fa9c120..c8ecc31103a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java @@ -5,6 +5,7 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.MapDataType; +import com.yahoo.document.PositionDataType; import com.yahoo.document.StructDataType; import com.yahoo.searchdefinition.DocumentReference; import com.yahoo.searchdefinition.DocumentReferences; @@ -50,7 +51,10 @@ public class ImportedFieldsResolver extends Processor { private void resolveImportedField(TemporaryImportedField importedField, boolean validate) { DocumentReference reference = validateDocumentReference(importedField); ImmutableSDField targetField = getTargetField(importedField, reference); - if (isArrayOfSimpleStruct(targetField)) { + if (targetField.getDataType().equals(PositionDataType.INSTANCE) || + targetField.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))) { + resolveImportedPositionField(importedField, reference, targetField, validate); + } else if (isArrayOfSimpleStruct(targetField)) { resolveImportedArrayOfStructField(importedField, reference, targetField, validate); } else if (isMapOfSimpleStruct(targetField)) { resolveImportedMapOfStructField(importedField, reference, targetField, validate); @@ -61,6 +65,15 @@ public class ImportedFieldsResolver extends Processor { } } + private void resolveImportedPositionField(TemporaryImportedField importedField, DocumentReference reference, + ImmutableSDField targetField, boolean validate) { + TemporaryImportedField importedZCurveField = new TemporaryImportedField(PositionDataType.getZCurveFieldName(importedField.fieldName()), + reference.referenceField().getName(), PositionDataType.getZCurveFieldName(targetField.getName())); + ImmutableSDField targetZCurveField = getTargetField(importedZCurveField, reference); + resolveImportedNormalField(importedZCurveField, reference, targetZCurveField, validate); + makeImportedComplexField(importedField, reference, targetField); + } + private void resolveImportedArrayOfStructField(TemporaryImportedField importedField, DocumentReference reference, ImmutableSDField targetField, boolean validate) { resolveImportedNestedStructField(importedField, reference, targetField, validate); @@ -74,6 +87,14 @@ public class ImportedFieldsResolver extends Processor { makeImportedComplexField(importedField, reference, targetField); } + private void makeImportedNormalField(TemporaryImportedField importedField, String name, DocumentReference reference, + ImmutableSDField targetField) { + if (importedFields.get(name) != null) { + fail(importedField, name, targetFieldAsString(targetField.getName(), reference) +": Field already imported"); + } + importedFields.put(name, new ImportedField(name, reference, targetField)); + } + private void makeImportedComplexField(TemporaryImportedField importedField, DocumentReference reference, ImmutableSDField targetField) { String name = importedField.fieldName(); @@ -89,7 +110,7 @@ public class ImportedFieldsResolver extends Processor { Attribute attribute = targetNestedField.getAttributes().get(targetNestedField.getName()); String importedNestedFieldName = makeImportedNestedFieldName(importedField, targetNestedField); if (attribute != null) { - importedFields.put(importedNestedFieldName, new ImportedField(importedNestedFieldName, reference, targetNestedField)); + makeImportedNormalField(importedField, importedNestedFieldName, reference, targetNestedField); } else if (requireAttribute) { fail(importedField, importedNestedFieldName, targetFieldAsString(targetNestedField.getName(), reference) + ": Is not an attribute field. Only attribute fields supported"); @@ -124,7 +145,7 @@ public class ImportedFieldsResolver extends Processor { if (validate) { validateTargetField(importedField, targetField, reference); } - importedFields.put(importedField.fieldName(), new ImportedField(importedField.fieldName(), reference, targetField)); + makeImportedNormalField(importedField, importedField.fieldName(), reference, targetField); } private DocumentReference validateDocumentReference(TemporaryImportedField importedField) { @@ -170,6 +191,9 @@ public class ImportedFieldsResolver extends Processor { } private void fail(TemporaryImportedField importedField, String importedNestedFieldName, String msg) { + if (importedField.fieldName().equals(importedNestedFieldName)) { + fail(importedField, msg); + } throw new IllegalArgumentException("For search '" + search.getName() + "', import field '" + importedField.fieldName() + "' (nested to '" + importedNestedFieldName + "'): " + msg); } diff --git a/config-model/src/test/derived/imported_position_field/attributes.cfg b/config-model/src/test/derived/imported_position_field/attributes.cfg new file mode 100644 index 00000000000..db2280a7846 --- /dev/null +++ b/config-model/src/test/derived/imported_position_field/attributes.cfg @@ -0,0 +1,42 @@ +attribute[0].name "parent_ref" +attribute[0].datatype REFERENCE +attribute[0].collectiontype SINGLE +attribute[0].removeifzero false +attribute[0].createifnonexistent false +attribute[0].fastsearch false +attribute[0].huge false +attribute[0].ismutable false +attribute[0].sortascending true +attribute[0].sortfunction UCA +attribute[0].sortstrength PRIMARY +attribute[0].sortlocale "" +attribute[0].enablebitvectors false +attribute[0].enableonlybitvector false +attribute[0].fastaccess false +attribute[0].arity 8 +attribute[0].lowerbound -9223372036854775808 +attribute[0].upperbound 9223372036854775807 +attribute[0].densepostinglistthreshold 0.4 +attribute[0].tensortype "" +attribute[0].imported false +attribute[1].name "my_pos_zcurve" +attribute[1].datatype INT64 +attribute[1].collectiontype SINGLE +attribute[1].removeifzero false +attribute[1].createifnonexistent false +attribute[1].fastsearch true +attribute[1].huge false +attribute[1].ismutable false +attribute[1].sortascending true +attribute[1].sortfunction UCA +attribute[1].sortstrength PRIMARY +attribute[1].sortlocale "" +attribute[1].enablebitvectors false +attribute[1].enableonlybitvector false +attribute[1].fastaccess false +attribute[1].arity 8 +attribute[1].lowerbound -9223372036854775808 +attribute[1].upperbound 9223372036854775807 +attribute[1].densepostinglistthreshold 0.4 +attribute[1].tensortype "" +attribute[1].imported true
\ No newline at end of file diff --git a/config-model/src/test/derived/imported_position_field/child.sd b/config-model/src/test/derived/imported_position_field/child.sd new file mode 100644 index 00000000000..29997b855bd --- /dev/null +++ b/config-model/src/test/derived/imported_position_field/child.sd @@ -0,0 +1,9 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search child { + document child { + field parent_ref type reference<parent> { + indexing: attribute | summary + } + } + import field parent_ref.pos as my_pos {} +} diff --git a/config-model/src/test/derived/imported_position_field/imported-fields.cfg b/config-model/src/test/derived/imported_position_field/imported-fields.cfg new file mode 100644 index 00000000000..51b371827c5 --- /dev/null +++ b/config-model/src/test/derived/imported_position_field/imported-fields.cfg @@ -0,0 +1,5 @@ +attribute[0].name "my_pos_zcurve" +attribute[0].referencefield "parent_ref" +attribute[0].targetfield "pos_zcurve" +attribute[0].datatype NONE +attribute[0].collectiontype SINGLE
\ No newline at end of file diff --git a/config-model/src/test/derived/imported_position_field/index-info.cfg b/config-model/src/test/derived/imported_position_field/index-info.cfg new file mode 100644 index 00000000000..d8fffbdfb2d --- /dev/null +++ b/config-model/src/test/derived/imported_position_field/index-info.cfg @@ -0,0 +1,27 @@ +indexinfo[0].name "child" +indexinfo[0].command[0].indexname "sddocname" +indexinfo[0].command[0].command "index" +indexinfo[0].command[1].indexname "sddocname" +indexinfo[0].command[1].command "word" +indexinfo[0].command[2].indexname "parent_ref" +indexinfo[0].command[2].command "index" +indexinfo[0].command[3].indexname "parent_ref" +indexinfo[0].command[3].command "attribute" +indexinfo[0].command[4].indexname "parent_ref" +indexinfo[0].command[4].command "word" +indexinfo[0].command[5].indexname "rankfeatures" +indexinfo[0].command[5].command "index" +indexinfo[0].command[6].indexname "summaryfeatures" +indexinfo[0].command[6].command "index" +indexinfo[0].command[7].indexname "my_pos_zcurve" +indexinfo[0].command[7].command "index" +indexinfo[0].command[8].indexname "my_pos_zcurve" +indexinfo[0].command[8].command "attribute" +indexinfo[0].command[9].indexname "my_pos_zcurve" +indexinfo[0].command[9].command "fast-search" +indexinfo[0].command[10].indexname "my_pos_zcurve" +indexinfo[0].command[10].command "numerical" +indexinfo[0].command[11].indexname "my_pos" +indexinfo[0].command[11].command "default-position" +indexinfo[0].command[12].indexname "my_pos" +indexinfo[0].command[12].command "index"
\ No newline at end of file diff --git a/config-model/src/test/derived/imported_position_field/parent.sd b/config-model/src/test/derived/imported_position_field/parent.sd new file mode 100644 index 00000000000..493e105e5fd --- /dev/null +++ b/config-model/src/test/derived/imported_position_field/parent.sd @@ -0,0 +1,8 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search parent { + document parent { + field pos type position { + indexing: attribute | summary + } + } +} diff --git a/config-model/src/test/derived/imported_position_field/summary.cfg b/config-model/src/test/derived/imported_position_field/summary.cfg new file mode 100644 index 00000000000..8da2598a5fc --- /dev/null +++ b/config-model/src/test/derived/imported_position_field/summary.cfg @@ -0,0 +1,17 @@ +defaultsummaryid 1570252291 +classes[0].id 1570252291 +classes[0].name "default" +classes[0].fields[0].name "parent_ref" +classes[0].fields[0].type "longstring" +classes[0].fields[1].name "rankfeatures" +classes[0].fields[1].type "featuredata" +classes[0].fields[2].name "summaryfeatures" +classes[0].fields[2].type "featuredata" +classes[0].fields[3].name "documentid" +classes[0].fields[3].type "longstring" +classes[1].id 1274088866 +classes[1].name "attributeprefetch" +classes[1].fields[0].name "rankfeatures" +classes[1].fields[0].type "featuredata" +classes[1].fields[1].name "summaryfeatures" +classes[1].fields[1].type "featuredata"
\ No newline at end of file diff --git a/config-model/src/test/derived/imported_position_field/summarymap.cfg b/config-model/src/test/derived/imported_position_field/summarymap.cfg new file mode 100644 index 00000000000..42b6e811ee6 --- /dev/null +++ b/config-model/src/test/derived/imported_position_field/summarymap.cfg @@ -0,0 +1,7 @@ +defaultoutputclass -1 +override[0].field "rankfeatures" +override[0].command "rankfeatures" +override[0].arguments "" +override[1].field "summaryfeatures" +override[1].command "summaryfeatures" +override[1].arguments ""
\ No newline at end of file diff --git a/config-model/src/test/derived/imported_position_field/vsmsummary.cfg b/config-model/src/test/derived/imported_position_field/vsmsummary.cfg new file mode 100644 index 00000000000..4a09e153dde --- /dev/null +++ b/config-model/src/test/derived/imported_position_field/vsmsummary.cfg @@ -0,0 +1,8 @@ +outputclass "" +fieldmap[0].summary "parent_ref" +fieldmap[0].document[0].field "parent_ref" +fieldmap[0].command NONE +fieldmap[1].summary "rankfeatures" +fieldmap[1].command NONE +fieldmap[2].summary "summaryfeatures" +fieldmap[2].command NONE
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java index f7641f97df6..b4c42c248ef 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java @@ -20,4 +20,9 @@ public class ImportedFieldsTestCase extends AbstractExportingTestCase { public void configs_for_imported_struct_fields_are_derived() throws IOException, ParseException { assertCorrectDeriving("imported_struct_fields", "child"); } + + @Test + public void configs_for_imported_position_field_are_derived() throws IOException, ParseException { + assertCorrectDeriving("imported_position_field", "child"); + } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java index 536a52dd1b6..b1bb282a7a2 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java @@ -111,7 +111,7 @@ public class ImportedFieldsTestCase { @Test public void check_illegal_struct_import_missing_array_of_struct_attributes() throws ParseException { exception.expect(IllegalArgumentException.class); - exception.expectMessage("For search 'child', import field 'my_elem_array' (nested to 'my_elem_array'): Field 'elem_array' via reference field 'parent_ref': Is not a struct containing an attribute field."); + exception.expectMessage("For search 'child', import field 'my_elem_array': Field 'elem_array' via reference field 'parent_ref': Is not a struct containing an attribute field."); checkStructImport(new ParentSdBuilder().elem_array_name_attr(false).elem_array_weight_attr(false)); } @@ -235,6 +235,95 @@ public class ImportedFieldsTestCase { return builder.getSearch("child"); } + private static class ParentPosSdBuilder { + public String build() { + return joinLines("search parent {", + " document parent {", + "field pos type position {", + "indexing: attribute | summary", + " }", + " }", + "}"); + } + } + + private static class ChildPosSdBuilder { + private boolean import_pos; + private boolean import_pos_zcurve_before; + private boolean import_pos_zcurve_after; + + public ChildPosSdBuilder() { + import_pos = true; + import_pos_zcurve_before = false; + import_pos_zcurve_after = false; + } + + ChildPosSdBuilder import_pos(boolean v) { import_pos = v; return this; } + ChildPosSdBuilder import_pos_zcurve_before(boolean v) { import_pos_zcurve_before = v; return this; } + ChildPosSdBuilder import_pos_zcurve_after(boolean v) { import_pos_zcurve_after = v; return this; } + + public String build() { + return joinLines("search child {", + " document child {", + " field parent_ref type reference<parent> {", + " indexing: attribute | summary", + " }", + " }", + importPosZCurve(import_pos_zcurve_before), + importPos(import_pos), + importPosZCurve(import_pos_zcurve_after), + "}"); + } + + private static String importPos(boolean doImport) { + return doImport ? "import field parent_ref.pos as my_pos {}" : ""; + } + + private static String importPosZCurve(boolean doImport) { + return doImport ? "import field parent_ref.pos_zcurve as my_pos_zcurve {}" : ""; + } + } + + private static void checkPosImport(ParentPosSdBuilder parentBuilder, ChildPosSdBuilder childBuilder) throws ParseException { + Search search = buildChildSearch(parentBuilder.build(), childBuilder.build()); + assertEquals(1, search.importedFields().get().fields().size()); + assertSearchContainsImportedField("my_pos_zcurve", "parent_ref", "parent", "pos_zcurve", search); + } + + @Test + public void check_pos_import() throws ParseException { + checkPosImport(new ParentPosSdBuilder(), new ChildPosSdBuilder()); + } + + private void expectPosImportFailure() { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For search 'child', import field 'my_pos_zcurve': Field 'pos_zcurve' via reference field 'parent_ref': Field already imported"); + } + + @Test + public void check_pos_import_after_pos_zcurve_import() throws ParseException { + expectPosImportFailure(); + checkPosImport(new ParentPosSdBuilder(), new ChildPosSdBuilder().import_pos_zcurve_before(true)); + } + + @Test + public void check_pos_import_before_pos_zcurve_import() throws ParseException { + expectPosImportFailure(); + checkPosImport(new ParentPosSdBuilder(), new ChildPosSdBuilder().import_pos_zcurve_after(true)); + } + + @Test + public void check_pos_zcurve_import() throws ParseException { + checkPosImport(new ParentPosSdBuilder(), new ChildPosSdBuilder().import_pos(false).import_pos_zcurve_after(true)); + } + + @Test + public void check_pos_zcurve_import_twice() throws ParseException { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For search 'child', import field as 'my_pos_zcurve': Field already imported"); + checkPosImport(new ParentPosSdBuilder(), new ChildPosSdBuilder().import_pos(false).import_pos_zcurve_before(true).import_pos_zcurve_after(true)); + } + private static void assertSearchNotContainsImportedField(String fieldName, Search search) { ImportedField importedField = search.importedFields().get().fields().get(fieldName); assertNull(importedField); |