diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2018-09-25 11:52:26 +0200 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2018-09-25 11:52:26 +0200 |
commit | 0e14347268b0804a43bcfbe8c4232d57f6712cfd (patch) | |
tree | 2d40b8212b28edf3777a763e4d34735a56016ddd | |
parent | 74ef29c8b1b493a80b4a582acf90b0dbc5953f85 (diff) |
Allow mutable fields to be modifiable in indexing docproc and to have an initial value.
3 files changed, 54 insertions, 26 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java index 3f1d6c55536..6c7396d607b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java @@ -4,6 +4,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.document.DataType; import com.yahoo.document.PositionDataType; import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig.Ilscript.Builder; @@ -22,7 +23,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; -import java.util.LinkedList; import java.util.List; import java.util.Set; @@ -34,19 +34,24 @@ import java.util.Set; */ public final class IndexingScript extends Derived implements IlscriptsConfig.Producer { - private final List<String> docFields = new LinkedList<>(); - private final List<Expression> expressions = new LinkedList<>(); + private final List<String> docFields = new ArrayList<>(); + private final List<Expression> expressions = new ArrayList<>(); public IndexingScript(Search search) { derive(search); } + private boolean hasFullIndexingDocprocRights(ImmutableSDField field) { + Attribute self = field.getAttributes().get(field.getName()); + return (!field.isExtraField() || ((self != null) && self.isMutable())); + } + @Override protected void derive(ImmutableSDField field, Search search) { if (field.isImportedField()) { return; } - if (!field.isExtraField()) { + if (hasFullIndexingDocprocRights(field)) { docFields.add(field.getName()); } if (field.usesStructOrMap() && @@ -75,9 +80,7 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro public void getConfig(IlscriptsConfig.Builder configBuilder) { IlscriptsConfig.Ilscript.Builder ilscriptBuilder = new IlscriptsConfig.Ilscript.Builder(); ilscriptBuilder.doctype(getName()); - for (String fieldName : docFields) { - ilscriptBuilder.docfield(fieldName); - } + ilscriptBuilder.docfield(docFields); addContentInOrder(ilscriptBuilder); configBuilder.ilscript(ilscriptBuilder); } @@ -102,9 +105,9 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro } private void generateSyntheticStatementsForUntouchedFields(Builder ilscriptBuilder, Set<String> touchedFields) { - Set<String> fieldsWithSyntheticStatements = new HashSet<String>(docFields); + Set<String> fieldsWithSyntheticStatements = new HashSet<>(docFields); fieldsWithSyntheticStatements.removeAll(touchedFields); - List<String> orderedFields = new ArrayList<String>(fieldsWithSyntheticStatements); + List<String> orderedFields = new ArrayList<>(fieldsWithSyntheticStatements); Collections.sort(orderedFields); for (String fieldName : orderedFields) { StatementExpression copyField = new StatementExpression(new InputExpression(fieldName), diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java index d7f26cd9851..cadb2cda324 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java @@ -3,11 +3,13 @@ package com.yahoo.searchdefinition; import com.yahoo.document.StructDataType; import com.yahoo.searchdefinition.derived.AttributeFields; +import com.yahoo.searchdefinition.derived.IndexingScript; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.configdefinition.IlscriptsConfig; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -154,24 +156,28 @@ public class AttributeSettingsTestCase extends SearchDefinitionTestCase { assertTrue(attr.isMutable()); } + private Search getSearchWithMutables() throws ParseException { + return getSearch( + "search test {\n" + + " document test { \n" + + " field a type int { \n" + + " indexing: attribute \n" + + " }\n" + + " }\n" + + " field m type long {\n" + + " indexing: 0 | to_long | attribute\n" + + " attribute: mutable\n" + + " }\n" + + " field f type long {\n" + + " indexing: 0 | to_long | attribute\n" + + " }\n" + + "}\n"); + } + @Test - public void requireThatMutableConfigIsProperlyPropagated() throws ParseException{ + public void requireThatMutableConfigIsProperlyPropagated() throws ParseException { - AttributeFields attributes = new AttributeFields(getSearch( - "search test {\n" + - " document test { \n" + - " field a type int { \n" + - " indexing: attribute \n" + - " }\n" + - " }\n" + - " field m type long {\n" + - " indexing: 0 | to_long | attribute\n" + - " attribute: mutable\n" + - " }\n" + - " field f type long {\n" + - " indexing: 0 | to_long | attribute\n" + - " }\n" + - "}\n")); + AttributeFields attributes = new AttributeFields(getSearchWithMutables()); AttributesConfig.Builder builder = new AttributesConfig.Builder(); attributes.getConfig(builder); AttributesConfig cfg = new AttributesConfig(builder); @@ -187,6 +193,21 @@ public class AttributeSettingsTestCase extends SearchDefinitionTestCase { } @Test + public void requireThatMutableIsAllowedThroughIndexing() throws ParseException { + IndexingScript script = new IndexingScript(getSearchWithMutables()); + IlscriptsConfig.Builder builder = new IlscriptsConfig.Builder(); + script.getConfig(builder); + IlscriptsConfig cfg = new IlscriptsConfig(builder); + assertEquals(1, cfg.ilscript().size()); + IlscriptsConfig.Ilscript ils = cfg.ilscript(0); + assertEquals("test", ils.doctype()); + assertEquals(2, ils.docfield().size()); + assertEquals("a", ils.docfield(0)); + assertEquals("m", ils.docfield(1)); + + } + + @Test public void attribute_convert_to_array_copies_internal_state() { StructDataType refType = new StructDataType("my_struct"); Attribute single = new Attribute("foo", Attribute.Type.STRING, Attribute.CollectionType.SINGLE, diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java index f25603deee9..10e3ce561d7 100644 --- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java +++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java @@ -20,7 +20,11 @@ import com.yahoo.document.update.ValueUpdate; import com.yahoo.vespa.indexinglanguage.AdapterFactory; import com.yahoo.vespa.indexinglanguage.expressions.Expression; -import java.util.*; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; /** * @author Simon Thoresen Hult |