diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2021-04-12 15:30:24 +0200 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2021-04-12 15:30:24 +0200 |
commit | 5f714290d920238819aa55fecd820e006f9e5324 (patch) | |
tree | db121e1e906b6bb77b4dd56450a8081b9dcead1a /config-model | |
parent | 787fa05cebd66820136372164b7e1b2805f9f061 (diff) |
Add cased/uncased to match settings too.
Only allow btree uncase/uncased, and hash:cased/cased for now.
Diffstat (limited to 'config-model')
10 files changed, 139 insertions, 94 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index 4277f503440..65df7f395ae 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -6,6 +6,7 @@ import com.yahoo.document.DataType; import com.yahoo.document.PositionDataType; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.Case; import com.yahoo.searchdefinition.document.Dictionary; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.Ranking; @@ -271,7 +272,7 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce } return AttributesConfig.Attribute.Dictionary.Type.BTREE; } - private static AttributesConfig.Attribute.Dictionary.Match.Enum convert(Dictionary.Match type) { + private static AttributesConfig.Attribute.Dictionary.Match.Enum convert(Case type) { switch (type) { case CASED: return AttributesConfig.Attribute.Dictionary.Match.CASED; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Case.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Case.java new file mode 100644 index 00000000000..cb56b343a86 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Case.java @@ -0,0 +1,9 @@ +package com.yahoo.searchdefinition.document; + +public enum Case { + CASED("cased"), + UNCASED("uncased"); + private String name; + Case(String name) { this.name = name; } + public String getName() { return name;} +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Dictionary.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Dictionary.java index 8f22b344e44..5a25953ce9e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Dictionary.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Dictionary.java @@ -9,9 +9,8 @@ package com.yahoo.searchdefinition.document; */ public class Dictionary { public enum Type { BTREE, HASH, BTREE_AND_HASH }; - public enum Match { CASED, UNCASED }; private Type type = null; - private Match match = null; + private Case casing= null; public void updateType(Type type) { if (this.type == null) { @@ -25,12 +24,12 @@ public class Dictionary { " with current " + type); } } - public void updateMatch(Match match) { - if (this.match != null) { - throw new IllegalArgumentException("dictionary match mode has already been set to " + this.match); + public void updateMatch(Case casing) { + if (this.casing != null) { + throw new IllegalArgumentException("dictionary match mode has already been set to " + this.casing); } - this.match = match; + this.casing = casing; } public Type getType() { return (type != null) ? type : Type.BTREE; } - public Match getMatch() { return (match != null) ? match : Match.UNCASED; } + public Case getMatch() { return (casing != null) ? casing : Case.UNCASED; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java index e10d313bd27..e2c036c2444 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java @@ -35,6 +35,7 @@ public class Matching implements Cloneable, Serializable { } private Type type = Type.TEXT; + private Case casing = Case.UNCASED; /** The basic match algorithm */ private Algorithm algorithm = Algorithm.NORMAL; @@ -59,12 +60,15 @@ public class Matching implements Cloneable, Serializable { } public Type getType() { return type; } + public Case getCase() { return casing; } public void setType(Type type) { this.type = type; typeUserSet = true; } + public void setCase(Case casing) { this.casing = casing; } + public Integer maxLength() { return maxLength; } public Matching maxLength(int maxLength) { this.maxLength = maxLength; return this; } public boolean isTypeUserSet() { return typeUserSet; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index ebfdf6b1f6e..7d87caa68e7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -558,6 +558,17 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, } /** + * Set the matching type for this field and all subfields. + */ + // TODO: When this is not the same as getMatching().setthis we have a potential for inconsistency. Find the right + // Matching object for struct fields as lookup time instead. + public void setMatchingCase(Case casing) { + this.getMatching().setCase(casing); + for (SDField structField : getStructFields()) { + structField.setMatchingCase(casing); + } + } + /** * Set matching algorithm for this field and all subfields. */ // TODO: When this is not the same as getMatching().setthis we have a potential for inconsistency. Find the right diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/DictionaryOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/DictionaryOperation.java index 70b0706df29..12f0509686a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/DictionaryOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/DictionaryOperation.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.fieldoperation; +import com.yahoo.searchdefinition.document.Case; import com.yahoo.searchdefinition.document.Dictionary; import com.yahoo.searchdefinition.document.SDField; @@ -28,10 +29,10 @@ public class DictionaryOperation implements FieldOperation { dictionary.updateType(Dictionary.Type.BTREE); break; case CASED: - dictionary.updateMatch(Dictionary.Match.CASED); + dictionary.updateMatch(Case.CASED); break; case UNCASED: - dictionary.updateMatch(Dictionary.Match.UNCASED); + dictionary.updateMatch(Case.UNCASED); break; default: throw new IllegalArgumentException("Unhandled operation " + operation); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java index 5bfd2c38586..eba56bf964a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.fieldoperation; +import com.yahoo.searchdefinition.document.Case; import com.yahoo.searchdefinition.document.Matching; import com.yahoo.searchdefinition.document.SDField; @@ -10,6 +11,7 @@ import com.yahoo.searchdefinition.document.SDField; public class MatchOperation implements FieldOperation { private Matching.Type matchingType; + private Case casing; private Integer gramSize; private Integer maxLength; private Matching.Algorithm matchingAlgorithm; @@ -34,10 +36,17 @@ public class MatchOperation implements FieldOperation { this.exactMatchTerminator = exactMatchTerminator; } + public void setCase(Case casing) { + this.casing = casing; + } + public void apply(SDField field) { if (matchingType != null) { field.setMatchingType(matchingType); } + if (casing != null) { + field.setMatchingCase(casing); + } if (gramSize != null) { field.getMatching().setGramSize(gramSize); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DictionaryProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DictionaryProcessor.java index 66df78466e6..7b1dfde5cb3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DictionaryProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DictionaryProcessor.java @@ -7,7 +7,9 @@ import com.yahoo.document.PrimitiveDataType; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.Case; import com.yahoo.searchdefinition.document.Dictionary; +import com.yahoo.searchdefinition.document.Matching; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -36,6 +38,19 @@ public class DictionaryProcessor extends Processor { } } else if (attribute.getDataType().getPrimitiveType() == PrimitiveDataType.STRING) { attribute.setDictionary(dictionary); + Matching matching = field.getMatching(); + if (dictionary.getType() == Dictionary.Type.HASH) { + if (dictionary.getMatch() != Case.CASED) { + fail(search, field, "hash dictionary require cased match"); + } + } else { + if (dictionary.getMatch() != Case.UNCASED) { + fail(search, field, "btree dictionary require uncased match"); + } + } + if (! dictionary.getMatch().equals(matching.getCase())) { + fail(search, field, "Dictionary casing '" + dictionary.getMatch() + "' does not match field match casing '" + matching.getCase() + "'"); + } } else { fail(search, field, "You can only specify 'dictionary:' for numeric or string fields"); } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 0e47050d34d..6a4dc6367d1 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -1627,19 +1627,21 @@ Object matchItem(FieldOperationContainer field) : { } Object matchType(FieldOperationContainer container) : { - MatchOperation field = new MatchOperation(); + MatchOperation matchOp = new MatchOperation(); } { - ( <MTOKEN> { field.setMatchingType(Matching.Type.TEXT); } // Deprecated synonym to TEXT - | <TEXT> { field.setMatchingType(Matching.Type.TEXT); } - | <WORD> { field.setMatchingType(Matching.Type.WORD); } - | <EXACT> { field.setMatchingType(Matching.Type.EXACT); } - | <GRAM> { field.setMatchingType(Matching.Type.GRAM); } - | <PREFIX> { field.setMatchingAlgorithm(Matching.Algorithm.PREFIX); } - | <SUBSTRING> { field.setMatchingAlgorithm(Matching.Algorithm.SUBSTRING); } - | <SUFFIX> { field.setMatchingAlgorithm(Matching.Algorithm.SUFFIX); } ) + ( <MTOKEN> { matchOp.setMatchingType(Matching.Type.TEXT); } // Deprecated synonym to TEXT + | <TEXT> { matchOp.setMatchingType(Matching.Type.TEXT); } + | <WORD> { matchOp.setMatchingType(Matching.Type.WORD); } + | <EXACT> { matchOp.setMatchingType(Matching.Type.EXACT); } + | <GRAM> { matchOp.setMatchingType(Matching.Type.GRAM); } + | <CASED> { matchOp.setCase(Case.CASED); } + | <UNCASED> { matchOp.setCase(Case.UNCASED); } + | <PREFIX> { matchOp.setMatchingAlgorithm(Matching.Algorithm.PREFIX); } + | <SUBSTRING> { matchOp.setMatchingAlgorithm(Matching.Algorithm.SUBSTRING); } + | <SUFFIX> { matchOp.setMatchingAlgorithm(Matching.Algorithm.SUFFIX); } ) { - container.addOperation(field); + container.addOperation(matchOp); return null; } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java index 7bda2665272..a94ba54b750 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java @@ -6,7 +6,10 @@ import com.yahoo.config.model.test.TestUtil; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.derived.AttributeFields; +import com.yahoo.searchdefinition.document.Case; import com.yahoo.searchdefinition.document.Dictionary; +import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.searchdefinition.document.Matching; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.config.search.AttributesConfig; import org.junit.Test; @@ -57,10 +60,7 @@ public class DictionaryTestCase { getConfig(search).attribute().get(1).dictionary().match()); } - Search verifyDictionaryControl(Dictionary.Type expected, - AttributesConfig.Attribute.Dictionary.Type.Enum expectedConfig, - String type, - String ... cfg) throws ParseException + Search verifyDictionaryControl(Dictionary.Type expected, String type, String ... cfg) throws ParseException { String def = TestUtil.joinLines( "search test {", @@ -73,137 +73,131 @@ public class DictionaryTestCase { " }", "}"); Search search = createSearch(def); + AttributesConfig.Attribute.Dictionary.Type.Enum expectedConfig = toCfg(expected); assertEquals(expected, search.getAttribute("n1").getDictionary().getType()); assertEquals(expectedConfig, getConfig(search).attribute().get(0).dictionary().type()); return search; } - void verifyStringDictionaryControl(Dictionary.Type expectedType,Dictionary.Match expectedCase, - AttributesConfig.Attribute.Dictionary.Type.Enum expectedTypeCfg, - AttributesConfig.Attribute.Dictionary.Match.Enum expectedCaseCfg, - String type, + AttributesConfig.Attribute.Dictionary.Type.Enum toCfg(Dictionary.Type v) { + return (v == Dictionary.Type.HASH) + ? AttributesConfig.Attribute.Dictionary.Type.Enum.HASH + : (v == Dictionary.Type.BTREE) + ? AttributesConfig.Attribute.Dictionary.Type.Enum.BTREE + : AttributesConfig.Attribute.Dictionary.Type.Enum.BTREE_AND_HASH; + } + AttributesConfig.Attribute.Dictionary.Match.Enum toCfg(Case v) { + return (v == Case.CASED) + ? AttributesConfig.Attribute.Dictionary.Match.Enum.CASED + : AttributesConfig.Attribute.Dictionary.Match.Enum.UNCASED; + } + + void verifyStringDictionaryControl(Dictionary.Type expectedType, Case expectedCase, Case matchCasing, String ... cfg) throws ParseException { - - Search search = verifyDictionaryControl(expectedType, expectedTypeCfg, type, cfg); + Search search = verifyDictionaryControl(expectedType, "string", cfg); + ImmutableSDField f = search.getField("n1"); + AttributesConfig.Attribute.Dictionary.Match.Enum expectedCaseCfg = toCfg(expectedCase); + assertEquals(matchCasing, f.getMatching().getCase()); assertEquals(expectedCase, search.getAttribute("n1").getDictionary().getMatch()); assertEquals(expectedCaseCfg, getConfig(search).attribute().get(0).dictionary().match()); } @Test public void testCasedBtreeSettings() throws ParseException { - verifyDictionaryControl(Dictionary.Type.BTREE, - AttributesConfig.Attribute.Dictionary.Type.BTREE, - "int", - "dictionary:cased"); + verifyDictionaryControl(Dictionary.Type.BTREE, "int", "dictionary:cased"); } @Test public void testNumericBtreeSettings() throws ParseException { - verifyDictionaryControl(Dictionary.Type.BTREE, - AttributesConfig.Attribute.Dictionary.Type.BTREE, - "int", - "dictionary:btree"); + verifyDictionaryControl(Dictionary.Type.BTREE, "int", "dictionary:btree"); } @Test public void testNumericHashSettings() throws ParseException { - verifyDictionaryControl(Dictionary.Type.HASH, - AttributesConfig.Attribute.Dictionary.Type.HASH, - "int", - "dictionary:hash"); + verifyDictionaryControl(Dictionary.Type.HASH, "int", "dictionary:hash"); } @Test public void testNumericBtreeAndHashSettings() throws ParseException { - verifyDictionaryControl(Dictionary.Type.BTREE_AND_HASH, - AttributesConfig.Attribute.Dictionary.Type.BTREE_AND_HASH, - "int", - "dictionary:btree", "dictionary:hash"); + verifyDictionaryControl(Dictionary.Type.BTREE_AND_HASH, "int", "dictionary:btree", "dictionary:hash"); } @Test public void testNumericArrayBtreeAndHashSettings() throws ParseException { - verifyDictionaryControl(Dictionary.Type.BTREE_AND_HASH, - AttributesConfig.Attribute.Dictionary.Type.BTREE_AND_HASH, - "array<int>", - "dictionary:btree", "dictionary:hash"); + verifyDictionaryControl(Dictionary.Type.BTREE_AND_HASH, "array<int>", "dictionary:btree", "dictionary:hash"); } @Test public void testNumericWSetBtreeAndHashSettings() throws ParseException { - verifyDictionaryControl(Dictionary.Type.BTREE_AND_HASH, - AttributesConfig.Attribute.Dictionary.Type.BTREE_AND_HASH, - "weightedset<int>", - "dictionary:btree", "dictionary:hash"); + verifyDictionaryControl(Dictionary.Type.BTREE_AND_HASH, "weightedset<int>", "dictionary:btree", "dictionary:hash"); } @Test public void testStringBtreeSettings() throws ParseException { - verifyStringDictionaryControl(Dictionary.Type.BTREE, Dictionary.Match.UNCASED, - AttributesConfig.Attribute.Dictionary.Type.BTREE, - AttributesConfig.Attribute.Dictionary.Match.UNCASED, - "string", + verifyStringDictionaryControl(Dictionary.Type.BTREE, Case.UNCASED, Case.UNCASED, "dictionary:btree"); } @Test public void testStringBtreeUnCasedSettings() throws ParseException { - verifyStringDictionaryControl(Dictionary.Type.BTREE, Dictionary.Match.UNCASED, - AttributesConfig.Attribute.Dictionary.Type.BTREE, - AttributesConfig.Attribute.Dictionary.Match.UNCASED, - "string", + verifyStringDictionaryControl(Dictionary.Type.BTREE, Case.UNCASED, Case.UNCASED, "dictionary { btree\nuncased\n}"); } @Test public void testStringBtreeCasedSettings() throws ParseException { - verifyStringDictionaryControl(Dictionary.Type.BTREE, Dictionary.Match.CASED, - AttributesConfig.Attribute.Dictionary.Type.BTREE, - AttributesConfig.Attribute.Dictionary.Match.CASED, - "string", - "dictionary { btree\ncased\n}"); + try { + verifyStringDictionaryControl(Dictionary.Type.BTREE, Case.CASED, Case.CASED, + "dictionary { btree\ncased\n}"); + } catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 'n1': btree dictionary require uncased match", e.getMessage()); + } } @Test public void testStringHashSettings() throws ParseException { - verifyStringDictionaryControl(Dictionary.Type.HASH, Dictionary.Match.UNCASED, - AttributesConfig.Attribute.Dictionary.Type.HASH, - AttributesConfig.Attribute.Dictionary.Match.UNCASED, - "string", + try { + verifyStringDictionaryControl(Dictionary.Type.HASH, Case.UNCASED, Case.UNCASED, "dictionary:hash"); + } catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 'n1': hash dictionary require cased match", e.getMessage()); + } } @Test public void testStringHashUnCasedSettings() throws ParseException { - verifyStringDictionaryControl(Dictionary.Type.HASH, Dictionary.Match.UNCASED, - AttributesConfig.Attribute.Dictionary.Type.HASH, - AttributesConfig.Attribute.Dictionary.Match.UNCASED, - "string", - "dictionary { hash\nuncased\n}"); + try { + verifyStringDictionaryControl(Dictionary.Type.HASH, Case.UNCASED, Case.UNCASED, + "dictionary { hash\nuncased\n}"); + } catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 'n1': hash dictionary require cased match", e.getMessage()); + } + } + @Test + public void testStringHashBothCasedSettings() throws ParseException { + verifyStringDictionaryControl(Dictionary.Type.HASH, Case.CASED, Case.CASED, + "dictionary { hash\ncased\n}", "match:cased"); } @Test public void testStringHashCasedSettings() throws ParseException { - verifyStringDictionaryControl(Dictionary.Type.HASH, Dictionary.Match.CASED, - AttributesConfig.Attribute.Dictionary.Type.HASH, - AttributesConfig.Attribute.Dictionary.Match.CASED, - "string", - "dictionary { hash\ncased\n}"); + try { + verifyStringDictionaryControl(Dictionary.Type.HASH, Case.CASED, Case.CASED, + "dictionary { hash\ncased\n}"); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 'n1': Dictionary casing 'CASED' does not match field match casing 'UNCASED'", e.getMessage()); + } } @Test public void testStringBtreeHashSettings() throws ParseException { - verifyStringDictionaryControl(Dictionary.Type.BTREE_AND_HASH, Dictionary.Match.UNCASED, - AttributesConfig.Attribute.Dictionary.Type.BTREE_AND_HASH, - AttributesConfig.Attribute.Dictionary.Match.UNCASED, - "string", + verifyStringDictionaryControl(Dictionary.Type.BTREE_AND_HASH, Case.UNCASED, Case.UNCASED, "dictionary{hash\nbtree\n}"); } @Test public void testStringBtreeHashUnCasedSettings() throws ParseException { - verifyStringDictionaryControl(Dictionary.Type.BTREE_AND_HASH, Dictionary.Match.UNCASED, - AttributesConfig.Attribute.Dictionary.Type.BTREE_AND_HASH, - AttributesConfig.Attribute.Dictionary.Match.UNCASED, - "string", + verifyStringDictionaryControl(Dictionary.Type.BTREE_AND_HASH, Case.UNCASED, Case.UNCASED, "dictionary { hash\nbtree\nuncased\n}"); } @Test public void testStringBtreeHashCasedSettings() throws ParseException { - verifyStringDictionaryControl(Dictionary.Type.BTREE_AND_HASH, Dictionary.Match.CASED, - AttributesConfig.Attribute.Dictionary.Type.BTREE_AND_HASH, - AttributesConfig.Attribute.Dictionary.Match.CASED, - "string", - "dictionary { btree\nhash\ncased\n}"); + try { + verifyStringDictionaryControl(Dictionary.Type.BTREE_AND_HASH, Case.CASED, Case.CASED, + "dictionary { btree\nhash\ncased\n}"); + } catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 'n1': btree dictionary require uncased match", e.getMessage()); + } } @Test public void testNonNumericFieldsFailsDictionaryControl() throws ParseException { |