1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
|
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.schema.derived;
import com.yahoo.config.model.application.provider.MockFileRegistry;
import com.yahoo.config.model.deploy.TestProperties;
import com.yahoo.config.model.test.MockApplicationPackage;
import com.yahoo.document.DataType;
import com.yahoo.documentmodel.NewDocumentReferenceDataType;
import com.yahoo.schema.ApplicationBuilder;
import com.yahoo.schema.Schema;
import com.yahoo.schema.document.Case;
import com.yahoo.schema.document.MatchType;
import com.yahoo.schema.document.Matching;
import com.yahoo.schema.document.SDDocumentType;
import com.yahoo.schema.document.SDField;
import com.yahoo.schema.document.TemporarySDField;
import com.yahoo.schema.parser.ParseException;
import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig;
import org.junit.jupiter.api.Test;
import java.util.HashSet;
import java.util.Set;
import static com.yahoo.config.model.test.TestUtil.joinLines;
import static org.junit.jupiter.api.Assertions.assertEquals;
/**
* @author geirst
*/
public class VsmFieldsTestCase {
static Schema createSchema() {
Schema schema = new Schema("test", MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties());
var sdoc = new SDDocumentType("test");
schema.addDocument(sdoc);
return schema;
}
private static VsmfieldsConfig vsmfieldsConfig(Schema schema) {
VsmFields vsmFields = new VsmFields(schema);
VsmfieldsConfig.Builder cfgBuilder = new VsmfieldsConfig.Builder();
vsmFields.getConfig(cfgBuilder);
return cfgBuilder.build();
}
@Test
void reference_type_field_is_unsearchable() {
Schema schema = createSchema();
SDField field = new TemporarySDField(schema.getDocument(), "ref_field", NewDocumentReferenceDataType.forDocumentName("parent_type"));
field.parseIndexingScript(schema.getName(), "{ summary }");
schema.getDocument().addField(field);
VsmfieldsConfig cfg = vsmfieldsConfig(schema);
assertEquals(1, cfg.fieldspec().size());
VsmfieldsConfig.Fieldspec fieldSpec = cfg.fieldspec().get(0);
assertEquals("ref_field", fieldSpec.name());
assertEquals(VsmfieldsConfig.Fieldspec.Searchmethod.NONE, fieldSpec.searchmethod());
}
private void testIndexMatching(Matching matching, VsmfieldsConfig.Fieldspec.Normalize.Enum normalize, String arg1) {
Schema schema = createSchema();
SDField field = new TemporarySDField(schema.getDocument(), "f", DataType.STRING);
field.parseIndexingScript(schema.getName(), "{ index }");
field.setMatching(matching);
schema.getDocument().addField(field);
VsmfieldsConfig cfg = vsmfieldsConfig(schema);
VsmfieldsConfig.Fieldspec fieldSpec = cfg.fieldspec().get(0);
assertEquals("f", fieldSpec.name());
assertEquals(VsmfieldsConfig.Fieldspec.Searchmethod.AUTOUTF8, fieldSpec.searchmethod());
assertEquals(normalize, fieldSpec.normalize());
assertEquals(arg1, fieldSpec.arg1());
}
@Test
void test_exact_string() {
testIndexMatching(new Matching(MatchType.TEXT),
VsmfieldsConfig.Fieldspec.Normalize.LOWERCASE_AND_FOLD, "");
testIndexMatching(new Matching(MatchType.TEXT).setCase(Case.CASED),
VsmfieldsConfig.Fieldspec.Normalize.NONE, "");
testIndexMatching(new Matching(MatchType.EXACT).setCase(Case.CASED),
VsmfieldsConfig.Fieldspec.Normalize.LOWERCASE, "exact");
testIndexMatching(new Matching(MatchType.WORD),
VsmfieldsConfig.Fieldspec.Normalize.LOWERCASE, "word");
testIndexMatching(new Matching(MatchType.WORD).setCase(Case.CASED),
VsmfieldsConfig.Fieldspec.Normalize.NONE, "word");
}
private static Set<String> getIndexes(VsmfieldsConfig config, String field) {
var indexes = new HashSet<String>();
var doctype = config.documenttype(0);
for (var index : doctype.index()) {
for (var indexField : index.field()) {
if (field.equals(indexField.name())) {
indexes.add(index.name());
break;
}
}
}
return indexes;
}
@Test
void deriveIndexFromNestedAttributes() throws ParseException {
String sd = joinLines(
"schema test {",
" document test {",
" field map_field type map<string,int> {",
" indexing: summary",
" struct-field key { indexing: attribute }",
" struct-field value { indexing: attribute }",
" }",
" }",
"}");
var schema = ApplicationBuilder.createFromString(sd).getSchema();
var config = vsmfieldsConfig(schema);
assertEquals(Set.of("map_field", "map_field.key"), getIndexes(config, "map_field.key"));
assertEquals(Set.of("map_field", "map_field.value"), getIndexes(config, "map_field.value"));
}
@Test
void deriveIndexFromIndexStatement() throws ParseException {
String sd = joinLines(
"schema test {",
" document test {",
" field map_field type map<string,int> {",
" indexing: summary | index",
" }",
" }",
"}");
var schema = ApplicationBuilder.createFromString(sd).getSchema();
var config = vsmfieldsConfig(schema);
assertEquals(Set.of("map_field", "map_field.key"), getIndexes(config, "map_field.key"));
assertEquals(Set.of("map_field", "map_field.value"), getIndexes(config, "map_field.value"));
}
@Test
void positionFieldTypeBlocksderivingOfIndexFromNestedAttributes() throws ParseException {
String sd = joinLines(
"schema test {",
" document test {",
" field pos type position {",
" indexing: attribute | summary",
" struct-field x { indexing: attribute }",
" struct-field y { indexing: attribute }",
" }",
" }",
"}");
var schema = ApplicationBuilder.createFromString(sd).getSchema();
var config = vsmfieldsConfig(schema);
assertEquals(Set.of("pos"), getIndexes(config, "pos"));
assertEquals(Set.of(), getIndexes(config, "pos.x"));
assertEquals(Set.of(), getIndexes(config, "pos.y"));
}
}
|
