1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
|
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.search.significance.test;
import com.yahoo.component.chain.Chain;
import com.yahoo.language.Language;
import com.yahoo.language.significance.SignificanceModel;
import com.yahoo.language.significance.SignificanceModelRegistry;
import com.yahoo.language.significance.impl.DefaultSignificanceModelRegistry;
import com.yahoo.prelude.query.AndItem;
import com.yahoo.prelude.query.WordItem;
import com.yahoo.search.Query;
import com.yahoo.search.Result;
import com.yahoo.search.searchchain.Execution;
import com.yahoo.search.significance.SignificanceSearcher;
import org.junit.jupiter.api.Test;
import java.nio.file.Path;
import java.util.HashMap;
import static com.yahoo.test.JunitCompat.assertEquals;
/**
* Tests significance term in the search chain.
*
* @author MariusArhaug
*/
public class SignificanceSearcherTest {
SignificanceModelRegistry significanceModelRegistry;
SignificanceSearcher searcher;
public SignificanceSearcherTest() {
HashMap<Language, Path> map = new HashMap<>();
map.put(Language.ENGLISH, Path.of("src/test/java/com/yahoo/search/significance/model/en.json"));
significanceModelRegistry = new DefaultSignificanceModelRegistry(map);
searcher = new SignificanceSearcher(significanceModelRegistry);
}
private Execution createExecution(SignificanceSearcher searcher) {
return new Execution(new Chain<>(searcher), Execution.Context.createContextStub());
}
private Execution createExecution() {
return new Execution(new Chain<>(), Execution.Context.createContextStub());
}
@Test
void testSignificanceValueOnSimpleQuery() {
Query q = new Query();
AndItem root = new AndItem();
WordItem tmp;
tmp = new WordItem("Hello", true);
root.addItem(tmp);
tmp = new WordItem("world", true);
root.addItem(tmp);
q.getModel().getQueryTree().setRoot(root);
SignificanceModel model = significanceModelRegistry.getModel(Language.ENGLISH).get();
var helloFrequency = model.documentFrequency("Hello");
var helloSignificanceValue = SignificanceSearcher.calculateIDF(helloFrequency.corpusSize(), helloFrequency.frequency());
var worldFrequency = model.documentFrequency("world");
var worldSignificanceValue = SignificanceSearcher.calculateIDF(worldFrequency.corpusSize(), worldFrequency.frequency());
Result r = createExecution(searcher).search(q);
root = (AndItem) r.getQuery().getModel().getQueryTree().getRoot();
WordItem w0 = (WordItem) root.getItem(0);
WordItem w1 = (WordItem) root.getItem(1);
assertEquals(helloSignificanceValue, w0.getSignificance());
assertEquals(worldSignificanceValue, w1.getSignificance());
}
@Test
void testSignificanceValueOnRecursiveQuery() {
Query q = new Query();
AndItem root = new AndItem();
WordItem child1 = new WordItem("hello", true);
AndItem child2 = new AndItem();
WordItem child2_1 = new WordItem("test", true);
AndItem child3 = new AndItem();
AndItem child3_1 = new AndItem();
WordItem child3_1_1 = new WordItem("usa", true);
root.addItem(child1);
root.addItem(child2);
root.addItem(child3);
child2.addItem(child2_1);
child3.addItem(child3_1);
child3_1.addItem(child3_1_1);
q.getModel().getQueryTree().setRoot(root);
SignificanceModel model = significanceModelRegistry.getModel(Language.ENGLISH).get();
var helloFrequency = model.documentFrequency("hello");
var helloSignificanceValue = SignificanceSearcher.calculateIDF(helloFrequency.corpusSize(), helloFrequency.frequency());
var testFrequency = model.documentFrequency("test");
var testSignificanceValue = SignificanceSearcher.calculateIDF(testFrequency.corpusSize(), testFrequency.frequency());
Result r = createExecution(searcher).search(q);
root = (AndItem) r.getQuery().getModel().getQueryTree().getRoot();
WordItem w0 = (WordItem) root.getItem(0);
WordItem w1 = (WordItem) ((AndItem) root.getItem(1)).getItem(0);
WordItem w3 = (WordItem) ((AndItem) ((AndItem) root.getItem(2)).getItem(0)).getItem(0);
assertEquals(helloSignificanceValue, w0.getSignificance());
assertEquals(testSignificanceValue, w1.getSignificance());
assertEquals(SignificanceSearcher.calculateIDF(10, 2), w3.getSignificance());
}
@Test
void testSignificanceValueOnEmptyQuery() {
Query q = new Query();
q.getModel().setLanguage(Language.NORWEGIAN_BOKMAL);
AndItem root = new AndItem();
WordItem tmp;
tmp = new WordItem("Hei", true);
root.addItem(tmp);
tmp = new WordItem("Verden", true);
root.addItem(tmp);
q.getModel().getQueryTree().setRoot(root);
Result r = createExecution(searcher).search(q);
root = (AndItem) r.getQuery().getModel().getQueryTree().getRoot();
WordItem w0 = (WordItem) root.getItem(0);
WordItem w1 = (WordItem) root.getItem(1);
Result r0 = createExecution().search(q);
root = (AndItem) r0.getQuery().getModel().getQueryTree().getRoot();
WordItem w0_0 = (WordItem) root.getItem(0);
WordItem w0_1 = (WordItem) root.getItem(1);
assertEquals(w0_0.getSignificance(), w0.getSignificance());
assertEquals(w0_1.getSignificance(), w1.getSignificance());
}
}
|
