From a9a169969de7807ab2fad017286dd63855b8602d Mon Sep 17 00:00:00 2001 From: MariusArhaug Date: Tue, 7 May 2024 12:59:07 +0200 Subject: Add additional significance searcher tests --- .../com/yahoo/search/significance/model/docv1.json | 18 +++++++++++++ .../com/yahoo/search/significance/model/docv2.json | 31 ++++++++++++++++++++++ .../com/yahoo/search/significance/model/en.json | 5 ++-- .../test/SignificanceSearcherTest.java | 24 +++++++++++++++-- 4 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 container-search/src/test/java/com/yahoo/search/significance/model/docv1.json create mode 100644 container-search/src/test/java/com/yahoo/search/significance/model/docv2.json (limited to 'container-search') diff --git a/container-search/src/test/java/com/yahoo/search/significance/model/docv1.json b/container-search/src/test/java/com/yahoo/search/significance/model/docv1.json new file mode 100644 index 00000000000..d1fcf5bd411 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/significance/model/docv1.json @@ -0,0 +1,18 @@ +{ + "version" : "1.0", + "id" : "test::1", + "description" : "desc", + "languages" : { + "en": { + "description" : "english model", + "document-count" : 12, + "language" : "en", + "document-frequencies" : { + "usa" : 2, + "hello": 3, + "world": 5, + "test": 2 + } + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/significance/model/docv2.json b/container-search/src/test/java/com/yahoo/search/significance/model/docv2.json new file mode 100644 index 00000000000..7c751544803 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/significance/model/docv2.json @@ -0,0 +1,31 @@ +{ + "version" : "2.0", + "id" : "test::2", + "description" : "desc", + "languages" : { + "en": { + "description" : "english model", + "document-count" : 16, + "document-frequencies" : { + "usa" : 2, + "hello": 3, + "world": 5, + "test": 4, + "additional": 2 + } + }, + "nb": { + "description" : "norwegian model", + "document-count" : 20, + "document-frequencies" : { + "usa" : 2, + "hello": 10, + "verden": 5, + "test": 2, + "norge": 11, + "ja": 12, + "nei": 3 + } + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/significance/model/en.json b/container-search/src/test/java/com/yahoo/search/significance/model/en.json index 04010959a58..a0acd1747ce 100644 --- a/container-search/src/test/java/com/yahoo/search/significance/model/en.json +++ b/container-search/src/test/java/com/yahoo/search/significance/model/en.json @@ -5,13 +5,14 @@ "languages" : { "en": { "description" : "english model", - "document-count" : 10, + "document-count" : 16, "language" : "en", "document-frequencies" : { "usa" : 2, "hello": 3, "world": 5, - "test": 2 + "test": 2, + "additional": 2 } } } diff --git a/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java b/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java index cb5722074ff..2589276a286 100644 --- a/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java @@ -40,8 +40,8 @@ public class SignificanceSearcherTest { public SignificanceSearcherTest() { List models = new ArrayList<>(); - models.add( Path.of("src/test/java/com/yahoo/search/significance/model/en.json")); - + models.add(Path.of("src/test/java/com/yahoo/search/significance/model/docv1.json")); + models.add(Path.of("src/test/java/com/yahoo/search/significance/model/docv2.json")); var schema = new Schema.Builder("music") .add(new DocumentSummary.Builder("default").build()) .add(new RankProfile.Builder("significance-ranking") @@ -61,6 +61,26 @@ public class SignificanceSearcherTest { @Test void testSignificanceValueOnSimpleQuery() { + Query q = new Query(); + q.getRanking().setProfile("significance-ranking"); + AndItem root = new AndItem(); + WordItem tmp; + tmp = new WordItem("hello", true); + root.addItem(tmp); + + q.getModel().getQueryTree().setRoot(root); + + SignificanceModel model = significanceModelRegistry.getModel(Language.ENGLISH).get(); + var helloFrequency = model.documentFrequency("hello"); + var helloSignificanceValue = SignificanceSearcher.calculateIDF(helloFrequency.corpusSize(), helloFrequency.frequency()); + Result r = createExecution(searcher).search(q); + + root = (AndItem) r.getQuery().getModel().getQueryTree().getRoot(); + WordItem w0 = (WordItem) root.getItem(0); + assertEquals(helloSignificanceValue, w0.getSignificance()); + } + @Test + void testSignificanceValueOnSimpleANDQuery() { Query q = new Query(); q.getRanking().setProfile("significance-ranking"); -- cgit v1.2.3 From 394de4700c8501e3f8aad24e2a7fe79ece047f7e Mon Sep 17 00:00:00 2001 From: MariusArhaug Date: Wed, 15 May 2024 11:09:48 +0200 Subject: Update corpus size value in test to match updated docv2.json file --- .../com/yahoo/search/significance/test/SignificanceSearcherTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'container-search') diff --git a/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java b/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java index 2589276a286..eaa66755608 100644 --- a/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java @@ -153,7 +153,7 @@ public class SignificanceSearcherTest { assertEquals(helloSignificanceValue, w0.getSignificance()); assertEquals(testSignificanceValue, w1.getSignificance()); - assertEquals(SignificanceSearcher.calculateIDF(10, 2), w3.getSignificance()); + assertEquals(SignificanceSearcher.calculateIDF(16, 2), w3.getSignificance()); } -- cgit v1.2.3