Revert "Revert "Bratseth/phrase segmenting feature flag""

This reverts commit c72315a338a02858c8371d3d70fe091c43b672dd.

2 files changed, 84 insertions, 23 deletions

diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java
index dbeced57c52..aa507d38be5 100644
--- a/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java
@@ -5,6 +5,7 @@ import static org.junit.Assert.*;
 
 import java.io.UnsupportedEncodingException;
 import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 
 import org.junit.After;
 import org.junit.Before;
@@ -23,50 +24,50 @@ import com.yahoo.text.Utf8;
 /**
  * Functional test for InputCheckingSearcher.
  *
- * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @author Steinar Knutsen
  */
 public class InputCheckingSearcherTestCase {
 
     Execution execution;
 
     @Before
-    public void setUp() throws Exception {
+    public void setUp() {
         execution = new Execution(new Chain<Searcher>(new InputCheckingSearcher(MetricReceiver.nullImplementation)),
-                Execution.Context.createContextStub(new IndexFacts()));
+                                  Execution.Context.createContextStub(new IndexFacts()));
     }
 
     @After
-    public void tearDown() throws Exception {
+    public void tearDown() {
         execution = null;
     }
 
     @Test
-    public final void testCommonCase() {
+    public void testCommonCase() {
         Result r = execution.search(new Query("/search/?query=three+blind+mice"));
         assertNull(r.hits().getErrorHit());
     }
 
     @Test
-    public final void candidateButAsciiOnly() {
+    public void candidateButAsciiOnly() {
         Result r = execution.search(new Query("/search/?query=a+a+a+a+a+a"));
         assertNull(r.hits().getErrorHit());
     }
 
     @Test
-    public final void candidateButValid() throws UnsupportedEncodingException {
+    public void candidateButValid() throws UnsupportedEncodingException {
         Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode("å å å å å å", "UTF-8")));
         assertNull(r.hits().getErrorHit());
     }
 
     @Test
-    public final void candidateButValidAndOutsideFirst256() throws UnsupportedEncodingException {
+    public void candidateButValidAndOutsideFirst256() throws UnsupportedEncodingException {
         Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode("œ œ œ œ œ œ", "UTF-8")));
         assertNull(r.hits().getErrorHit());
     }
 
 
     @Test
-    public final void testDoubleEncoded() throws UnsupportedEncodingException {
+    public void testDoubleEncoded() throws UnsupportedEncodingException {
         String rawQuery = "å å å å å å";
         byte[] encodedOnce = Utf8.toBytes(rawQuery);
         char[] secondEncodingBuffer = new char[encodedOnce.length];
@@ -74,33 +75,42 @@ public class InputCheckingSearcherTestCase {
             secondEncodingBuffer[i] = (char) (encodedOnce[i] & 0xFF);
         }
         String query = new String(secondEncodingBuffer);
-        Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode(query, "UTF-8")));
+        Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode(query, StandardCharsets.UTF_8)));
         assertEquals(1, r.hits().getErrorHit().errors().size());
     }
 
     @Test
-    public final void testRepeatedConsecutiveTermsInPhrase() {
-        Result r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.c"));
+    public void testRepeatedConsecutiveTermsInPhrase() {
+        Result r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.c%22"));
         assertNull(r.hits().getErrorHit());
-        r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.0.c"));
+        r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.0.c%22"));
         assertNotNull(r.hits().getErrorHit());
+        assertEquals("More than 5 ocurrences of term '0' in a row detected in phrase : \"a b 0 0 0 0 0 0 c\"",
+                     r.hits().getErrorHit().errorIterator().next().getDetailedMessage());
         r = execution.search(new Query("/search/?query=a.b.0.0.0.1.0.0.0.c"));
         assertNull(r.hits().getErrorHit());
     }
+
     @Test
-    public final void testThatMaxRepeatedConsecutiveTermsInPhraseIs5() {
-        Result r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.c"));
+    public void testThatMaxRepeatedConsecutiveTermsInPhraseIs5() {
+        Result r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.c%22"));
         assertNull(r.hits().getErrorHit());
-        r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.0.c"));
+        r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.0.c%22"));
         assertNotNull(r.hits().getErrorHit());
-        r = execution.search(new Query("/search/?query=a.b.0.0.0.1.0.0.0.c"));
+        assertEquals("More than 5 ocurrences of term '0' in a row detected in phrase : \"a b 0 0 0 0 0 0 c\"",
+                     r.hits().getErrorHit().errorIterator().next().getDetailedMessage());
+        r = execution.search(new Query("/search/?query=%22a.b.0.0.0.1.0.0.0.c%22"));
         assertNull(r.hits().getErrorHit());
     }
+
     @Test
-    public final void testThatMaxRepeatedTermsInPhraseIs10() {
-        Result r = execution.search(new Query("/search/?query=0.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.9.a"));
+    public void testThatMaxRepeatedTermsInPhraseIs10() {
+        Result r = execution.search(new Query("/search/?query=%220.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.9.a%22"));
         assertNull(r.hits().getErrorHit());
-        r = execution.search(new Query("/search/?query=0.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.8.a.9.a.10.a"));
+        r = execution.search(new Query("/search/?query=%220.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.8.a.9.a.10.a%22"));
         assertNotNull(r.hits().getErrorHit());
+        assertEquals("Phrase contains more than 10 occurrences of term 'a' in phrase : \"0 a 1 a 2 a 3 a 4 a 5 a 6 a 7 a 8 a 9 a 10 a\"",
+                     r.hits().getErrorHit().errorIterator().next().getDetailedMessage());
     }
+
 }
diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java
index 34c3da395b7..c831ee29631 100644
--- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java
@@ -16,6 +16,7 @@ import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.IndexModel;
 import com.yahoo.prelude.SearchDefinition;
 import com.yahoo.prelude.query.AndItem;
+import com.yahoo.prelude.query.AndSegmentItem;
 import com.yahoo.prelude.query.CompositeItem;
 import com.yahoo.prelude.query.Highlight;
 import com.yahoo.prelude.query.IndexedItem;
@@ -941,12 +942,12 @@ public class QueryTestCase {
     @Test
     public void testImplicitPhraseIsDefault() {
         Query query = new Query(httpEncode("?query=it's fine"));
-        assertEquals("AND 'it s' fine", query.getModel().getQueryTree().toString());
+        assertEquals("AND (SAND it s) fine", query.getModel().getQueryTree().toString());
     }
 
     @Test
     public void testImplicitPhrase() {
-        Query query = new Query(httpEncode("?query=myfield:it's myfield:a-b myfield:c"));
+        Query query = new Query(httpEncode("?query=myfield:it's myfield:a.b myfield:c"));
 
         SearchDefinition test = new SearchDefinition("test");
         Index myField = new Index("myfield");
@@ -961,7 +962,7 @@ public class QueryTestCase {
 
     @Test
     public void testImplicitAnd() {
-        Query query = new Query(httpEncode("?query=myfield:it's myfield:a-b myfield:c"));
+        Query query = new Query(httpEncode("?query=myfield:it's myfield:a.b myfield:c"));
 
         SearchDefinition test = new SearchDefinition("test");
         Index myField = new Index("myfield");
@@ -972,6 +973,56 @@ public class QueryTestCase {
         query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel))));
 
         assertEquals("AND (SAND myfield:it myfield:s) myfield:a myfield:b myfield:c", query.getModel().getQueryTree().toString());
+        // 'it' and 's' should have connectivity 1
+        AndItem root = (AndItem)query.getModel().getQueryTree().getRoot();
+        AndSegmentItem sand = (AndSegmentItem)root.getItem(0);
+        WordItem it = (WordItem)sand.getItem(0);
+        assertEquals("it", it.getWord());
+        WordItem s = (WordItem)sand.getItem(1);
+        assertEquals("s", s.getWord());
+        assertEquals(s, it.getConnectedItem());
+        assertEquals(1.0, it.getConnectivity(), 0.00000001);
+    }
+
+    @Test
+    public void testImplicitAndConnectivity() {
+        SearchDefinition test = new SearchDefinition("test");
+        Index myField = new Index("myfield");
+        myField.addCommand("phrase-segmenting false");
+        test.addIndex(myField);
+        IndexModel indexModel = new IndexModel(test);
+
+        {
+            Query query = new Query(httpEncode("?query=myfield:b.c.d"));
+            query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel))));
+            assertEquals("AND myfield:b myfield:c myfield:d", query.getModel().getQueryTree().toString());
+            AndItem root = (AndItem) query.getModel().getQueryTree().getRoot();
+            WordItem b = (WordItem) root.getItem(0);
+            WordItem c = (WordItem) root.getItem(1);
+            WordItem d = (WordItem) root.getItem(2);
+            assertEquals(c, b.getConnectedItem());
+            assertEquals(1.0, b.getConnectivity(), 0.00000001);
+            assertEquals(d, c.getConnectedItem());
+            assertEquals(1.0, c.getConnectivity(), 0.00000001);
+        }
+
+        {
+            Query query = new Query(httpEncode("?query=myfield:a myfield:b.c.d myfield:e"));
+            query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel))));
+            assertEquals("AND myfield:a myfield:b myfield:c myfield:d myfield:e", query.getModel().getQueryTree().toString());
+            AndItem root = (AndItem) query.getModel().getQueryTree().getRoot();
+            WordItem a = (WordItem) root.getItem(0);
+            WordItem b = (WordItem) root.getItem(1);
+            WordItem c = (WordItem) root.getItem(2);
+            WordItem d = (WordItem) root.getItem(3);
+            WordItem e = (WordItem) root.getItem(4);
+            assertNull(a.getConnectedItem());
+            assertEquals(c, b.getConnectedItem());
+            assertEquals(1.0, b.getConnectivity(), 0.00000001);
+            assertEquals(d, c.getConnectedItem());
+            assertEquals(1.0, c.getConnectivity(), 0.00000001);
+            assertNull(d.getConnectedItem());
+        }
     }
 
     @Test