Properly generate attribute config for streamig search.

6 files changed, 114 insertions, 48 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
index 9f9a93caf97..e2e7276b592 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
@@ -32,13 +32,9 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce
      * Flag indicating if a position-attribute has been found
      */
     private boolean hasPosition = false;
-    private final boolean needOnlyFastAccess;
 
     public AttributeFields(Search search) {
-        this(search, false);
-    }
-    public AttributeFields(Search search, boolean onlyFastAccess) {
-        needOnlyFastAccess = onlyFastAccess;
+
         derive(search);
     }
 
@@ -87,16 +83,13 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce
      */
     private void deriveAttributes(ImmutableSDField field) {
         for (Attribute fieldAttribute : field.getAttributes().values()) {
-            if (fieldAttribute.isFastAccess() || !needOnlyFastAccess) {
-                deriveAttribute(field, fieldAttribute);
-            }
+            deriveAttribute(field, fieldAttribute);
         }
 
         if (field.containsExpression(ToPositionExpression.class)) {
             // TODO: Move this check to processing and remove this
             if (hasPosition) {
-                throw new IllegalArgumentException("Can not specify more than one " +
-                                                   "set of position attributes per " + "field: " + field.getName());
+                throw new IllegalArgumentException("Can not specify more than one set of position attributes per field: " + field.getName());
             }
             hasPosition = true;
         }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
index 108715f96fc..59f595ba842 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
@@ -78,8 +78,8 @@ public class DerivedConfiguration {
                 }
             }
         }
-        attributeFields = new AttributeFields(search, search.isDocumentsOnly());
         if ( ! search.isDocumentsOnly()) {
+            attributeFields = new AttributeFields(search);
             summaries = new Summaries(search, deployLogger);
             summaryMap = new SummaryMap(search, summaries);
             juniperrc = new Juniperrc(search);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
index c7d81d6829a..f5e3bb7e55b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
@@ -106,11 +106,9 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot
         }
 
         private void buildStreamingSearchCluster(ModelElement clusterElem, String clusterName, ContentSearchCluster search, ModelElement docType) {
-            StreamingSearchCluster cluster = new StreamingSearchCluster(search, clusterName + "." + docType.getStringAttribute("type"), 0, clusterName, clusterName);
-
-            List<ModelElement> def = new ArrayList<>();
-            def.add(docType);
-            search.addSearchCluster(cluster, getQueryTimeout(clusterElem), def);
+            String docTypeName = docType.getStringAttribute("type");
+            StreamingSearchCluster cluster = new StreamingSearchCluster(search, clusterName + "." + docTypeName, 0, docTypeName, clusterName);
+            search.addSearchCluster(cluster, getQueryTimeout(clusterElem), Arrays.asList(docType));
         }
 
         private void buildIndexedSearchCluster(ModelElement clusterElem,
@@ -266,6 +264,18 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot
         this.redundancy = redundancy;
     }
 
+    private StreamingSearchCluster findStreamingCluster(String docType) {
+        for (AbstractSearchCluster sc : getClusters().values()) {
+            if (sc instanceof StreamingSearchCluster) {
+                StreamingSearchCluster ssc = (StreamingSearchCluster) sc;
+                if (docType.equals(ssc.getSdConfig().getSearch().getName())) {
+                    return ssc;
+                }
+            }
+        }
+        return null;
+    }
+
     @Override
     public void getConfig(ProtonConfig.Builder builder) {
         double visibilityDelay = hasIndexedCluster() ? getIndexed().getVisibilityDelay() : 0.0;
@@ -276,19 +286,15 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot
                 .configid(getConfigId())
                 .visibilitydelay(visibilityDelay)
                 .global(isGloballyDistributed(type));
-            if (hasIndexedCluster()) {
+            StreamingSearchCluster ssc = findStreamingCluster(docTypeName);
+            if (ssc != null) {
+                ddbB.inputdoctypename(type.getFullName().getName()).configid(ssc.getDocumentDBConfigId());
+            } else if (hasIndexedCluster()) {
                 getIndexed().fillDocumentDBConfig(type.getFullName().getName(), ddbB);
             }
             builder.documentdb(ddbB);
         }
-        for (AbstractSearchCluster sc : getClusters().values()) {
-            if (sc instanceof StreamingSearchCluster) {
-                NewDocumentType type = repo.getDocumentType(((StreamingSearchCluster)sc).getSdConfig().getSearch().getName());
-                ProtonConfig.Documentdb.Builder ddbB = new ProtonConfig.Documentdb.Builder();
-                ddbB.inputdoctypename(type.getFullName().getName()).configid(getConfigId());
-                builder.documentdb(ddbB);
-            }
-        }
+
         int numDocumentDbs = builder.documentdb.size();
         builder.initialize(new ProtonConfig.Initialize.Builder().threads(numDocumentDbs + 1));
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
index a679c2dd5fd..f75d27b4f5b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
@@ -29,17 +29,34 @@ public class StreamingSearchCluster extends SearchCluster implements
         SummaryConfig.Producer
 {
 
+    private class AttributesProducer extends AbstractConfigProducer implements AttributesConfig.Producer {
+        AttributesProducer(AbstractConfigProducer parent, String docType) {
+            super(parent, docType);
+        }
+
+        @Override
+        public void getConfig(AttributesConfig.Builder builder) {
+            StreamingSearchCluster.this.getConfig(builder);
+        }
+    }
+
     private final String storageRouteSpec;
+    private final AttributesProducer attributesConfig;
     private DerivedConfiguration sdConfig = null;
 
-    public StreamingSearchCluster(AbstractConfigProducer parent, String clusterName, int index, String storageClusterName, String storageRouteSpec) {
+    public StreamingSearchCluster(AbstractConfigProducer parent, String clusterName, int index, String docTypeName, String storageRouteSpec) {
         super(parent, clusterName, index);
+        attributesConfig = new AttributesProducer(this, docTypeName);
         this.storageRouteSpec = storageRouteSpec;
     }
 
+    public final String getDocumentDBConfigId() {
+        return attributesConfig.getConfigId();
+    }
     @Override
     protected IndexingMode getIndexingMode() { return IndexingMode.STREAMING; }
     public final String getStorageRouteSpec()       { return storageRouteSpec; }
+    @Override
     public int getRowBits() { return 0; }
 
     @Override
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
index 585c6fe0fb9..29ff68cbac8 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
@@ -14,16 +14,20 @@ import com.yahoo.vespa.model.content.ContentSearchCluster;
 import com.yahoo.vespa.model.search.IndexedSearchCluster;
 import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils;
 import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.xml.sax.SAXException;
 import java.io.IOException;
-import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Collections;
 
 import static org.junit.Assert.assertEquals;
 
-// TODO: Author!
+/**
+ * @author geirst
+ */
 public class DocumentDatabaseTestCase {
 
     private String vespaHosts = "<?xml version='1.0' encoding='utf-8' ?>" +
@@ -33,7 +37,7 @@ public class DocumentDatabaseTestCase {
             "  </host>" +
             "</hosts>";
 
-    private String createVespaServices(List<String> sdNames, String selection, String mode) {
+    private String createVespaServices(List<String> sdNames, String mode) {
         StringBuilder retval = new StringBuilder();
         retval.append("" +
                       "<?xml version='1.0' encoding='utf-8' ?>\n" +
@@ -51,9 +55,7 @@ public class DocumentDatabaseTestCase {
                       "   <redundancy>1</redundancy>\n");
         retval.append("   <documents>\n");
         for (String sdName : sdNames) {
-            retval.append("").append("      <document type='").append(sdName).append("' mode='" + mode + "'");
-            if (selection != null)
-                retval.append(" selection='").append(selection).append("'");
+            retval.append("").append("      <document type='").append(sdName).append("' mode='").append(mode).append("'");
             retval.append("/>\n");
         }
         retval.append("   </documents>\n");
@@ -72,10 +74,10 @@ public class DocumentDatabaseTestCase {
         return new ProtonConfig(pb);
     }
 
-    @Test
-    public void requireThatWeCanHaveOneSearchDefinition() throws IOException, SAXException, ParseException {
+    private void assertSingleSD(String mode) {
         final List<String> sds = Arrays.asList("type1");
-        VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, null, "index"), ApplicationPackageUtils.generateSearchDefinitions(sds)).create();
+        VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode),
+                ApplicationPackageUtils.generateSearchDefinitions(sds)).create();
         IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0);
         ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch();
         assertEquals(1, indexedSearchCluster.getDocumentDbs().size());
@@ -84,10 +86,10 @@ public class DocumentDatabaseTestCase {
         assertEquals(1, proton.documentdb().size());
         assertEquals("type1", proton.documentdb(0).inputdoctypename());
         assertEquals(type1Id, proton.documentdb(0).configid());
-        ProtonConfig nodeCfg = getProtonCfg(contentSearchCluster);
-        assertEquals(1, nodeCfg.documentdb().size());
-        assertEquals("type1", nodeCfg.documentdb(0).inputdoctypename());
-        assertEquals(type1Id, nodeCfg.documentdb(0).configid());
+    }
+    @Test
+    public void requireThatWeCanHaveOneSDForIndexedMode() throws IOException, SAXException, ParseException {
+        assertSingleSD("index");
     }
 
     private void assertDocTypeConfig(VespaModel model, String configId, String indexField, String attributeField) {
@@ -104,7 +106,8 @@ public class DocumentDatabaseTestCase {
     @Test
     public void requireThatWeCanHaveMultipleSearchDefinitions() throws IOException, SAXException, ParseException {
         final List<String> sds = Arrays.asList("type1", "type2", "type3");
-        VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, null, "index"), ApplicationPackageUtils.generateSearchDefinitions(sds)).create();
+        VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, "index"),
+                ApplicationPackageUtils.generateSearchDefinitions(sds)).create();
         IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0);
         ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch();
         String type1Id = "test/search/cluster.test/type1";
@@ -132,10 +135,10 @@ public class DocumentDatabaseTestCase {
             assertEquals("type3", iicfg.indexinfo().get(2).name());
         }
         {
-            AttributesConfig rac1 = model.getConfig(AttributesConfig.class, "test/search/cluster.test/type1");
+            AttributesConfig rac1 = model.getConfig(AttributesConfig.class, type1Id);
             assertEquals(1, rac1.attribute().size());
             assertEquals("f2", rac1.attribute(0).name());
-            AttributesConfig rac2 = model.getConfig(AttributesConfig.class, "test/search/cluster.test/type2");
+            AttributesConfig rac2 = model.getConfig(AttributesConfig.class, type2Id);
             assertEquals(1, rac2.attribute().size());
             assertEquals("f4", rac2.attribute(0).name());
         }
@@ -151,7 +154,8 @@ public class DocumentDatabaseTestCase {
     @Test
     public void requireThatRelevantConfigIsAvailableForClusterSearcher() throws ParseException, IOException, SAXException {
         final List<String> sds = Arrays.asList("type1", "type2");
-        VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, null, "index"), ApplicationPackageUtils.generateSearchDefinitions(sds)).create();
+        VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, "index"),
+                ApplicationPackageUtils.generateSearchDefinitions(sds)).create();
         String searcherId = "container/searchchains/chain/test/component/com.yahoo.prelude.cluster.ClusterSearcher";
 
         { // documentdb-info config
@@ -207,11 +211,10 @@ public class DocumentDatabaseTestCase {
         assertEquals(dynamic, field.dynamic());
     }
 
-
-    @Test
-    public void requireThatConfigIsAvailableForStreaming() throws ParseException, IOException, SAXException {
+    private void assertDocumentDBConfigAvailableForStreaming(String mode) {
         final List<String> sds = Arrays.asList("type");
-        VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts,  createVespaServices(sds, null, "streaming"), ApplicationPackageUtils.generateSearchDefinitions(sds)).create();
+        VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts,  createVespaServices(sds, mode),
+                ApplicationPackageUtils.generateSearchDefinitions(sds)).create();
 
         DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.type");
         assertEquals(1, dcfg.documentdb().size());
@@ -219,4 +222,50 @@ public class DocumentDatabaseTestCase {
         assertEquals("type", db.name());
     }
 
+    @Test
+    public void requireThatDocumentDBConfigIsAvailableForStreaming() throws ParseException, IOException, SAXException {
+        assertDocumentDBConfigAvailableForStreaming("streaming");
+    }
+
+
+    private void assertAttributesConfigIndependentOfMode(String mode, List<String> sds,
+                                                         List<String> documentDBConfigIds,
+                                                         Map<String, List<String>> expectedAttributesMap) {
+        VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts,  createVespaServices(sds, mode),
+                ApplicationPackageUtils.generateSearchDefinitions(sds)).create();
+        ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch();
+
+        ProtonConfig proton = getProtonCfg(contentSearchCluster);
+        assertEquals(sds.size(), proton.documentdb().size());
+        for (int i = 0; i < sds.size(); i++) {
+            assertEquals(sds.get(i), proton.documentdb(i).inputdoctypename());
+            assertEquals(documentDBConfigIds.get(i), proton.documentdb(i).configid());
+            List<String> expectedAttributes = expectedAttributesMap.get(sds.get(i));
+            if (expectedAttributes != null) {
+                AttributesConfig rac1 = model.getConfig(AttributesConfig.class, proton.documentdb(i).configid());
+                assertEquals(expectedAttributes.size(), rac1.attribute().size());
+                for (int j = 0; j < expectedAttributes.size(); j++) {
+                    assertEquals(expectedAttributes.get(j), rac1.attribute(j).name());
+                }
+            }
+        }
+    }
+
+    @Test
+    public void testThatAttributesConfigIsProducedForIndexed() {
+        Map<String, List<String>> expectedAttributesMap = new HashMap<>();
+        expectedAttributesMap.put("type1", Arrays.asList("f2"));
+        assertAttributesConfigIndependentOfMode("index", Arrays.asList("type1"), Arrays.asList("test/search/cluster.test/type1"), expectedAttributesMap);
+    }
+    @Test
+    public void testThatAttributesConfigIsProducedForStreamingForFastAccessFields() {
+        Map<String, List<String>> expectedAttributesMap = new HashMap<>();
+        expectedAttributesMap.put("type1", Arrays.asList("f2"));
+        assertAttributesConfigIndependentOfMode("streaming", Arrays.asList("type1"), Arrays.asList("test/search/cluster.test.type1/type1"), expectedAttributesMap);
+    }
+    @Test
+    public void testThatAttributesConfigIsNotProducedForStoreOnlyEvenForFastAccessFields() {
+        assertAttributesConfigIndependentOfMode("store-only", Arrays.asList("type1"), Arrays.asList("test/search"), Collections.emptyMap());
+    }
+
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java
index b746b14e6f3..f38b7592e31 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java
@@ -27,6 +27,7 @@ public class ApplicationPackageUtils {
                 "    }\n" +
                 "    field " + field2 + " type int {\n" +
                 "      indexing: attribute | summary\n" +
+                "      attribute: fast-access\n" +
                 "      header\n" +
                 "    }\n" +
                 "  }\n" +