Verify that documentdb[].allocation.initialnumdocs and documentdb[].mode is populated according to spec.

6 files changed, 102 insertions, 18 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
index 3e87b4ba627..4c8a086bdcb 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
@@ -321,15 +321,23 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot
                 hasAnyNonIndexedCluster = true;
                 ddbB.inputdoctypename(type.getFullName().getName())
                     .configid(ssc.get().getDocumentDBConfigId())
+                        .mode(ProtonConfig.Documentdb.Mode.Enum.STREAMING)
                     .feeding.concurrency(0.0);
             } else if (hasIndexedCluster()) {
-                getIndexed().fillDocumentDBConfig(type.getFullName().getName(), ddbB);
-                if (tuning != null && tuning.searchNode != null && tuning.searchNode.feeding != null) {
-                    ddbB.feeding.concurrency(tuning.searchNode.feeding.concurrency/2);
+                if (getIndexed().hasDocumentDB(type.getFullName().getName())) {
+                    getIndexed().fillDocumentDBConfig(type.getFullName().getName(), ddbB);
+                    if (tuning != null && tuning.searchNode != null && tuning.searchNode.feeding != null) {
+                        ddbB.feeding.concurrency(tuning.searchNode.feeding.concurrency / 2);
+                    }
+                } else {
+                    hasAnyNonIndexedCluster = true;
+                    ddbB.feeding.concurrency(0.0);
+                    ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STORE_ONLY);
                 }
             } else {
                 hasAnyNonIndexedCluster = true;
                 ddbB.feeding.concurrency(0.0);
+                ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STORE_ONLY);
             }
             if (globalDocType) {
                 ddbB.visibilitydelay(0.0);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
index 80bba10276d..2d11f868f04 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
@@ -309,6 +309,14 @@ public class IndexedSearchCluster extends SearchCluster
     public List<DocumentDatabase> getDocumentDbs() {
         return documentDbs;
     }
+    public boolean hasDocumentDB(String name) {
+        for (DocumentDatabase db : documentDbs) {
+            if (db.getName().equals(name)) {
+                return true;
+            }
+        }
+        return false;
+    }
 
     public void setSearchCoverage(SearchCoverage searchCoverage) {
         this.searchCoverage = searchCoverage;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java
index b845b180548..8cf1db3d6f5 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java
@@ -31,6 +31,17 @@ public class NodeFlavorTuning implements ProtonConfig.Producer {
         tuneSummaryReadIo(builder.summary.read);
         tuneSummaryCache(builder.summary.cache);
         tuneSearchReadIo(builder.search.mmap);
+        for (ProtonConfig.Documentdb.Builder dbb : builder.documentdb) {
+            getConfig(dbb);
+        }
+    }
+
+    private void getConfig(ProtonConfig.Documentdb.Builder builder) {
+        ProtonConfig.Documentdb dbCfg = builder.build();
+        if (dbCfg.mode() != ProtonConfig.Documentdb.Mode.Enum.INDEX) {
+            long numDocs = (long)nodeFlavor.getMinMainMemoryAvailableGb()*GB/40L;
+            builder.allocation.initialnumdocs(numDocs);
+        }
     }
 
     private void tuneSummaryCache(ProtonConfig.Summary.Cache.Builder builder) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
index 5dcf60e5fac..5be51310504 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.model.search;
 import com.yahoo.cloud.config.filedistribution.FiledistributorrpcConfig;
 import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.config.provision.Flavor;
 import com.yahoo.metrics.MetricsmanagerConfig;
 import com.yahoo.searchlib.TranslogserverConfig;
 import com.yahoo.vespa.config.content.LoadTypeConfig;
@@ -260,7 +261,10 @@ public class SearchNode extends AbstractService implements
             builder.pruneremoveddocumentsage(4 * 24 * 3600 + 3600 + 60);
         }
         if (getHostResource() != null && getHostResource().getFlavor().isPresent()) {
-            new NodeFlavorTuning(getHostResource().getFlavor().get()).getConfig(builder);
+            Flavor nodeFlavor = getHostResource().getFlavor().get();
+            NodeFlavorTuning nodeFlavorTuning = new NodeFlavorTuning(nodeFlavor);
+            nodeFlavorTuning.getConfig(builder);
+
             if (tuning.isPresent()) {
                 tuning.get().getConfig(builder);
             }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java
index 95503550767..9af690c9d21 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java
@@ -1,12 +1,17 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model.search;
 
+import com.yahoo.collections.Pair;
 import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provisioning.FlavorsConfig;
 import com.yahoo.vespa.config.search.core.ProtonConfig;
 import org.junit.Test;
 
+import java.util.Arrays;
+import java.util.List;
+
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 import static com.yahoo.vespa.model.search.NodeFlavorTuning.MB;
 import static com.yahoo.vespa.model.search.NodeFlavorTuning.GB;
 
@@ -27,6 +32,29 @@ public class NodeFlavorTuningTest {
         assertEquals(24 * GB, cfg.hwinfo().memory().size());
     }
 
+    private ProtonConfig getProtonMemoryConfig(List<Pair<String, String>> sdAndMode, int gb) {
+        ProtonConfig.Builder builder = new ProtonConfig.Builder();
+        for (Pair<String, String> sdMode : sdAndMode) {
+            builder.documentdb.add(new ProtonConfig.Documentdb.Builder()
+                                   .inputdoctypename(sdMode.getFirst())
+                                   .configid("some/config/id/" + sdMode.getFirst())
+                                   .mode(ProtonConfig.Documentdb.Mode.Enum.valueOf(sdMode.getSecond())));
+        }
+        return configFromMemorySetting(gb, builder);
+    }
+    @Test
+    public void require_that_initial_is_dependent_of_mode() {
+        ProtonConfig cfg = getProtonMemoryConfig(Arrays.asList(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")), 24);
+        assertEquals(1024, cfg.grow().initial());
+        assertEquals(3, cfg.documentdb().size());
+        assertEquals(1024, cfg.documentdb(0).allocation().initialnumdocs());
+        assertEquals("a", cfg.documentdb(0).inputdoctypename());
+        assertEquals(644245094L, cfg.documentdb(1).allocation().initialnumdocs());
+        assertEquals("b", cfg.documentdb(1).inputdoctypename());
+        assertEquals(644245094L, cfg.documentdb(2).allocation().initialnumdocs());
+        assertEquals("c", cfg.documentdb(2).inputdoctypename());
+    }
+
     @Test
     public void require_that_hwinfo_cpu_cores_is_set() {
         ProtonConfig cfg = configFromNumCoresSetting(24);
@@ -143,6 +171,10 @@ public class NodeFlavorTuningTest {
         return getConfig(new FlavorsConfig.Flavor.Builder().
                 minMainMemoryAvailableGb(memoryGb));
     }
+    private static ProtonConfig configFromMemorySetting(int memoryGb, ProtonConfig.Builder builder) {
+        return getConfig(new FlavorsConfig.Flavor.Builder().
+                minMainMemoryAvailableGb(memoryGb), builder);
+    }
 
     private static ProtonConfig configFromNumCoresSetting(double numCores) {
         return getConfig(new FlavorsConfig.Flavor.Builder().minCpuCores(numCores));
@@ -154,11 +186,18 @@ public class NodeFlavorTuningTest {
     }
 
     private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder) {
+        getConfig(flavorBuilder, new ProtonConfig.Builder());
         flavorBuilder.name("my_flavor");
         NodeFlavorTuning tuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)));
         ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder();
         tuning.getConfig(protonBuilder);
         return new ProtonConfig(protonBuilder);
     }
+    private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder) {
+        flavorBuilder.name("my_flavor");
+        NodeFlavorTuning tuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)));
+        tuning.getConfig(protonBuilder);
+        return new ProtonConfig(protonBuilder);
+    }
 
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
index 541ae9112c3..0255e7c531f 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
@@ -70,21 +70,21 @@ public class DocumentDatabaseTestCase {
             retval.append("/>\n");
         }
         retval.append("   </documents>\n");
-        retval.append("" +
-                      "<engine>\n" +
-                "<proton>\n" +
-                "<tuning>\n" +
-                "<searchnode>\n" +
+        retval.append(
+                "    <engine>\n" +
+                "      <proton>\n" +
+                "        <tuning>\n" +
+                "          <searchnode>\n" +
                 xmlTuning +
-                "</searchnode>\n" +
-                "</tuning\n>" +
-                "</proton\n>" +
-                "</engine\n>" +
-                      "   <nodes>\n" +
-                      "      <node hostalias='node0' distribution-key='0'/>\n" +
-                      "   </nodes>\n" +
-                      "</content>\n" +
-                      "</services>\n");
+                "          </searchnode>\n" +
+                "        </tuning\n>" +
+                "      </proton\n>" +
+                "    </engine\n>" +
+                "    <nodes>\n" +
+                "      <node hostalias='node0' distribution-key='0'/>\n" +
+                "    </nodes>\n" +
+                "  </content>\n" +
+                "</services>\n");
         return retval.toString();
     }
 
@@ -162,6 +162,20 @@ public class DocumentDatabaseTestCase {
         }
     }
 
+    @Test
+    public void requireThatModeIsSet() {
+        VespaModel model = createModel(Arrays.asList(new Pair("a", "index"), new Pair("b", "streaming"), new Pair("c", "store-only")), "");
+        ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch();
+        ProtonConfig proton = getProtonCfg(contentSearchCluster);
+        assertEquals(3, proton.documentdb().size());
+        assertEquals(ProtonConfig.Documentdb.Mode.Enum.INDEX, proton.documentdb(0).mode());
+        assertEquals("a", proton.documentdb(0).inputdoctypename());
+        assertEquals(ProtonConfig.Documentdb.Mode.Enum.STREAMING, proton.documentdb(1).mode());
+        assertEquals("b", proton.documentdb(1).inputdoctypename());
+        assertEquals(ProtonConfig.Documentdb.Mode.Enum.STORE_ONLY, proton.documentdb(2).mode());
+        assertEquals("c", proton.documentdb(2).inputdoctypename());
+    }
+
     private void verifyInitialDocumentCount(List<Pair<String, String>> nameAndModes, String xmlTuning, long global, List<Long> local) {
         assertEquals(nameAndModes.size(), local.size());
         VespaModel model = createModel(nameAndModes, xmlTuning);