Merge pull request #10091 from vespa-engine/balder/initialnumdocs-does-include-copies-only-unique-docs

Number of searchable copies are accounted in the backend. Only estima…

4 files changed, 30 insertions, 19 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
index a3099fe3ed4..483e500077d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
@@ -286,6 +286,9 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot
             indexedCluster.setSearchableCopies(redundancy.searchableCopies());
         }
         this.redundancy = redundancy;
+        for (SearchNode node : getSearchNodes()) {
+            node.setSearchableCopies(redundancy.searchableCopies());
+        }
     }
 
     private Optional<StreamingSearchCluster> findStreamingCluster(String docType) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java
index da193a17801..86ea7351092 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java
@@ -16,9 +16,11 @@ public class NodeFlavorTuning implements ProtonConfig.Producer {
     static long MB = 1024 * 1024;
     static long GB = MB * 1024;
     private final Flavor nodeFlavor;
+    private final int searchableCopies;
 
-    public NodeFlavorTuning(Flavor nodeFlavor) {
+    public NodeFlavorTuning(Flavor nodeFlavor, int searchableCopies) {
         this.nodeFlavor = nodeFlavor;
+        this.searchableCopies = searchableCopies;
     }
 
     @Override
@@ -40,7 +42,7 @@ public class NodeFlavorTuning implements ProtonConfig.Producer {
         ProtonConfig.Documentdb dbCfg = builder.build();
         if (dbCfg.mode() != ProtonConfig.Documentdb.Mode.Enum.INDEX) {
             long numDocs = (long)nodeFlavor.getMinMainMemoryAvailableGb()*GB/64L;
-            builder.allocation.initialnumdocs(numDocs);
+            builder.allocation.initialnumdocs(numDocs/searchableCopies);
         }
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
index e255da5b487..64b1b79eb75 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
@@ -54,6 +54,7 @@ public class SearchNode extends AbstractService implements
     private final boolean flushOnShutdown;
     private NodeSpec nodeSpec;
     private int distributionKey;
+    private int searchableCopies = 1;
     private final String clusterName;
     private TransactionLogServer tls;
     private AbstractService serviceLayerService;
@@ -140,6 +141,9 @@ public class SearchNode extends AbstractService implements
     private String getBaseDir() {
         return getDefaults().underVespaHome("var/db/vespa/search/cluster." + getClusterName()) + "/n" + distributionKey;
     }
+    public void setSearchableCopies(int searchableCopies) {
+        this.searchableCopies = searchableCopies;
+    }
 
     public void updatePartition(int partitionId) {
         nodeSpec = new NodeSpec(nodeSpec.groupIndex(), partitionId);
@@ -267,7 +271,7 @@ public class SearchNode extends AbstractService implements
         }
         if (getHostResource() != null && getHostResource().getFlavor().isPresent()) {
             Flavor nodeFlavor = getHostResource().getFlavor().get();
-            NodeFlavorTuning nodeFlavorTuning = new NodeFlavorTuning(nodeFlavor);
+            NodeFlavorTuning nodeFlavorTuning = new NodeFlavorTuning(nodeFlavor, searchableCopies);
             nodeFlavorTuning.getConfig(builder);
 
             if (tuning.isPresent()) {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java
index 90d41de8939..cf77ac94f21 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java
@@ -31,7 +31,7 @@ public class NodeFlavorTuningTest {
         assertEquals(24 * GB, cfg.hwinfo().memory().size());
     }
 
-    private ProtonConfig getProtonMemoryConfig(List<Pair<String, String>> sdAndMode, int gb) {
+    private ProtonConfig getProtonMemoryConfig(List<Pair<String, String>> sdAndMode, int gb, int searchableCopies) {
         ProtonConfig.Builder builder = new ProtonConfig.Builder();
         for (Pair<String, String> sdMode : sdAndMode) {
             builder.documentdb.add(new ProtonConfig.Documentdb.Builder()
@@ -39,19 +39,23 @@ public class NodeFlavorTuningTest {
                                    .configid("some/config/id/" + sdMode.getFirst())
                                    .mode(ProtonConfig.Documentdb.Mode.Enum.valueOf(sdMode.getSecond())));
         }
-        return configFromMemorySetting(gb, builder);
+        return configFromMemorySetting(gb, builder, searchableCopies);
     }
-    @Test
-    public void require_that_initial_numdocs_is_dependent_of_mode() {
-        ProtonConfig cfg = getProtonMemoryConfig(Arrays.asList(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")), 24);
+    private void verify_that_initial_numdocs_is_dependent_of_mode(int searchablecopies) {
+        ProtonConfig cfg = getProtonMemoryConfig(Arrays.asList(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")), 24, searchablecopies);
         assertEquals(3, cfg.documentdb().size());
         assertEquals(1024, cfg.documentdb(0).allocation().initialnumdocs());
         assertEquals("a", cfg.documentdb(0).inputdoctypename());
-        assertEquals(402653184, cfg.documentdb(1).allocation().initialnumdocs());
+        assertEquals(402653184/searchablecopies, cfg.documentdb(1).allocation().initialnumdocs());
         assertEquals("b", cfg.documentdb(1).inputdoctypename());
-        assertEquals(402653184, cfg.documentdb(2).allocation().initialnumdocs());
+        assertEquals(402653184/searchablecopies, cfg.documentdb(2).allocation().initialnumdocs());
         assertEquals("c", cfg.documentdb(2).inputdoctypename());
     }
+    @Test
+    public void require_that_initial_numdocs_is_dependent_of_mode_and_searchablecopies() {
+        verify_that_initial_numdocs_is_dependent_of_mode(1);
+        verify_that_initial_numdocs_is_dependent_of_mode(3);
+    }
 
     @Test
     public void require_that_hwinfo_cpu_cores_is_set() {
@@ -169,9 +173,9 @@ public class NodeFlavorTuningTest {
         return getConfig(new FlavorsConfig.Flavor.Builder().
                 minMainMemoryAvailableGb(memoryGb));
     }
-    private static ProtonConfig configFromMemorySetting(int memoryGb, ProtonConfig.Builder builder) {
+    private static ProtonConfig configFromMemorySetting(int memoryGb, ProtonConfig.Builder builder, int searchableCopies) {
         return getConfig(new FlavorsConfig.Flavor.Builder().
-                minMainMemoryAvailableGb(memoryGb), builder);
+                minMainMemoryAvailableGb(memoryGb), builder, searchableCopies);
     }
 
     private static ProtonConfig configFromNumCoresSetting(double numCores) {
@@ -184,16 +188,14 @@ public class NodeFlavorTuningTest {
     }
 
     private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder) {
-        getConfig(flavorBuilder, new ProtonConfig.Builder());
-        flavorBuilder.name("my_flavor");
-        NodeFlavorTuning tuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)));
-        ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder();
-        tuning.getConfig(protonBuilder);
-        return new ProtonConfig(protonBuilder);
+        return getConfig(flavorBuilder, new ProtonConfig.Builder());
     }
     private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder) {
+        return getConfig(flavorBuilder, protonBuilder, 1);
+    }
+    private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, int searchableCopies) {
         flavorBuilder.name("my_flavor");
-        NodeFlavorTuning tuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)));
+        NodeFlavorTuning tuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)), searchableCopies);
         tuning.getConfig(protonBuilder);
         return new ProtonConfig(protonBuilder);
     }