Revert "Revert "- initial document count must be number of unique documents per node." MERGEOK"

5 files changed, 55 insertions, 28 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
index 3eb56096e18..0b07ef59410 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
@@ -303,7 +303,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
         if (element == null) {
             searchNode = SearchNode.create(parent, "" + node.getDistributionKey(), node.getDistributionKey(), spec,
                                            clusterName, node, flushOnShutdown, tuning, resourceLimits, deployState.isHosted(),
-                                           fractionOfMemoryReserved, deployState.featureFlags());
+                                           fractionOfMemoryReserved, redundancy, deployState.featureFlags());
             searchNode.setHostResource(node.getHostResource());
             searchNode.initService(deployState);
 
@@ -312,7 +312,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
             tls.initService(deployState);
         } else {
             searchNode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown,
-                                                tuning, resourceLimits, fractionOfMemoryReserved)
+                                                tuning, resourceLimits, fractionOfMemoryReserved, redundancy)
                     .build(deployState, parent, element.getXml());
             tls = new TransactionLogServer.Builder(clusterName, syncTransactionLog).build(deployState, searchNode, element.getXml());
         }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java
index 1ad99404823..02f15ed06cd 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.model.search;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.vespa.config.search.core.ProtonConfig;
 import com.yahoo.vespa.model.Host;
+import com.yahoo.vespa.model.content.Redundancy;
 
 import static java.lang.Long.min;
 import static java.lang.Long.max;
@@ -27,13 +28,16 @@ public class NodeResourcesTuning implements ProtonConfig.Producer {
     private final NodeResources resources;
     private final int threadsPerSearch;
     private final double fractionOfMemoryReserved;
+    private final Redundancy redundancy;
 
     public NodeResourcesTuning(NodeResources resources,
                                int threadsPerSearch,
-                               double fractionOfMemoryReserved) {
+                               double fractionOfMemoryReserved,
+                               Redundancy redundancy) {
         this.resources = resources;
         this.threadsPerSearch = threadsPerSearch;
         this.fractionOfMemoryReserved = fractionOfMemoryReserved;
+        this.redundancy = redundancy;
     }
 
     @Override
@@ -57,7 +61,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer {
         ProtonConfig.Documentdb dbCfg = builder.build();
         if (dbCfg.mode() != ProtonConfig.Documentdb.Mode.Enum.INDEX) {
             long numDocs = (long)usableMemoryGb() * GB / MEMORY_COST_PER_DOCUMENT_STORE_ONLY;
-            builder.allocation.initialnumdocs(numDocs);
+            builder.allocation.initialnumdocs(numDocs/redundancy.readyCopies());
         }
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
index c3de655af92..f238d30308b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
@@ -21,6 +21,7 @@ import com.yahoo.vespa.model.admin.monitoring.Monitoring;
 import com.yahoo.vespa.model.application.validation.RestartConfigs;
 import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
 import com.yahoo.vespa.model.content.ContentNode;
+import com.yahoo.vespa.model.content.Redundancy;
 import com.yahoo.vespa.model.content.ResourceLimits;
 import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer;
 import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProvider;
@@ -69,6 +70,7 @@ public class SearchNode extends AbstractService implements
     private final Optional<Tuning> tuning;
     private final Optional<ResourceLimits> resourceLimits;
     private final double fractionOfMemoryReserved;
+    private final Redundancy redundancy;
 
     public static class Builder extends VespaDomBuilder.DomConfigProducerBuilderBase<SearchNode> {
 
@@ -80,10 +82,11 @@ public class SearchNode extends AbstractService implements
         private final Optional<Tuning> tuning;
         private final Optional<ResourceLimits> resourceLimits;
         private final double fractionOfMemoryReserved;
+        private final Redundancy redundancy;
 
         public Builder(String name, NodeSpec nodeSpec, String clusterName, ContentNode node,
                        boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits,
-                       double fractionOfMemoryReserved) {
+                       double fractionOfMemoryReserved, Redundancy redundancy) {
             this.name = name;
             this.nodeSpec = nodeSpec;
             this.clusterName = clusterName;
@@ -92,23 +95,26 @@ public class SearchNode extends AbstractService implements
             this.tuning = tuning;
             this.resourceLimits = resourceLimits;
             this.fractionOfMemoryReserved = fractionOfMemoryReserved;
+            this.redundancy = redundancy;
         }
 
         @Override
-        protected SearchNode doBuild(DeployState deployState, TreeConfigProducer<AnyConfigProducer> ancestor, Element producerSpec) {
+        protected SearchNode doBuild(DeployState deployState, TreeConfigProducer<AnyConfigProducer> ancestor,
+                                     Element producerSpec) {
             return SearchNode.create(ancestor, name, contentNode.getDistributionKey(), nodeSpec, clusterName, contentNode,
                                      flushOnShutdown, tuning, resourceLimits, deployState.isHosted(),
-                                     fractionOfMemoryReserved, deployState.featureFlags());
+                                     fractionOfMemoryReserved, redundancy, deployState.featureFlags());
         }
 
     }
 
     public static SearchNode create(TreeConfigProducer<?> parent, String name, int distributionKey, NodeSpec nodeSpec,
                                     String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown,
-                                    Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa,
-                                    double fractionOfMemoryReserved, ModelContext.FeatureFlags featureFlags) {
+                                    Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits,
+                                    boolean isHostedVespa, double fractionOfMemoryReserved, Redundancy redundancy,
+                                    ModelContext.FeatureFlags featureFlags) {
         SearchNode node = new SearchNode(parent, name, distributionKey, nodeSpec, clusterName, serviceLayerService, flushOnShutdown,
-                              tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved);
+                              tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved, redundancy);
         if (featureFlags.loadCodeAsHugePages()) {
             node.addEnvironmentVariable("VESPA_LOAD_CODE_AS_HUGEPAGES", true);
         }
@@ -121,7 +127,7 @@ public class SearchNode extends AbstractService implements
     private SearchNode(TreeConfigProducer<?> parent, String name, int distributionKey, NodeSpec nodeSpec,
                        String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown,
                        Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa,
-                       double fractionOfMemoryReserved) {
+                       double fractionOfMemoryReserved, Redundancy redundancy) {
         super(parent, name);
         this.distributionKey = distributionKey;
         this.serviceLayerService = serviceLayerService;
@@ -138,6 +144,7 @@ public class SearchNode extends AbstractService implements
         // Properties are set in DomSearchBuilder
         this.tuning = tuning;
         this.resourceLimits = resourceLimits;
+        this.redundancy = redundancy;
         setPropertiesElastic(clusterName, distributionKey);
         addEnvironmentVariable("OMP_NUM_THREADS", 1);
     }
@@ -279,7 +286,7 @@ public class SearchNode extends AbstractService implements
         if (nodeResources.isPresent()) {
             var nodeResourcesTuning = new NodeResourcesTuning(nodeResources.get(),
                                                               tuning.map(Tuning::threadsPerSearch).orElse(1),
-                                                              fractionOfMemoryReserved);
+                                                              fractionOfMemoryReserved, redundancy);
             nodeResourcesTuning.getConfig(builder);
 
             tuning.ifPresent(t -> t.getConfig(builder));
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java
index 8e719fa90c3..d5215ab5752 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java
@@ -6,9 +6,9 @@ import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provisioning.FlavorsConfig;
 import com.yahoo.vespa.config.search.core.ProtonConfig;
 import com.yahoo.vespa.model.container.ApplicationContainerCluster;
+import com.yahoo.vespa.model.content.Redundancy;
 import org.junit.jupiter.api.Test;
 
-import java.util.Arrays;
 import java.util.List;
 
 import static com.yahoo.vespa.model.Host.memoryOverheadGb;
@@ -22,7 +22,6 @@ import static com.yahoo.vespa.model.search.NodeResourcesTuning.GB;
 public class NodeResourcesTuningTest {
 
     private static final double delta = 0.00001;
-    private static final double combinedFactor = 1 - 18.0/100;
     private static final double DEFAULT_MEMORY_GAIN = 0.08;
 
     @Test
@@ -42,7 +41,7 @@ public class NodeResourcesTuningTest {
         assertEquals(0.7, memoryOverheadGb, delta);
     }
 
-    private ProtonConfig getProtonMemoryConfig(List<Pair<String, String>> sdAndMode, double gb) {
+    private ProtonConfig getProtonMemoryConfig(List<Pair<String, String>> sdAndMode, double gb, Redundancy redundancy) {
         ProtonConfig.Builder builder = new ProtonConfig.Builder();
         for (Pair<String, String> sdMode : sdAndMode) {
             builder.documentdb.add(new ProtonConfig.Documentdb.Builder()
@@ -50,24 +49,25 @@ public class NodeResourcesTuningTest {
                                    .configid("some/config/id/" + sdMode.getFirst())
                                    .mode(ProtonConfig.Documentdb.Mode.Enum.valueOf(sdMode.getSecond())));
         }
-        return configFromMemorySetting(gb, builder);
+        return configFromMemorySetting(gb, builder, redundancy);
     }
 
-    private void verify_that_initial_numdocs_is_dependent_of_mode() {
-        ProtonConfig cfg = getProtonMemoryConfig(Arrays.asList(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")), 24 + memoryOverheadGb);
+    private void verify_that_initial_numdocs_is_dependent_of_mode(int readyCopies) {
+        ProtonConfig cfg = getProtonMemoryConfig(List.of(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")),
+                24 + memoryOverheadGb, new Redundancy(readyCopies+1,readyCopies+1, readyCopies,1, readyCopies));
         assertEquals(3, cfg.documentdb().size());
         assertEquals(1024, cfg.documentdb(0).allocation().initialnumdocs());
         assertEquals("a", cfg.documentdb(0).inputdoctypename());
-        assertEquals(24 * GB / 46, cfg.documentdb(1).allocation().initialnumdocs());
+        assertEquals(24 * GB / (46 * readyCopies), cfg.documentdb(1).allocation().initialnumdocs());
         assertEquals("b", cfg.documentdb(1).inputdoctypename());
-        assertEquals(24 * GB / 46, cfg.documentdb(2).allocation().initialnumdocs());
+        assertEquals(24 * GB / (46 * readyCopies), cfg.documentdb(2).allocation().initialnumdocs());
         assertEquals("c", cfg.documentdb(2).inputdoctypename());
     }
 
     @Test
     void require_that_initial_numdocs_is_dependent_of_mode_and_searchablecopies() {
-        verify_that_initial_numdocs_is_dependent_of_mode();
-
+        verify_that_initial_numdocs_is_dependent_of_mode(1);
+        verify_that_initial_numdocs_is_dependent_of_mode(2);
     }
 
     @Test
@@ -227,9 +227,9 @@ public class NodeResourcesTuningTest {
         return getConfig(new FlavorsConfig.Flavor.Builder().minMainMemoryAvailableGb(memoryGb), fractionOfMemoryReserved);
     }
 
-    private static ProtonConfig configFromMemorySetting(double memoryGb, ProtonConfig.Builder builder) {
+    private static ProtonConfig configFromMemorySetting(double memoryGb, ProtonConfig.Builder builder, Redundancy redundancy) {
         return getConfig(new FlavorsConfig.Flavor.Builder()
-                                 .minMainMemoryAvailableGb(memoryGb), builder);
+                                 .minMainMemoryAvailableGb(memoryGb), builder, redundancy);
     }
 
     private static ProtonConfig configFromNumCoresSetting(double numCores) {
@@ -255,7 +255,11 @@ public class NodeResourcesTuningTest {
     }
 
     private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder) {
-        return getConfig(flavorBuilder, protonBuilder,1);
+        return getConfig(flavorBuilder, protonBuilder, new Redundancy(1,1,1,1,1));
+    }
+    private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder,
+                                          Redundancy redundancy) {
+        return getConfig(flavorBuilder, protonBuilder,1, redundancy);
     }
     private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, double fractionOfMemoryReserved) {
         return getConfig(flavorBuilder, protonBuilder, 1, fractionOfMemoryReserved);
@@ -263,13 +267,24 @@ public class NodeResourcesTuningTest {
 
     private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder,
                                           int numThreadsPerSearch) {
-        return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, 0);
+        return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, new Redundancy(1,1,1,1,1));
+    }
+
+    private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder,
+                                          int numThreadsPerSearch, Redundancy redundancy) {
+        return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, 0, redundancy);
     }
 
     private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder,
                                           int numThreadsPerSearch, double fractionOfMemoryReserved) {
+        return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, fractionOfMemoryReserved,
+                new Redundancy(1,1,1,1,1));
+    }
+    private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder,
+                                          int numThreadsPerSearch, double fractionOfMemoryReserved, Redundancy redundancy) {
         flavorBuilder.name("my_flavor");
-        NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), numThreadsPerSearch, fractionOfMemoryReserved);
+        NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(),
+                numThreadsPerSearch, fractionOfMemoryReserved, redundancy);
         tuning.getConfig(protonBuilder);
         return new ProtonConfig(protonBuilder);
     }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java
index f60441bac03..b357a3e6718 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java
@@ -10,6 +10,7 @@ import com.yahoo.vespa.config.search.core.ProtonConfig;
 import com.yahoo.vespa.defaults.Defaults;
 import com.yahoo.vespa.model.Host;
 import com.yahoo.vespa.model.HostResource;
+import com.yahoo.vespa.model.content.Redundancy;
 import com.yahoo.vespa.model.search.NodeSpec;
 import com.yahoo.vespa.model.search.SearchNode;
 import com.yahoo.vespa.model.search.TransactionLogServer;
@@ -49,7 +50,7 @@ public class SearchNodeTest {
     private static SearchNode createSearchNode(MockRoot root, String name, int distributionKey, NodeSpec nodeSpec,
                                                boolean flushOnShutDown, boolean isHosted, ModelContext.FeatureFlags featureFlags) {
         return SearchNode.create(root, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown,
-                Optional.empty(), Optional.empty(), isHosted, 0.0, featureFlags);
+                Optional.empty(), Optional.empty(), isHosted, 0.0, new Redundancy(1,1,1,1,1), featureFlags);
     }
 
     private static SearchNode createSearchNode(MockRoot root) {