Merge pull request #10118 from vespa-engine/balder/redundancy-can-not-be-null

Allow streaming search with no searchable copies too....

5 files changed, 29 insertions, 17 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
index 483e500077d..b324b496b1a 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
@@ -287,6 +287,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot
         }
         this.redundancy = redundancy;
         for (SearchNode node : getSearchNodes()) {
+            node.setRedundancy(redundancy.redundancyFromSearchNodePerspective());
             node.setSearchableCopies(redundancy.searchableCopies());
         }
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java
index b21a0da0d57..09a6f9b5c53 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java
@@ -58,6 +58,7 @@ public class Redundancy implements StorDistributionConfig.Producer, ProtonConfig
     public boolean isEffectivelyGloballyDistributed() {
         return totalNodes == effectiveFinalRedundancy();
     }
+    public int redundancyFromSearchNodePerspective() { return finalRedundancy/explicitGroups; }
     public int searchableCopies() { return readyCopies/(explicitGroups*implicitGroups); }
 
     @Override
@@ -69,7 +70,7 @@ public class Redundancy implements StorDistributionConfig.Producer, ProtonConfig
     @Override
     public void getConfig(ProtonConfig.Builder builder) {
         ProtonConfig.Distribution.Builder distBuilder = new ProtonConfig.Distribution.Builder();
-        distBuilder.redundancy(finalRedundancy/explicitGroups);
+        distBuilder.redundancy(redundancyFromSearchNodePerspective());
         distBuilder.searchablecopies(searchableCopies());
         builder.distribution(distBuilder);
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java
index 86ea7351092..ea28809b289 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java
@@ -16,10 +16,13 @@ public class NodeFlavorTuning implements ProtonConfig.Producer {
     static long MB = 1024 * 1024;
     static long GB = MB * 1024;
     private final Flavor nodeFlavor;
+    private final int redundancy;
     private final int searchableCopies;
 
-    public NodeFlavorTuning(Flavor nodeFlavor, int searchableCopies) {
+
+    public NodeFlavorTuning(Flavor nodeFlavor, int redundancy, int searchableCopies) {
         this.nodeFlavor = nodeFlavor;
+        this.redundancy = redundancy;
         this.searchableCopies = searchableCopies;
     }
 
@@ -42,7 +45,7 @@ public class NodeFlavorTuning implements ProtonConfig.Producer {
         ProtonConfig.Documentdb dbCfg = builder.build();
         if (dbCfg.mode() != ProtonConfig.Documentdb.Mode.Enum.INDEX) {
             long numDocs = (long)nodeFlavor.getMinMainMemoryAvailableGb()*GB/64L;
-            builder.allocation.initialnumdocs(numDocs/searchableCopies);
+            builder.allocation.initialnumdocs(numDocs/Math.max(searchableCopies, redundancy));
         }
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
index 64b1b79eb75..21882ee640a 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
@@ -54,6 +54,7 @@ public class SearchNode extends AbstractService implements
     private final boolean flushOnShutdown;
     private NodeSpec nodeSpec;
     private int distributionKey;
+    private int redundancy = 1;
     private int searchableCopies = 1;
     private final String clusterName;
     private TransactionLogServer tls;
@@ -144,6 +145,9 @@ public class SearchNode extends AbstractService implements
     public void setSearchableCopies(int searchableCopies) {
         this.searchableCopies = searchableCopies;
     }
+    public void setRedundancy(int redundancy) {
+        this.redundancy = redundancy;
+    }
 
     public void updatePartition(int partitionId) {
         nodeSpec = new NodeSpec(nodeSpec.groupIndex(), partitionId);
@@ -271,7 +275,7 @@ public class SearchNode extends AbstractService implements
         }
         if (getHostResource() != null && getHostResource().getFlavor().isPresent()) {
             Flavor nodeFlavor = getHostResource().getFlavor().get();
-            NodeFlavorTuning nodeFlavorTuning = new NodeFlavorTuning(nodeFlavor, searchableCopies);
+            NodeFlavorTuning nodeFlavorTuning = new NodeFlavorTuning(nodeFlavor, redundancy, searchableCopies);
             nodeFlavorTuning.getConfig(builder);
 
             if (tuning.isPresent()) {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java
index cf77ac94f21..b0897c963c0 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java
@@ -31,7 +31,7 @@ public class NodeFlavorTuningTest {
         assertEquals(24 * GB, cfg.hwinfo().memory().size());
     }
 
-    private ProtonConfig getProtonMemoryConfig(List<Pair<String, String>> sdAndMode, int gb, int searchableCopies) {
+    private ProtonConfig getProtonMemoryConfig(List<Pair<String, String>> sdAndMode, int gb, int redundancy, int searchableCopies) {
         ProtonConfig.Builder builder = new ProtonConfig.Builder();
         for (Pair<String, String> sdMode : sdAndMode) {
             builder.documentdb.add(new ProtonConfig.Documentdb.Builder()
@@ -39,22 +39,25 @@ public class NodeFlavorTuningTest {
                                    .configid("some/config/id/" + sdMode.getFirst())
                                    .mode(ProtonConfig.Documentdb.Mode.Enum.valueOf(sdMode.getSecond())));
         }
-        return configFromMemorySetting(gb, builder, searchableCopies);
+        return configFromMemorySetting(gb, builder, redundancy, searchableCopies);
     }
-    private void verify_that_initial_numdocs_is_dependent_of_mode(int searchablecopies) {
-        ProtonConfig cfg = getProtonMemoryConfig(Arrays.asList(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")), 24, searchablecopies);
+    private void verify_that_initial_numdocs_is_dependent_of_mode(int redundancy, int searchablecopies) {
+        int divisor = Math.max(redundancy, searchablecopies);
+        ProtonConfig cfg = getProtonMemoryConfig(Arrays.asList(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")), 24, redundancy, searchablecopies);
         assertEquals(3, cfg.documentdb().size());
         assertEquals(1024, cfg.documentdb(0).allocation().initialnumdocs());
         assertEquals("a", cfg.documentdb(0).inputdoctypename());
-        assertEquals(402653184/searchablecopies, cfg.documentdb(1).allocation().initialnumdocs());
+        assertEquals(402653184/divisor, cfg.documentdb(1).allocation().initialnumdocs());
         assertEquals("b", cfg.documentdb(1).inputdoctypename());
-        assertEquals(402653184/searchablecopies, cfg.documentdb(2).allocation().initialnumdocs());
+        assertEquals(402653184/divisor, cfg.documentdb(2).allocation().initialnumdocs());
         assertEquals("c", cfg.documentdb(2).inputdoctypename());
     }
     @Test
     public void require_that_initial_numdocs_is_dependent_of_mode_and_searchablecopies() {
-        verify_that_initial_numdocs_is_dependent_of_mode(1);
-        verify_that_initial_numdocs_is_dependent_of_mode(3);
+        verify_that_initial_numdocs_is_dependent_of_mode(2,0);
+        verify_that_initial_numdocs_is_dependent_of_mode(1,1);
+        verify_that_initial_numdocs_is_dependent_of_mode(3, 2);
+        verify_that_initial_numdocs_is_dependent_of_mode(3, 3);
     }
 
     @Test
@@ -173,9 +176,9 @@ public class NodeFlavorTuningTest {
         return getConfig(new FlavorsConfig.Flavor.Builder().
                 minMainMemoryAvailableGb(memoryGb));
     }
-    private static ProtonConfig configFromMemorySetting(int memoryGb, ProtonConfig.Builder builder, int searchableCopies) {
+    private static ProtonConfig configFromMemorySetting(int memoryGb, ProtonConfig.Builder builder, int redundancy, int searchableCopies) {
         return getConfig(new FlavorsConfig.Flavor.Builder().
-                minMainMemoryAvailableGb(memoryGb), builder, searchableCopies);
+                minMainMemoryAvailableGb(memoryGb), builder, redundancy, searchableCopies);
     }
 
     private static ProtonConfig configFromNumCoresSetting(double numCores) {
@@ -191,11 +194,11 @@ public class NodeFlavorTuningTest {
         return getConfig(flavorBuilder, new ProtonConfig.Builder());
     }
     private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder) {
-        return getConfig(flavorBuilder, protonBuilder, 1);
+        return getConfig(flavorBuilder, protonBuilder, 1,1);
     }
-    private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, int searchableCopies) {
+    private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, int redundancy, int searchableCopies) {
         flavorBuilder.name("my_flavor");
-        NodeFlavorTuning tuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)), searchableCopies);
+        NodeFlavorTuning tuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)), redundancy, searchableCopies);
         tuning.getConfig(protonBuilder);
         return new ProtonConfig(protonBuilder);
     }