Merge remote-tracking branch 'origin/master' into ogronnesby/quota-decimal

26 files changed, 325 insertions, 238 deletions

diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java
index ea4b6a2b02d..2a798839752 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java
@@ -39,6 +39,8 @@ public interface ConfigChangeAction {
     boolean allowed();
 
     /** Returns whether this change should be ignored for internal redeploy */
-    boolean ignoreForInternalRedeploy();
+    default boolean ignoreForInternalRedeploy() {
+        return false;
+    };
 
 }
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/Quota.java b/config-model-api/src/main/java/com/yahoo/config/model/api/Quota.java
index 3a57e5bb66c..01ed33e023e 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/Quota.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/Quota.java
@@ -18,7 +18,10 @@ import java.util.Optional;
 public class Quota {
     private static final Quota UNLIMITED = new Quota(Optional.empty(), Optional.empty());
 
+    /** The max size of a cluster inside application */
     private final Optional<Integer> maxClusterSize;
+
+    /** The max budget in dollars per hour */
     private final Optional<BigDecimal> budget;
 
     // TODO: Remove constructor once Vespa < 7.300 is gone from production
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj
index bf752b39fa8..3bbcf7979f3 100644
--- a/config-model/src/main/javacc/SDParser.jj
+++ b/config-model/src/main/javacc/SDParser.jj
@@ -233,6 +233,7 @@ TOKEN :
 | < SUFFIX: "suffix" >
 | < CONSTANT: "constant">
 | < ONNXMODEL: "onnx-model">
+| < MODEL: "model" >
 | < RANKPROFILE: "rank-profile" >
 | < RANKDEGRADATIONFREQ: "rank-degradation-frequency" >
 | < RANKDEGRADATION: "rank-degradation" >
@@ -1976,7 +1977,7 @@ void rankProfile(Search search) :
     RankProfile profile;
 }
 {
-    ( <RANKPROFILE> name = identifierWithDash()
+    ( ( <MODEL> | <RANKPROFILE> ) name = identifierWithDash()
         {
             if (documentsOnly) {
                 profile = new DocumentsOnlyRankProfile(name, search, rankProfileRegistry);
@@ -2676,6 +2677,7 @@ String identifier() : { }
       | <QUATERNARY>
       | <QUERYCOMMAND>
       | <RANK>
+      | <MODEL>
       | <RANKPROFILE>
       | <RANKPROPERTIES>
       | <RANKSCOREDROPLIMIT>
diff --git a/config-model/src/test/derived/rankprofiles/rankprofiles.sd b/config-model/src/test/derived/rankprofiles/rankprofiles.sd
index abafee91533..6c3720df589 100644
--- a/config-model/src/test/derived/rankprofiles/rankprofiles.sd
+++ b/config-model/src/test/derived/rankprofiles/rankprofiles.sd
@@ -19,21 +19,21 @@ search rankprofiles {
 
   }
 
-  rank-profile default {
+  model default {
   }
 
-  rank-profile other1 inherits default {
+  model other1 inherits default {
     rank field1: filter
     rank none: filter
   }
 
-  rank-profile other2 inherits other1 {
+  model other2 inherits other1 {
   }
 
-  rank-profile other3 {
+  model other3 {
   }
 
-  rank-profile four {
+  model four {
     match-phase {
       attribute: field2
       order: ascending
@@ -41,7 +41,7 @@ search rankprofiles {
     }
   }
 
-  rank-profile five {
+  model five {
     match-phase {
       attribute: field2
       order: descending
@@ -49,14 +49,14 @@ search rankprofiles {
     }
   }
 
-  rank-profile six {
+  model six {
     match-phase {
       attribute: field3
       max-hits: 666
     }
   }
 
-  rank-profile seven {
+  model seven {
     match-phase {
       attribute: field3
       max-hits:800
@@ -65,7 +65,7 @@ search rankprofiles {
     }
   }
 
-  rank-profile eight inherits seven {
+  model eight inherits seven {
   
   }
 
diff --git a/config-model/src/test/examples/desktop.sd b/config-model/src/test/examples/desktop.sd
deleted file mode 100644
index 82741dbe53f..00000000000
--- a/config-model/src/test/examples/desktop.sd
+++ /dev/null
@@ -1,108 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-# A search definition of medium complexity
-search desktop {
-
-  # A document with some fields.
-  # The fields implicitly defines some indices, summary fields and attributes
-  document desktop inherits product {
-
-    field title type text {
-      mandatory: true
-      indexing: summary | attribute | index_text
-      # index-to: title, default
-
-      rank-weight: 300
-      rank-type: identity
-    }
-
-    field manufacturer type text {
-      indexing: summary | attribute | index
-      # index-to: manufacturer, default
-      alias: producer, brand
-
-      rank-type: identity
-      rank-weight:200
-    }
-
-    field description type text {
-      indexing: summary | index
-
-      rank-type: about
-      rank-weight: 100
-      result-transform: dynamicteaser
-    }
-
-    field category type text {
-      indexing: index
-      # index-to: category, default
-      rank-weight: 50
-    }
-
-    field minprice type int {
-      indexing: summary | attribute | index
-      index-decimals: 2
-
-      rank-type: simple
-      weight: 30
-      staticrankbits: 16
-    }
-
-    field someorder type int {
-      indexing: attribute someorderranking
-      staticrankbits someorderranking: 32
-    }
-
-    # index_url implicitly defines some fields not contained in the document (contexts)
-    # If attributes needs to be set on these, it can be done by explicitly listing
-    # the fields outside documents (show).
-    # I think we should maybe allow setting such field attributes inside the parent
-    # field as well for convenience. Both is shown.
-    field url type url {
-      # Should index mean index_url for url type fields?
-      indexing: summary | index_url
-      parse-mode: url  # Must be default for url types, but shown here anyway
-
-      rank-type: link
-    }
-
-  }
-
-  field category_arr type array<text> {
-    indexing: input category | split ";" | attribute category_arr
-  }
-
-  # Overriding some url field setting from rank-type link
-  field url.host {
-    weight: 1000
-  }
-
-  # Setting an attribute on a non-field index
-  # This is redunant, as default is default
-  index default {
-    default: true
-  }
-
-  # Optionally specify a desire to use a shared dictionary ("catalogs")
-  shared-dictionary: normal, title, manufacturer
-
-  # Optionally set rank values for all indices
-  # Default is the name of the default one
-  # Rank settings from individual fields can be selectively overridden
-  rankprofile default {
-    firstocc-boost text: 200
-  }
-
-  # Another rank profile
-  rankprofile predefinedorder {
-    dynamicranking: off
-    attribute: someorder
-  }
-
-  # Some experimental ranking changes
-  rankprofile experimental inherits default {
-    firstocc-boost text: 300
-  }
-
-}
-
-
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java
index 70a6bbbf659..699fd2c9fe2 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java
@@ -43,7 +43,8 @@ public class QuotaValidatorTest {
             tester.deploy(null, getServices("testCluster", 10), Environment.prod, null);
             fail();
         } catch (RuntimeException e) {
-            assertEquals("Hourly spend for maximum specified resources ($1.60) exceeds budget from quota ($1.00)!", e.getMessage());
+            assertEquals("Hourly spend for maximum specified resources ($-.--) exceeds budget from quota ($-.--)!",
+                         ValidationTester.censorNumbers(e.getMessage()));
         }
     }
 
@@ -63,4 +64,6 @@ public class QuotaValidatorTest {
                 "   </content>" +
                 "</services>";
     }
+
+
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java
index 6961ffa682b..2b3b1a9fcc7 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java
@@ -96,6 +96,8 @@ public class ValidationTester {
         return new Pair<>(newModel, newModelCreator.configChangeActions);
     }
 
-
+    public static String censorNumbers(String s) {
+        return s.replaceAll("\\d", "-");
+    }
 
 }
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java
index fad276379cc..f695f0b75eb 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java
@@ -12,12 +12,11 @@ import java.util.Optional;
 public class NodeResources {
 
     // Standard unit cost in dollars per hour
-    private static final double cpuUnitCost =    0.12;
-    private static final double memoryUnitCost = 0.012;
-    private static final double diskUnitCost =   0.0004;
+    private static final double cpuUnitCost =    0.09;
+    private static final double memoryUnitCost = 0.009;
+    private static final double diskUnitCost =   0.0003;
 
-    // TODO: Remove when models older than 7.226 are gone
-    public static final NodeResources unspecified = new NodeResources(0, 0, 0, 0);
+    private static final NodeResources unspecified = new NodeResources(0, 0, 0, 0);
 
     public enum DiskSpeed {
 
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java
index 5992d47855f..26abe92a6f1 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java
@@ -22,6 +22,7 @@ public class RpcPing implements Pinger, Client.ResponseReceiver {
     private static final Logger log = Logger.getLogger(RpcPing.class.getName());
     private static final String RPC_METHOD = "vespa.searchprotocol.ping";
     private static final CompressionType PING_COMPRESSION = CompressionType.NONE;
+    private static final boolean triggeredClassLoading = ErrorMessage.createBackendCommunicationError("TriggerClassLoading") instanceof ErrorMessage;
 
     private final Node node;
     private final RpcResourcePool resourcePool;
@@ -86,7 +87,7 @@ public class RpcPing implements Pinger, Client.ResponseReceiver {
 
     @Override
     public void receive(ResponseOrError<ProtobufResponse> response) {
-        if (clusterMonitor.isClosed()) return;
+        if (clusterMonitor.isClosed() && ! triggeredClassLoading) return;
         if (node.isLastReceivedPong(pingSequenceId)) {
             pongHandler.handle(toPong(response));
         } else {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json
index b2e634e5d6b..65fa2a4bf70 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json
@@ -12,7 +12,7 @@
           "diskSpeed": "slow",
           "storageType": "remote"
         },
-        "cost": 0.12
+        "cost": "(ignore)"
       },
       "max": {
         "nodes": 2,
@@ -25,7 +25,7 @@
           "diskSpeed": "slow",
           "storageType": "remote"
         },
-        "cost": 0.47
+        "cost": "(ignore)"
       },
       "current": {
         "nodes": 2,
@@ -38,7 +38,7 @@
           "diskSpeed": "slow",
           "storageType": "remote"
         },
-        "cost": 0.24
+        "cost": "(ignore)"
       },
       "target": {
         "nodes": 2,
@@ -51,7 +51,7 @@
           "diskSpeed": "slow",
           "storageType": "remote"
         },
-        "cost": 0.32
+        "cost": "(ignore)"
       }
     }
   }
diff --git a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp
index ccc71fdd0f6..31777e233f6 100644
--- a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp
+++ b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp
@@ -30,11 +30,13 @@
 #include <vespa/eval/eval/tensor_function.h>
 #include <vespa/eval/tensor/default_tensor_engine.h>
 #include <vespa/eval/tensor/default_value_builder_factory.h>
+#include <vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h>
 #include <vespa/vespalib/util/benchmark_timer.h>
 #include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/util/stash.h>
 #include <vespa/vespalib/gtest/gtest.h>
 #include <optional>
+#include <algorithm>
 
 using namespace vespalib;
 using namespace vespalib::eval;
@@ -50,7 +52,11 @@ template <typename T> using CREF = std::reference_wrapper<const T>;
 //-----------------------------------------------------------------------------
 
 struct Impl {
-    virtual const vespalib::string &name() const = 0;
+    size_t order;
+    vespalib::string name;
+    vespalib::string short_name;
+    Impl(size_t order_in, const vespalib::string &name_in, const vespalib::string &short_name_in)
+        : order(order_in), name(name_in), short_name(short_name_in) {}
     virtual Value::UP create_value(const TensorSpec &spec) const = 0;
     virtual TensorSpec create_spec(const Value &value) const = 0;
     virtual Instruction create_join(const ValueType &lhs, const ValueType &rhs, operation::op2_t function, Stash &stash) const = 0;
@@ -59,11 +65,9 @@ struct Impl {
 };
 
 struct ValueImpl : Impl {
-    vespalib::string my_name;
     const ValueBuilderFactory &my_factory;
-    ValueImpl(const vespalib::string &name_in, const ValueBuilderFactory &factory)
-        : my_name(name_in), my_factory(factory) {}
-    const vespalib::string &name() const override { return my_name; }
+    ValueImpl(size_t order_in, const vespalib::string &name_in, const vespalib::string &short_name_in, const ValueBuilderFactory &factory)
+        : Impl(order_in, name_in, short_name_in), my_factory(factory) {}
     Value::UP create_value(const TensorSpec &spec) const override { return value_from_spec(spec, my_factory); }
     TensorSpec create_spec(const Value &value) const override { return spec_from_value(value); }
     Instruction create_join(const ValueType &lhs, const ValueType &rhs, operation::op2_t function, Stash &stash) const override {
@@ -72,11 +76,9 @@ struct ValueImpl : Impl {
 };
 
 struct EngineImpl : Impl {
-    vespalib::string my_name;
     const TensorEngine &my_engine;
-    EngineImpl(const vespalib::string &name_in, const TensorEngine &engine_in)
-        : my_name(name_in), my_engine(engine_in) {}
-    const vespalib::string &name() const override { return my_name; }
+    EngineImpl(size_t order_in, const vespalib::string &name_in, const vespalib::string &short_name_in, const TensorEngine &engine_in)
+        : Impl(order_in, name_in, short_name_in), my_engine(engine_in) {}
     Value::UP create_value(const TensorSpec &spec) const override { return my_engine.from_spec(spec); }
     TensorSpec create_spec(const Value &value) const override { return my_engine.to_spec(value); }
     Instruction create_join(const ValueType &lhs, const ValueType &rhs, operation::op2_t function, Stash &stash) const override {
@@ -91,19 +93,80 @@ struct EngineImpl : Impl {
 
 //-----------------------------------------------------------------------------
 
-EngineImpl  simple_tensor_engine_impl(" [SimpleTensorEngine]", SimpleTensorEngine::ref());
-EngineImpl default_tensor_engine_impl("[DefaultTensorEngine]", DefaultTensorEngine::ref());
-ValueImpl           simple_value_impl("        [SimpleValue]", SimpleValueBuilderFactory::get());
-ValueImpl   default_tensor_value_impl("     [Adaptive Value]", DefaultValueBuilderFactory::get());
+EngineImpl  simple_tensor_engine_impl(4, " SimpleTensorEngine", " SimpleT", SimpleTensorEngine::ref());
+EngineImpl default_tensor_engine_impl(1, "DefaultTensorEngine", "OLD PROD", DefaultTensorEngine::ref());
+ValueImpl           simple_value_impl(3, "        SimpleValue", " SimpleV", SimpleValueBuilderFactory::get());
+ValueImpl    packed_mixed_tensor_impl(2, "  PackedMixedTensor", "  Packed", PackedMixedTensorBuilderFactory::get());
+ValueImpl   default_tensor_value_impl(0, "       DefaultValue", "NEW PROD", DefaultValueBuilderFactory::get());
+vespalib::string                                   short_header("--------");
 
 double budget = 5.0;
 std::vector<CREF<Impl>> impl_list = {simple_tensor_engine_impl,
                                      default_tensor_engine_impl,
                                      simple_value_impl,
+                                     packed_mixed_tensor_impl,
                                      default_tensor_value_impl};
 
 //-----------------------------------------------------------------------------
 
+struct BenchmarkHeader {
+    std::vector<vespalib::string> short_names;
+    BenchmarkHeader() : short_names() {
+        short_names.resize(impl_list.size());
+        for (const Impl &impl: impl_list) {
+            short_names[impl.order] = impl.short_name;
+        }
+    }
+    void print_trailer() const {
+        for (size_t i = 0; i < short_names.size(); ++i) {
+            fprintf(stderr, "+%s", short_header.c_str());
+        }
+        fprintf(stderr, "+------------------------------------------------\n");
+    }
+    void print() const {
+        for (const auto &name: short_names) {
+            fprintf(stderr, "|%s", name.c_str());
+        }
+        fprintf(stderr, "| Benchmark description\n");
+        print_trailer();
+    }
+};
+
+struct BenchmarkResult {
+    vespalib::string desc;
+    std::optional<double> ref_time;
+    std::vector<double> relative_perf;
+    BenchmarkResult(const vespalib::string &desc_in, size_t num_values)
+        : desc(desc_in), ref_time(std::nullopt), relative_perf(num_values, 0.0) {}
+    ~BenchmarkResult();
+    void sample(size_t order, double time) {
+        relative_perf[order] = time;
+        if (order == 1) {
+            if (ref_time.has_value()) {
+                ref_time = std::min(ref_time.value(), time);
+            } else {
+                ref_time = time;
+            }
+        }
+    }
+    void normalize() {
+        for (double &perf: relative_perf) {
+            perf = ref_time.value() / perf;
+        }
+    }
+    void print() const {
+        for (double perf: relative_perf) {
+            fprintf(stderr, "|%8.2f", perf);
+        }
+        fprintf(stderr, "| %s\n", desc.c_str());
+    }
+};
+BenchmarkResult::~BenchmarkResult() = default;
+
+std::vector<BenchmarkResult> benchmark_results;
+
+//-----------------------------------------------------------------------------
+
 struct EvalOp {
     using UP = std::unique_ptr<EvalOp>;
     const Impl              &impl;
@@ -142,9 +205,14 @@ void benchmark(const vespalib::string &desc, const std::vector<EvalOp::UP> &list
             expect = eval->result();
         }
     }
+    BenchmarkResult result(desc, list.size());
     for (const auto &eval: list) {
-        fprintf(stderr, "    %s: %10.3f us\n", eval->impl.name().c_str(), eval->estimate_cost_us());
+        double time = eval->estimate_cost_us();
+        result.sample(eval->impl.order, time);
+        fprintf(stderr, "    %s(%s): %10.3f us\n", eval->impl.name.c_str(), eval->impl.short_name.c_str(), time);
     }
+    result.normalize();
+    benchmark_results.push_back(result);
     fprintf(stderr, "--------------------------------------------------------\n");
 }
 
@@ -318,4 +386,15 @@ TEST(MixedJoin, no_overlap) {
 
 //-----------------------------------------------------------------------------
 
+TEST(PrintResults, print_results) {
+    BenchmarkHeader header;
+    std::sort(benchmark_results.begin(), benchmark_results.end(),
+              [](const auto &a, const auto &b){ return (a.relative_perf[0] < b.relative_perf[0]); });
+    header.print();
+    for (const auto &result: benchmark_results) {
+        result.print();
+    }
+    header.print_trailer();
+}
+
 GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/eval/src/vespa/eval/eval/value.h b/eval/src/vespa/eval/eval/value.h
index 20923fcd621..a084d267cec 100644
--- a/eval/src/vespa/eval/eval/value.h
+++ b/eval/src/vespa/eval/eval/value.h
@@ -100,6 +100,23 @@ public:
 };
 
 /**
+ * A generic value without any mapped dimensions referencing its
+ * components without owning anything.
+ **/
+class DenseValueView final : public Value
+{
+private:
+    const ValueType &_type;
+    TypedCells _cells;
+public:
+    DenseValueView(const ValueType &type_in, TypedCells cells_in)
+        : _type(type_in), _cells(cells_in) {}
+    const ValueType &type() const final override { return _type; }
+    TypedCells cells() const final override { return _cells; }
+    const Index &index() const final override { return TrivialIndex::get(); }
+};
+
+/**
  * Tagging interface used as return type from factories before
  * downcasting to actual builder with specialized cell type.
  **/
@@ -162,3 +179,4 @@ protected:
 }
 
 VESPA_CAN_SKIP_DESTRUCTION(::vespalib::eval::DoubleValue);
+VESPA_CAN_SKIP_DESTRUCTION(::vespalib::eval::DenseValueView);
diff --git a/eval/src/vespa/eval/instruction/generic_join.cpp b/eval/src/vespa/eval/instruction/generic_join.cpp
index 8a1a199effa..b54f7d8952a 100644
--- a/eval/src/vespa/eval/instruction/generic_join.cpp
+++ b/eval/src/vespa/eval/instruction/generic_join.cpp
@@ -10,6 +10,7 @@
 
 namespace vespalib::eval::instruction {
 
+using TypedCells = Value::TypedCells;
 using State = InterpretedFunction::State;
 using Instruction = InterpretedFunction::Instruction;
 
@@ -95,13 +96,8 @@ struct SparseJoinState {
 };
 SparseJoinState::~SparseJoinState() = default;
 
-/*
 template <typename LCT, typename RCT, typename OCT, typename Fun>
-void generic_join()
-*/
-
-template <typename LCT, typename RCT, typename OCT, typename Fun>
-void my_generic_join_op(State &state, uint64_t param_in) {
+void my_mixed_join_op(State &state, uint64_t param_in) {
     const auto &param = unwrap_param<JoinParam>(param_in);
     Fun fun(param.function);
     const Value &lhs = state.peek(1);
@@ -126,9 +122,39 @@ void my_generic_join_op(State &state, uint64_t param_in) {
     state.pop_pop_push(result_ref);
 };
 
+template <typename LCT, typename RCT, typename OCT, typename Fun>
+void my_dense_join_op(State &state, uint64_t param_in) {
+    const auto &param = unwrap_param<JoinParam>(param_in);
+    Fun fun(param.function);
+    auto lhs_cells = state.peek(1).cells().typify<LCT>();
+    auto rhs_cells = state.peek(0).cells().typify<RCT>();
+    ArrayRef<OCT> out_cells = state.stash.create_array<OCT>(param.dense_plan.out_size);
+    OCT *dst = out_cells.begin();
+    auto join_cells = [&](size_t lhs_idx, size_t rhs_idx) { *dst++ = fun(lhs_cells[lhs_idx], rhs_cells[rhs_idx]); };
+    param.dense_plan.execute(0, 0, join_cells);
+    state.pop_pop_push(state.stash.create<DenseValueView>(param.res_type, TypedCells(out_cells)));
+};
+
+template <typename Fun>
+void my_double_join_op(State &state, uint64_t param_in) {
+    Fun fun(unwrap_param<JoinParam>(param_in).function);
+    state.pop_pop_push(state.stash.create<DoubleValue>(fun(state.peek(1).cells().typify<double>()[0],
+                                                           state.peek(0).cells().typify<double>()[0])));
+};
+
 struct SelectGenericJoinOp {
-    template <typename LCT, typename RCT, typename OCT, typename Fun> static auto invoke() {
-        return my_generic_join_op<LCT,RCT,OCT,Fun>;
+    template <typename LCT, typename RCT, typename OCT, typename Fun> static auto invoke(const JoinParam &param) {
+        if (param.res_type.is_double()) {
+            bool all_double = (std::is_same_v<LCT, double> &&
+                               std::is_same_v<RCT, double> &&
+                               std::is_same_v<OCT, double>);
+            assert(all_double);
+            return my_double_join_op<Fun>;
+        }
+        if (param.sparse_plan.sources.empty()) {
+            return my_dense_join_op<LCT,RCT,OCT,Fun>;
+        }
+        return my_mixed_join_op<LCT,RCT,OCT,Fun>;
     }
 };
 
@@ -225,7 +251,7 @@ GenericJoin::make_instruction(const ValueType &lhs_type, const ValueType &rhs_ty
                               const ValueBuilderFactory &factory, Stash &stash)
 {
     auto &param = stash.create<JoinParam>(lhs_type, rhs_type, function, factory);
-    auto fun = typify_invoke<4,JoinTypify,SelectGenericJoinOp>(lhs_type.cell_type(), rhs_type.cell_type(), param.res_type.cell_type(), function);
+    auto fun = typify_invoke<4,JoinTypify,SelectGenericJoinOp>(lhs_type.cell_type(), rhs_type.cell_type(), param.res_type.cell_type(), function, param);
     return Instruction(fun, wrap_param<JoinParam>(param));
 }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
index b41b7f15499..5df45bbc1b1 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
@@ -79,9 +79,9 @@ public abstract class ApplicationMaintainer extends NodeRepositoryMaintainer {
      * @return whether it was successfully deployed
      */
     protected final boolean deployWithLock(ApplicationId application) {
-        if ( ! canDeployNow(application)) return false; // redeployment is no longer needed
         try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, metric, nodeRepository())) {
             if ( ! deployment.isValid()) return false; // this will be done at another config server
+            if ( ! canDeployNow(application)) return false; // redeployment is no longer needed
             log.log(Level.INFO, application + " will be deployed, last deploy time " + getLastDeployTime(application));
             return deployment.activate().isPresent();
         } finally {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
index 77ef88f0952..2ce3fa6c0f6 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.provision.maintenance;
 
 import com.google.common.util.concurrent.UncheckedTimeoutException;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Deployer;
 import com.yahoo.config.provision.Deployment;
 import com.yahoo.config.provision.HostLivenessTracker;
@@ -187,27 +188,29 @@ public class NodeFailer extends NodeRepositoryMaintainer {
      * Otherwise we remove any "down" history record.
      */
     private void updateNodeDownState() {
-        Map<String, Node> activeNodesByHostname = nodeRepository().getNodes(Node.State.active).stream()
-                .collect(Collectors.toMap(Node::hostname, node -> node));
-
-        serviceMonitor.getServiceModelSnapshot().getServiceInstancesByHostName()
-                .forEach((hostName, serviceInstances) -> {
-                    Node node = activeNodesByHostname.get(hostName.s());
-                    if (node == null) return;
-                    try (var lock = nodeRepository().lock(node.allocation().get().owner())) {
-                        Optional<Node> currentNode = nodeRepository().getNode(node.hostname(), Node.State.active); // re-get inside lock
-                        if (currentNode.isEmpty()) return; // Node disappeared since acquiring lock
-                        node = currentNode.get();
-                        if (badNode(serviceInstances)) {
-                            recordAsDown(node, lock);
-                        } else {
-                            clearDownRecord(node, lock);
-                        }
-                    }
-                    catch (UncheckedTimeoutException e) {
-                        // Ignore - node may be locked on this round due to deployment
-                    }
-                });
+        NodeList activeNodes = NodeList.copyOf(nodeRepository().getNodes(Node.State.active));
+        serviceMonitor.getServiceModelSnapshot().getServiceInstancesByHostName().forEach((hostname, serviceInstances) -> {
+            Optional<Node> node = activeNodes.matching(n -> n.hostname().equals(hostname.toString())).first();
+            if (node.isEmpty()) return;
+
+            // Already correct record, nothing to do
+            boolean badNode = badNode(serviceInstances);
+            if (badNode == node.get().history().event(History.Event.Type.down).isPresent()) return;
+
+            // Lock and update status
+            ApplicationId owner = node.get().allocation().get().owner();
+            try (var lock = nodeRepository().lock(owner, Duration.ofSeconds(1))) {
+                node = getNode(hostname.toString(), owner, lock); // Re-get inside lock
+                if (node.isEmpty()) return; // Node disappeared or changed allocation
+                if (badNode) {
+                    recordAsDown(node.get(), lock);
+                } else {
+                    clearDownRecord(node.get(), lock);
+                }
+            } catch (UncheckedTimeoutException ignored) {
+                // Fine, we'll try updating this node in the next run
+            }
+        });
     }
 
     private Map<Node, String> getActiveNodesByFailureReason(List<Node> activeNodes) {
@@ -248,6 +251,13 @@ public class NodeFailer extends NodeRepositoryMaintainer {
         return reasonsToFailParentHost(hostNode).size() > 0;
     }
 
+    /** Get node by given hostname and application. The applicationLock must be held when calling this */
+    private Optional<Node> getNode(String hostname, ApplicationId application, @SuppressWarnings("unused") Mutex applicationLock) {
+        return nodeRepository().getNode(hostname, Node.State.active)
+                               .filter(node -> node.allocation().isPresent())
+                               .filter(node -> node.allocation().get().owner().equals(application));
+    }
+
     private boolean expectConfigRequests(Node node) {
         return !node.type().isHost();
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
index 75f3c892571..b2cb7f545ff 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
@@ -28,6 +28,8 @@ import java.util.stream.Collectors;
  */
 public class GroupPreparer {
 
+    private static final Mutex PROBE_LOCK = () -> {};
+
     private final NodeRepository nodeRepository;
     private final Optional<HostProvisioner> hostProvisioner;
     private final ListFlag<HostCapacity> preprovisionCapacityFlag;
@@ -59,28 +61,26 @@ public class GroupPreparer {
                               List<Node> surplusActiveNodes, MutableInteger highestIndex, int wantedGroups) {
         boolean dynamicProvisioningEnabled = nodeRepository.canProvisionHosts() && nodeRepository.zone().getCloud().dynamicProvisioning();
         boolean allocateFully = dynamicProvisioningEnabled && preprovisionCapacityFlag.value().isEmpty();
-        try (Mutex lock = nodeRepository.lock(application)) {
 
-            // Lock ready pool to ensure that the same nodes are not simultaneously allocated by others
-            try (Mutex allocationLock = nodeRepository.lockUnallocated()) {
+        // Try preparing in memory without lock. Most of the time there should be no changes and we can return nodes
+        // previously allocated.
+        {
+            MutableInteger probePrepareHighestIndex = new MutableInteger(highestIndex.get());
+            NodeAllocation probeAllocation = prepareAllocation(application, cluster, requestedNodes, surplusActiveNodes,
+                    probePrepareHighestIndex, wantedGroups, allocateFully, PROBE_LOCK);
+            if (probeAllocation.fulfilledAndNoChanges()) {
+                List<Node> acceptedNodes = probeAllocation.finalNodes();
+                surplusActiveNodes.removeAll(acceptedNodes);
+                highestIndex.set(probePrepareHighestIndex.get());
+                return acceptedNodes;
+            }
+        }
 
-                // Create a prioritized set of nodes
-                LockedNodeList allNodes = nodeRepository.list(allocationLock);
-                NodeAllocation allocation = new NodeAllocation(allNodes, application, cluster, requestedNodes,
-                                                               highestIndex,  nodeRepository);
-
-                NodePrioritizer prioritizer = new NodePrioritizer(allNodes,
-                                                                  application,
-                                                                  cluster,
-                                                                  requestedNodes,
-                                                                  wantedGroups,
-                                                                  allocateFully,
-                                                                  nodeRepository);
-                prioritizer.addApplicationNodes();
-                prioritizer.addSurplusNodes(surplusActiveNodes);
-                prioritizer.addReadyNodes();
-                prioritizer.addNewDockerNodes();
-                allocation.offer(prioritizer.prioritize());
+        // There were some changes, so re-do the allocation with locks
+        try (Mutex lock = nodeRepository.lock(application)) {
+            try (Mutex allocationLock = nodeRepository.lockUnallocated()) {
+                NodeAllocation allocation = prepareAllocation(application, cluster, requestedNodes, surplusActiveNodes,
+                        highestIndex, wantedGroups, allocateFully, allocationLock);
 
                 if (dynamicProvisioningEnabled) {
                     Version osVersion = nodeRepository.osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion);
@@ -122,4 +122,22 @@ public class GroupPreparer {
         }
     }
 
+    private NodeAllocation prepareAllocation(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes,
+                                           List<Node> surplusActiveNodes, MutableInteger highestIndex, int wantedGroups,
+                                           boolean allocateFully, Mutex allocationLock) {
+        LockedNodeList allNodes = nodeRepository.list(allocationLock);
+        NodeAllocation allocation = new NodeAllocation(allNodes, application, cluster, requestedNodes,
+                highestIndex, nodeRepository);
+        NodePrioritizer prioritizer = new NodePrioritizer(allNodes,
+                application, cluster, requestedNodes, wantedGroups, allocateFully, nodeRepository);
+
+        prioritizer.addApplicationNodes();
+        prioritizer.addSurplusNodes(surplusActiveNodes);
+        prioritizer.addReadyNodes();
+        prioritizer.addNewDockerNodes();
+        allocation.offer(prioritizer.prioritize());
+
+        return allocation;
+    }
+
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
index b07ce786685..0ebcb34703f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
@@ -305,6 +305,11 @@ class NodeAllocation {
         return requestedNodes.fulfilledBy(accepted);
     }
 
+    /** Returns true this allocation was already fulfilled and resulted in no new changes */
+    public boolean fulfilledAndNoChanges() {
+        return fulfilled() && reservableNodes().isEmpty() && newNodes().isEmpty();
+    }
+
     /**
      * Returns {@link FlavorCount} describing the docker node deficit for the given {@link NodeSpec}.
      *
diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp
index bd7feec0598..126a7afed4d 100644
--- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp
+++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp
@@ -209,6 +209,10 @@ Domain::getSynced() const
 void
 Domain::triggerSyncNow()
 {
+    {
+        vespalib::MonitorGuard guard(_currentChunkMonitor);
+        commitAndTransferResponses(guard);
+    }
     MonitorGuard guard(_syncMonitor);
     if (!_pendingSync) {
         _pendingSync = true;
@@ -352,12 +356,17 @@ Domain::startCommit(DoneCallback onDone) {
 void
 Domain::commitIfFull(const vespalib::MonitorGuard &guard) {
     if (_currentChunk->sizeBytes() > _config.getChunkSizeLimit()) {
-        auto completed = std::move(_currentChunk);
-        _currentChunk = std::make_unique<CommitChunk>(_config.getChunkSizeLimit(), completed->stealCallbacks());
-        commitChunk(std::move(completed), guard);
+        commitAndTransferResponses(guard);
     }
 }
 
+void
+Domain::commitAndTransferResponses(const vespalib::MonitorGuard &guard) {
+    auto completed = std::move(_currentChunk);
+    _currentChunk = std::make_unique<CommitChunk>(_config.getChunkSizeLimit(), completed->stealCallbacks());
+    commitChunk(std::move(completed), guard);
+}
+
 std::unique_ptr<CommitChunk>
 Domain::grabCurrentChunk(const vespalib::MonitorGuard & guard) {
     assert(guard.monitors(_currentChunkMonitor));
diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h
index 041ec27cf23..e41ad930840 100644
--- a/searchlib/src/vespa/searchlib/transactionlog/domain.h
+++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h
@@ -57,6 +57,7 @@ public:
     Domain & setConfig(const DomainConfig & cfg);
 private:
     void commitIfFull(const vespalib::MonitorGuard & guard);
+    void commitAndTransferResponses(const vespalib::MonitorGuard & guard);
 
     std::unique_ptr<CommitChunk> grabCurrentChunk(const vespalib::MonitorGuard & guard);
     void commitChunk(std::unique_ptr<CommitChunk> chunk, const vespalib::MonitorGuard & chunkOrderGuard);
diff --git a/storage/src/vespa/storage/storageserver/distributornode.cpp b/storage/src/vespa/storage/storageserver/distributornode.cpp
index 3d1f9bbaf2e..8c2441b5eed 100644
--- a/storage/src/vespa/storage/storageserver/distributornode.cpp
+++ b/storage/src/vespa/storage/storageserver/distributornode.cpp
@@ -20,7 +20,8 @@ DistributorNode::DistributorNode(
         DistributorNodeContext& context,
         ApplicationGenerationFetcher& generationFetcher,
         NeedActiveState activeState,
-        StorageLink::UP communicationManager)
+        StorageLink::UP communicationManager,
+        std::unique_ptr<IStorageChainBuilder> storage_chain_builder)
     : StorageNode(configUri, context, generationFetcher,
             std::unique_ptr<HostInfo>(new HostInfo()),
                   communicationManager.get() == 0 ? NORMAL
@@ -32,6 +33,9 @@ DistributorNode::DistributorNode(
       _manageActiveBucketCopies(activeState == NEED_ACTIVE_BUCKET_STATES_SET),
       _retrievedCommunicationManager(std::move(communicationManager))
 {
+    if (storage_chain_builder) {
+        set_storage_chain_builder(std::move(storage_chain_builder));
+    }
     try{
         initialize();
     } catch (const vespalib::NetworkSetupFailureException & e) {
diff --git a/storage/src/vespa/storage/storageserver/distributornode.h b/storage/src/vespa/storage/storageserver/distributornode.h
index 39614674bb5..2a5149aa8ac 100644
--- a/storage/src/vespa/storage/storageserver/distributornode.h
+++ b/storage/src/vespa/storage/storageserver/distributornode.h
@@ -15,6 +15,8 @@
 
 namespace storage {
 
+class IStorageChainBuilder;
+
 class DistributorNode
       : public StorageNode,
         private UniqueTimeCalculator
@@ -38,7 +40,8 @@ public:
                     DistributorNodeContext&,
                     ApplicationGenerationFetcher& generationFetcher,
                     NeedActiveState,
-                    std::unique_ptr<StorageLink> communicationManager);
+                    std::unique_ptr<StorageLink> communicationManager,
+                    std::unique_ptr<IStorageChainBuilder> storage_chain_builder);
     ~DistributorNode() override;
 
     const lib::NodeType& getNodeType() const override { return lib::NodeType::DISTRIBUTOR; }
diff --git a/storageserver/src/vespa/storageserver/app/distributorprocess.cpp b/storageserver/src/vespa/storageserver/app/distributorprocess.cpp
index d9972116559..e448c7c68bc 100644
--- a/storageserver/src/vespa/storageserver/app/distributorprocess.cpp
+++ b/storageserver/src/vespa/storageserver/app/distributorprocess.cpp
@@ -2,6 +2,7 @@
 
 #include "distributorprocess.h"
 #include <vespa/storage/common/storagelink.h>
+#include <vespa/storage/common/i_storage_chain_builder.h>
 #include <vespa/config/helper/configgetter.hpp>
 
 #include <vespa/log/log.h>
@@ -11,7 +12,8 @@ namespace storage {
 
 DistributorProcess::DistributorProcess(const config::ConfigUri & configUri)
     : Process(configUri),
-      _activeFlag(DistributorNode::NO_NEED_FOR_ACTIVE_STATES)
+      _activeFlag(DistributorNode::NO_NEED_FOR_ACTIVE_STATES),
+      _storage_chain_builder()
 {
 }
 
@@ -73,9 +75,15 @@ DistributorProcess::configUpdated()
 void
 DistributorProcess::createNode()
 {
-    _node.reset(new DistributorNode(_configUri, _context, *this, _activeFlag, StorageLink::UP()));
+    _node = std::make_unique<DistributorNode>(_configUri, _context, *this, _activeFlag, StorageLink::UP(), std::move(_storage_chain_builder));
     _node->handleConfigChange(*_distributorConfigHandler->getConfig());
     _node->handleConfigChange(*_visitDispatcherConfigHandler->getConfig());
 }
 
+void
+DistributorProcess::set_storage_chain_builder(std::unique_ptr<IStorageChainBuilder> builder)
+{
+    _storage_chain_builder = std::move(builder);
+}
+
 } // storage
diff --git a/storageserver/src/vespa/storageserver/app/distributorprocess.h b/storageserver/src/vespa/storageserver/app/distributorprocess.h
index 4a2289e6151..21e7e9b534a 100644
--- a/storageserver/src/vespa/storageserver/app/distributorprocess.h
+++ b/storageserver/src/vespa/storageserver/app/distributorprocess.h
@@ -12,6 +12,8 @@
 
 namespace storage {
 
+class IStorageChainBuilder;
+
 class DistributorProcess final : public Process {
     DistributorNodeContext _context;
     DistributorNode::NeedActiveState _activeFlag;
@@ -20,6 +22,7 @@ class DistributorProcess final : public Process {
             _distributorConfigHandler;
     config::ConfigHandle<vespa::config::content::core::StorVisitordispatcherConfig>::UP
             _visitDispatcherConfigHandler;
+    std::unique_ptr<IStorageChainBuilder> _storage_chain_builder;
 
 public:
     explicit DistributorProcess(const config::ConfigUri & configUri);
@@ -35,6 +38,7 @@ public:
     std::string getComponentName() const override { return "distributor"; }
 
     virtual DistributorNodeContext& getDistributorContext() { return _context; }
+    void set_storage_chain_builder(std::unique_ptr<IStorageChainBuilder> builder);
 };
 
 } // storage
diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java
index 2417a4acf71..733e5bca424 100644
--- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java
+++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java
@@ -47,7 +47,7 @@ public final class ConnectionParams {
         private int traceEveryXOperation = 0;
         private boolean printTraceToStdErr = true;
         private boolean useTlsConfigFromEnvironment = false;
-        private Duration connectionTimeToLive = Duration.ofSeconds(15);
+        private Duration connectionTimeToLive = Duration.ofSeconds(30);
         private Path privateKey;
         private Path certificate;
         private Path caCertificates;
diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java
index bd5cf761024..735ca0beb40 100644
--- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java
+++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java
@@ -23,6 +23,7 @@ import org.apache.http.entity.InputStreamEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.message.BasicHeader;
+import org.apache.http.util.EntityUtils;
 
 import javax.net.ssl.SSLContext;
 import java.io.ByteArrayInputStream;
@@ -246,24 +247,21 @@ class ApacheGatewayConnection implements GatewayConnection {
     }
 
     private InputStream executePost(HttpPost httpPost) throws ServerResponseException, IOException {
-        HttpResponse response;
-        try {
-            if (httpClient == null)
-                throw new IOException("Trying to executePost while not having a connection/http client");
-            response = httpClient.execute(httpPost);
-        } catch (Exception e) {
-            httpPost.abort();
-            throw e;
-        }
+        if (httpClient == null)
+            throw new IOException("Trying to executePost while not having a connection/http client");
+        HttpResponse response = httpClient.execute(httpPost);
         try {
             verifyServerResponseCode(response);
             verifyServerVersion(response.getFirstHeader(Headers.VERSION));
             verifySessionHeader(response.getFirstHeader(Headers.SESSION_ID));
         } catch (ServerResponseException e) {
-            httpPost.abort();
+            // Ensure response is consumed to allow connection reuse later on
+            EntityUtils.consumeQuietly(response.getEntity());
             throw e;
         }
-        return response.getEntity().getContent();
+        // Consume response now to allow connection to be reused immediately
+        byte[] responseData = EntityUtils.toByteArray(response.getEntity());
+        return responseData == null ? null : new ByteArrayInputStream(responseData);
     }
 
     private void verifyServerResponseCode(HttpResponse response) throws ServerResponseException {
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java
index 240adc29197..ef90d6853b1 100644
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java
+++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java
@@ -101,36 +101,36 @@ public class IOThreadTest {
         tester.send("doc1");
         tester.tick(1);
 
-        tester.clock().advance(Duration.ofSeconds(16)); // Default connection ttl is 15
+        tester.clock().advance(Duration.ofSeconds(31)); // Default connection ttl is 30
         tester.tick(3);
 
         assertEquals(1, ioThread.oldConnections().size());
         assertEquals(firstConnection, ioThread.oldConnections().get(0));
         assertNotSame(firstConnection, ioThread.currentConnection());
-        assertEquals(16, firstConnection.lastPollTime().toEpochMilli() / 1000);
+        assertEquals(31, firstConnection.lastPollTime().toEpochMilli() / 1000);
 
         // Check old connection poll pattern (exponential backoff)
-        assertLastPollTimeWhenAdvancing(16, 1, firstConnection, tester);
-        assertLastPollTimeWhenAdvancing(18, 1, firstConnection, tester);
-        assertLastPollTimeWhenAdvancing(18, 1, firstConnection, tester);
-        assertLastPollTimeWhenAdvancing(18, 1, firstConnection, tester);
-        assertLastPollTimeWhenAdvancing(18, 1, firstConnection, tester);
-        assertLastPollTimeWhenAdvancing(22, 1, firstConnection, tester);
-        assertLastPollTimeWhenAdvancing(22, 1, firstConnection, tester);
-        assertLastPollTimeWhenAdvancing(22, 1, firstConnection, tester);
-        assertLastPollTimeWhenAdvancing(22, 1, firstConnection, tester);
-        assertLastPollTimeWhenAdvancing(22, 1, firstConnection, tester);
-        assertLastPollTimeWhenAdvancing(22, 1, firstConnection, tester);
-        assertLastPollTimeWhenAdvancing(22, 1, firstConnection, tester);
-        assertLastPollTimeWhenAdvancing(22, 1, firstConnection, tester);
-        assertLastPollTimeWhenAdvancing(30, 1, firstConnection, tester);
-        assertLastPollTimeWhenAdvancing(30, 1, firstConnection, tester);
-        assertLastPollTimeWhenAdvancing(30, 1, firstConnection, tester);
-        assertLastPollTimeWhenAdvancing(30, 1, firstConnection, tester);
-        assertLastPollTimeWhenAdvancing(30, 1, firstConnection, tester);
-        assertLastPollTimeWhenAdvancing(30, 1, firstConnection, tester);
-        assertLastPollTimeWhenAdvancing(30, 1, firstConnection, tester);
-        assertLastPollTimeWhenAdvancing(30, 1, firstConnection, tester);
+        assertLastPollTimeWhenAdvancing(31, 1, firstConnection, tester);
+        assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester);
+        assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester);
+        assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester);
+        assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester);
+        assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester);
+        assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester);
+        assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester);
+        assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester);
+        assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester);
+        assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester);
+        assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester);
+        assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester);
+        assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester);
+        assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester);
+        assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester);
+        assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester);
+        assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester);
+        assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester);
+        assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester);
+        assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester);
 
         tester.clock().advance(Duration.ofSeconds(200));
         tester.tick(1);