diff options
author | Øyvind Grønnesby <oyving@verizonmedia.com> | 2020-09-30 11:08:36 +0200 |
---|---|---|
committer | Øyvind Grønnesby <oyving@verizonmedia.com> | 2020-09-30 11:08:36 +0200 |
commit | ef52f22d1a495ccd3497384d19feeebe70439378 (patch) | |
tree | 3d456e13abaa491e5c9f1439de592a08a28881e0 | |
parent | 68176288649b5ebc3bbf00ae934875c17b9a75b7 (diff) | |
parent | bd6373019e2844f0c20cc1f7696a3e017be9c08c (diff) |
Merge remote-tracking branch 'origin/master' into ogronnesby/quota-decimal
26 files changed, 325 insertions, 238 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java index ea4b6a2b02d..2a798839752 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java @@ -39,6 +39,8 @@ public interface ConfigChangeAction { boolean allowed(); /** Returns whether this change should be ignored for internal redeploy */ - boolean ignoreForInternalRedeploy(); + default boolean ignoreForInternalRedeploy() { + return false; + }; } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/Quota.java b/config-model-api/src/main/java/com/yahoo/config/model/api/Quota.java index 3a57e5bb66c..01ed33e023e 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/Quota.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/Quota.java @@ -18,7 +18,10 @@ import java.util.Optional; public class Quota { private static final Quota UNLIMITED = new Quota(Optional.empty(), Optional.empty()); + /** The max size of a cluster inside application */ private final Optional<Integer> maxClusterSize; + + /** The max budget in dollars per hour */ private final Optional<BigDecimal> budget; // TODO: Remove constructor once Vespa < 7.300 is gone from production diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index bf752b39fa8..3bbcf7979f3 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -233,6 +233,7 @@ TOKEN : | < SUFFIX: "suffix" > | < CONSTANT: "constant"> | < ONNXMODEL: "onnx-model"> +| < MODEL: "model" > | < RANKPROFILE: "rank-profile" > | < RANKDEGRADATIONFREQ: "rank-degradation-frequency" > | < RANKDEGRADATION: "rank-degradation" > @@ -1976,7 +1977,7 @@ void rankProfile(Search search) : RankProfile profile; } { - ( <RANKPROFILE> name = identifierWithDash() + ( ( <MODEL> | <RANKPROFILE> ) name = identifierWithDash() { if (documentsOnly) { profile = new DocumentsOnlyRankProfile(name, search, rankProfileRegistry); @@ -2676,6 +2677,7 @@ String identifier() : { } | <QUATERNARY> | <QUERYCOMMAND> | <RANK> + | <MODEL> | <RANKPROFILE> | <RANKPROPERTIES> | <RANKSCOREDROPLIMIT> diff --git a/config-model/src/test/derived/rankprofiles/rankprofiles.sd b/config-model/src/test/derived/rankprofiles/rankprofiles.sd index abafee91533..6c3720df589 100644 --- a/config-model/src/test/derived/rankprofiles/rankprofiles.sd +++ b/config-model/src/test/derived/rankprofiles/rankprofiles.sd @@ -19,21 +19,21 @@ search rankprofiles { } - rank-profile default { + model default { } - rank-profile other1 inherits default { + model other1 inherits default { rank field1: filter rank none: filter } - rank-profile other2 inherits other1 { + model other2 inherits other1 { } - rank-profile other3 { + model other3 { } - rank-profile four { + model four { match-phase { attribute: field2 order: ascending @@ -41,7 +41,7 @@ search rankprofiles { } } - rank-profile five { + model five { match-phase { attribute: field2 order: descending @@ -49,14 +49,14 @@ search rankprofiles { } } - rank-profile six { + model six { match-phase { attribute: field3 max-hits: 666 } } - rank-profile seven { + model seven { match-phase { attribute: field3 max-hits:800 @@ -65,7 +65,7 @@ search rankprofiles { } } - rank-profile eight inherits seven { + model eight inherits seven { } diff --git a/config-model/src/test/examples/desktop.sd b/config-model/src/test/examples/desktop.sd deleted file mode 100644 index 82741dbe53f..00000000000 --- a/config-model/src/test/examples/desktop.sd +++ /dev/null @@ -1,108 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# A search definition of medium complexity -search desktop { - - # A document with some fields. - # The fields implicitly defines some indices, summary fields and attributes - document desktop inherits product { - - field title type text { - mandatory: true - indexing: summary | attribute | index_text - # index-to: title, default - - rank-weight: 300 - rank-type: identity - } - - field manufacturer type text { - indexing: summary | attribute | index - # index-to: manufacturer, default - alias: producer, brand - - rank-type: identity - rank-weight:200 - } - - field description type text { - indexing: summary | index - - rank-type: about - rank-weight: 100 - result-transform: dynamicteaser - } - - field category type text { - indexing: index - # index-to: category, default - rank-weight: 50 - } - - field minprice type int { - indexing: summary | attribute | index - index-decimals: 2 - - rank-type: simple - weight: 30 - staticrankbits: 16 - } - - field someorder type int { - indexing: attribute someorderranking - staticrankbits someorderranking: 32 - } - - # index_url implicitly defines some fields not contained in the document (contexts) - # If attributes needs to be set on these, it can be done by explicitly listing - # the fields outside documents (show). - # I think we should maybe allow setting such field attributes inside the parent - # field as well for convenience. Both is shown. - field url type url { - # Should index mean index_url for url type fields? - indexing: summary | index_url - parse-mode: url # Must be default for url types, but shown here anyway - - rank-type: link - } - - } - - field category_arr type array<text> { - indexing: input category | split ";" | attribute category_arr - } - - # Overriding some url field setting from rank-type link - field url.host { - weight: 1000 - } - - # Setting an attribute on a non-field index - # This is redunant, as default is default - index default { - default: true - } - - # Optionally specify a desire to use a shared dictionary ("catalogs") - shared-dictionary: normal, title, manufacturer - - # Optionally set rank values for all indices - # Default is the name of the default one - # Rank settings from individual fields can be selectively overridden - rankprofile default { - firstocc-boost text: 200 - } - - # Another rank profile - rankprofile predefinedorder { - dynamicranking: off - attribute: someorder - } - - # Some experimental ranking changes - rankprofile experimental inherits default { - firstocc-boost text: 300 - } - -} - - diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java index 70a6bbbf659..699fd2c9fe2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java @@ -43,7 +43,8 @@ public class QuotaValidatorTest { tester.deploy(null, getServices("testCluster", 10), Environment.prod, null); fail(); } catch (RuntimeException e) { - assertEquals("Hourly spend for maximum specified resources ($1.60) exceeds budget from quota ($1.00)!", e.getMessage()); + assertEquals("Hourly spend for maximum specified resources ($-.--) exceeds budget from quota ($-.--)!", + ValidationTester.censorNumbers(e.getMessage())); } } @@ -63,4 +64,6 @@ public class QuotaValidatorTest { " </content>" + "</services>"; } + + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java index 6961ffa682b..2b3b1a9fcc7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java @@ -96,6 +96,8 @@ public class ValidationTester { return new Pair<>(newModel, newModelCreator.configChangeActions); } - + public static String censorNumbers(String s) { + return s.replaceAll("\\d", "-"); + } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index fad276379cc..f695f0b75eb 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -12,12 +12,11 @@ import java.util.Optional; public class NodeResources { // Standard unit cost in dollars per hour - private static final double cpuUnitCost = 0.12; - private static final double memoryUnitCost = 0.012; - private static final double diskUnitCost = 0.0004; + private static final double cpuUnitCost = 0.09; + private static final double memoryUnitCost = 0.009; + private static final double diskUnitCost = 0.0003; - // TODO: Remove when models older than 7.226 are gone - public static final NodeResources unspecified = new NodeResources(0, 0, 0, 0); + private static final NodeResources unspecified = new NodeResources(0, 0, 0, 0); public enum DiskSpeed { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java index 5992d47855f..26abe92a6f1 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java @@ -22,6 +22,7 @@ public class RpcPing implements Pinger, Client.ResponseReceiver { private static final Logger log = Logger.getLogger(RpcPing.class.getName()); private static final String RPC_METHOD = "vespa.searchprotocol.ping"; private static final CompressionType PING_COMPRESSION = CompressionType.NONE; + private static final boolean triggeredClassLoading = ErrorMessage.createBackendCommunicationError("TriggerClassLoading") instanceof ErrorMessage; private final Node node; private final RpcResourcePool resourcePool; @@ -86,7 +87,7 @@ public class RpcPing implements Pinger, Client.ResponseReceiver { @Override public void receive(ResponseOrError<ProtobufResponse> response) { - if (clusterMonitor.isClosed()) return; + if (clusterMonitor.isClosed() && ! triggeredClassLoading) return; if (node.isLastReceivedPong(pingSequenceId)) { pongHandler.handle(toPong(response)); } else { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json index b2e634e5d6b..65fa2a4bf70 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json @@ -12,7 +12,7 @@ "diskSpeed": "slow", "storageType": "remote" }, - "cost": 0.12 + "cost": "(ignore)" }, "max": { "nodes": 2, @@ -25,7 +25,7 @@ "diskSpeed": "slow", "storageType": "remote" }, - "cost": 0.47 + "cost": "(ignore)" }, "current": { "nodes": 2, @@ -38,7 +38,7 @@ "diskSpeed": "slow", "storageType": "remote" }, - "cost": 0.24 + "cost": "(ignore)" }, "target": { "nodes": 2, @@ -51,7 +51,7 @@ "diskSpeed": "slow", "storageType": "remote" }, - "cost": 0.32 + "cost": "(ignore)" } } } diff --git a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp index ccc71fdd0f6..31777e233f6 100644 --- a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp +++ b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp @@ -30,11 +30,13 @@ #include <vespa/eval/eval/tensor_function.h> #include <vespa/eval/tensor/default_tensor_engine.h> #include <vespa/eval/tensor/default_value_builder_factory.h> +#include <vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h> #include <vespa/vespalib/util/benchmark_timer.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/stash.h> #include <vespa/vespalib/gtest/gtest.h> #include <optional> +#include <algorithm> using namespace vespalib; using namespace vespalib::eval; @@ -50,7 +52,11 @@ template <typename T> using CREF = std::reference_wrapper<const T>; //----------------------------------------------------------------------------- struct Impl { - virtual const vespalib::string &name() const = 0; + size_t order; + vespalib::string name; + vespalib::string short_name; + Impl(size_t order_in, const vespalib::string &name_in, const vespalib::string &short_name_in) + : order(order_in), name(name_in), short_name(short_name_in) {} virtual Value::UP create_value(const TensorSpec &spec) const = 0; virtual TensorSpec create_spec(const Value &value) const = 0; virtual Instruction create_join(const ValueType &lhs, const ValueType &rhs, operation::op2_t function, Stash &stash) const = 0; @@ -59,11 +65,9 @@ struct Impl { }; struct ValueImpl : Impl { - vespalib::string my_name; const ValueBuilderFactory &my_factory; - ValueImpl(const vespalib::string &name_in, const ValueBuilderFactory &factory) - : my_name(name_in), my_factory(factory) {} - const vespalib::string &name() const override { return my_name; } + ValueImpl(size_t order_in, const vespalib::string &name_in, const vespalib::string &short_name_in, const ValueBuilderFactory &factory) + : Impl(order_in, name_in, short_name_in), my_factory(factory) {} Value::UP create_value(const TensorSpec &spec) const override { return value_from_spec(spec, my_factory); } TensorSpec create_spec(const Value &value) const override { return spec_from_value(value); } Instruction create_join(const ValueType &lhs, const ValueType &rhs, operation::op2_t function, Stash &stash) const override { @@ -72,11 +76,9 @@ struct ValueImpl : Impl { }; struct EngineImpl : Impl { - vespalib::string my_name; const TensorEngine &my_engine; - EngineImpl(const vespalib::string &name_in, const TensorEngine &engine_in) - : my_name(name_in), my_engine(engine_in) {} - const vespalib::string &name() const override { return my_name; } + EngineImpl(size_t order_in, const vespalib::string &name_in, const vespalib::string &short_name_in, const TensorEngine &engine_in) + : Impl(order_in, name_in, short_name_in), my_engine(engine_in) {} Value::UP create_value(const TensorSpec &spec) const override { return my_engine.from_spec(spec); } TensorSpec create_spec(const Value &value) const override { return my_engine.to_spec(value); } Instruction create_join(const ValueType &lhs, const ValueType &rhs, operation::op2_t function, Stash &stash) const override { @@ -91,19 +93,80 @@ struct EngineImpl : Impl { //----------------------------------------------------------------------------- -EngineImpl simple_tensor_engine_impl(" [SimpleTensorEngine]", SimpleTensorEngine::ref()); -EngineImpl default_tensor_engine_impl("[DefaultTensorEngine]", DefaultTensorEngine::ref()); -ValueImpl simple_value_impl(" [SimpleValue]", SimpleValueBuilderFactory::get()); -ValueImpl default_tensor_value_impl(" [Adaptive Value]", DefaultValueBuilderFactory::get()); +EngineImpl simple_tensor_engine_impl(4, " SimpleTensorEngine", " SimpleT", SimpleTensorEngine::ref()); +EngineImpl default_tensor_engine_impl(1, "DefaultTensorEngine", "OLD PROD", DefaultTensorEngine::ref()); +ValueImpl simple_value_impl(3, " SimpleValue", " SimpleV", SimpleValueBuilderFactory::get()); +ValueImpl packed_mixed_tensor_impl(2, " PackedMixedTensor", " Packed", PackedMixedTensorBuilderFactory::get()); +ValueImpl default_tensor_value_impl(0, " DefaultValue", "NEW PROD", DefaultValueBuilderFactory::get()); +vespalib::string short_header("--------"); double budget = 5.0; std::vector<CREF<Impl>> impl_list = {simple_tensor_engine_impl, default_tensor_engine_impl, simple_value_impl, + packed_mixed_tensor_impl, default_tensor_value_impl}; //----------------------------------------------------------------------------- +struct BenchmarkHeader { + std::vector<vespalib::string> short_names; + BenchmarkHeader() : short_names() { + short_names.resize(impl_list.size()); + for (const Impl &impl: impl_list) { + short_names[impl.order] = impl.short_name; + } + } + void print_trailer() const { + for (size_t i = 0; i < short_names.size(); ++i) { + fprintf(stderr, "+%s", short_header.c_str()); + } + fprintf(stderr, "+------------------------------------------------\n"); + } + void print() const { + for (const auto &name: short_names) { + fprintf(stderr, "|%s", name.c_str()); + } + fprintf(stderr, "| Benchmark description\n"); + print_trailer(); + } +}; + +struct BenchmarkResult { + vespalib::string desc; + std::optional<double> ref_time; + std::vector<double> relative_perf; + BenchmarkResult(const vespalib::string &desc_in, size_t num_values) + : desc(desc_in), ref_time(std::nullopt), relative_perf(num_values, 0.0) {} + ~BenchmarkResult(); + void sample(size_t order, double time) { + relative_perf[order] = time; + if (order == 1) { + if (ref_time.has_value()) { + ref_time = std::min(ref_time.value(), time); + } else { + ref_time = time; + } + } + } + void normalize() { + for (double &perf: relative_perf) { + perf = ref_time.value() / perf; + } + } + void print() const { + for (double perf: relative_perf) { + fprintf(stderr, "|%8.2f", perf); + } + fprintf(stderr, "| %s\n", desc.c_str()); + } +}; +BenchmarkResult::~BenchmarkResult() = default; + +std::vector<BenchmarkResult> benchmark_results; + +//----------------------------------------------------------------------------- + struct EvalOp { using UP = std::unique_ptr<EvalOp>; const Impl &impl; @@ -142,9 +205,14 @@ void benchmark(const vespalib::string &desc, const std::vector<EvalOp::UP> &list expect = eval->result(); } } + BenchmarkResult result(desc, list.size()); for (const auto &eval: list) { - fprintf(stderr, " %s: %10.3f us\n", eval->impl.name().c_str(), eval->estimate_cost_us()); + double time = eval->estimate_cost_us(); + result.sample(eval->impl.order, time); + fprintf(stderr, " %s(%s): %10.3f us\n", eval->impl.name.c_str(), eval->impl.short_name.c_str(), time); } + result.normalize(); + benchmark_results.push_back(result); fprintf(stderr, "--------------------------------------------------------\n"); } @@ -318,4 +386,15 @@ TEST(MixedJoin, no_overlap) { //----------------------------------------------------------------------------- +TEST(PrintResults, print_results) { + BenchmarkHeader header; + std::sort(benchmark_results.begin(), benchmark_results.end(), + [](const auto &a, const auto &b){ return (a.relative_perf[0] < b.relative_perf[0]); }); + header.print(); + for (const auto &result: benchmark_results) { + result.print(); + } + header.print_trailer(); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/vespa/eval/eval/value.h b/eval/src/vespa/eval/eval/value.h index 20923fcd621..a084d267cec 100644 --- a/eval/src/vespa/eval/eval/value.h +++ b/eval/src/vespa/eval/eval/value.h @@ -100,6 +100,23 @@ public: }; /** + * A generic value without any mapped dimensions referencing its + * components without owning anything. + **/ +class DenseValueView final : public Value +{ +private: + const ValueType &_type; + TypedCells _cells; +public: + DenseValueView(const ValueType &type_in, TypedCells cells_in) + : _type(type_in), _cells(cells_in) {} + const ValueType &type() const final override { return _type; } + TypedCells cells() const final override { return _cells; } + const Index &index() const final override { return TrivialIndex::get(); } +}; + +/** * Tagging interface used as return type from factories before * downcasting to actual builder with specialized cell type. **/ @@ -162,3 +179,4 @@ protected: } VESPA_CAN_SKIP_DESTRUCTION(::vespalib::eval::DoubleValue); +VESPA_CAN_SKIP_DESTRUCTION(::vespalib::eval::DenseValueView); diff --git a/eval/src/vespa/eval/instruction/generic_join.cpp b/eval/src/vespa/eval/instruction/generic_join.cpp index 8a1a199effa..b54f7d8952a 100644 --- a/eval/src/vespa/eval/instruction/generic_join.cpp +++ b/eval/src/vespa/eval/instruction/generic_join.cpp @@ -10,6 +10,7 @@ namespace vespalib::eval::instruction { +using TypedCells = Value::TypedCells; using State = InterpretedFunction::State; using Instruction = InterpretedFunction::Instruction; @@ -95,13 +96,8 @@ struct SparseJoinState { }; SparseJoinState::~SparseJoinState() = default; -/* template <typename LCT, typename RCT, typename OCT, typename Fun> -void generic_join() -*/ - -template <typename LCT, typename RCT, typename OCT, typename Fun> -void my_generic_join_op(State &state, uint64_t param_in) { +void my_mixed_join_op(State &state, uint64_t param_in) { const auto ¶m = unwrap_param<JoinParam>(param_in); Fun fun(param.function); const Value &lhs = state.peek(1); @@ -126,9 +122,39 @@ void my_generic_join_op(State &state, uint64_t param_in) { state.pop_pop_push(result_ref); }; +template <typename LCT, typename RCT, typename OCT, typename Fun> +void my_dense_join_op(State &state, uint64_t param_in) { + const auto ¶m = unwrap_param<JoinParam>(param_in); + Fun fun(param.function); + auto lhs_cells = state.peek(1).cells().typify<LCT>(); + auto rhs_cells = state.peek(0).cells().typify<RCT>(); + ArrayRef<OCT> out_cells = state.stash.create_array<OCT>(param.dense_plan.out_size); + OCT *dst = out_cells.begin(); + auto join_cells = [&](size_t lhs_idx, size_t rhs_idx) { *dst++ = fun(lhs_cells[lhs_idx], rhs_cells[rhs_idx]); }; + param.dense_plan.execute(0, 0, join_cells); + state.pop_pop_push(state.stash.create<DenseValueView>(param.res_type, TypedCells(out_cells))); +}; + +template <typename Fun> +void my_double_join_op(State &state, uint64_t param_in) { + Fun fun(unwrap_param<JoinParam>(param_in).function); + state.pop_pop_push(state.stash.create<DoubleValue>(fun(state.peek(1).cells().typify<double>()[0], + state.peek(0).cells().typify<double>()[0]))); +}; + struct SelectGenericJoinOp { - template <typename LCT, typename RCT, typename OCT, typename Fun> static auto invoke() { - return my_generic_join_op<LCT,RCT,OCT,Fun>; + template <typename LCT, typename RCT, typename OCT, typename Fun> static auto invoke(const JoinParam ¶m) { + if (param.res_type.is_double()) { + bool all_double = (std::is_same_v<LCT, double> && + std::is_same_v<RCT, double> && + std::is_same_v<OCT, double>); + assert(all_double); + return my_double_join_op<Fun>; + } + if (param.sparse_plan.sources.empty()) { + return my_dense_join_op<LCT,RCT,OCT,Fun>; + } + return my_mixed_join_op<LCT,RCT,OCT,Fun>; } }; @@ -225,7 +251,7 @@ GenericJoin::make_instruction(const ValueType &lhs_type, const ValueType &rhs_ty const ValueBuilderFactory &factory, Stash &stash) { auto ¶m = stash.create<JoinParam>(lhs_type, rhs_type, function, factory); - auto fun = typify_invoke<4,JoinTypify,SelectGenericJoinOp>(lhs_type.cell_type(), rhs_type.cell_type(), param.res_type.cell_type(), function); + auto fun = typify_invoke<4,JoinTypify,SelectGenericJoinOp>(lhs_type.cell_type(), rhs_type.cell_type(), param.res_type.cell_type(), function, param); return Instruction(fun, wrap_param<JoinParam>(param)); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java index b41b7f15499..5df45bbc1b1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java @@ -79,9 +79,9 @@ public abstract class ApplicationMaintainer extends NodeRepositoryMaintainer { * @return whether it was successfully deployed */ protected final boolean deployWithLock(ApplicationId application) { - if ( ! canDeployNow(application)) return false; // redeployment is no longer needed try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, metric, nodeRepository())) { if ( ! deployment.isValid()) return false; // this will be done at another config server + if ( ! canDeployNow(application)) return false; // redeployment is no longer needed log.log(Level.INFO, application + " will be deployed, last deploy time " + getLastDeployTime(application)); return deployment.activate().isPresent(); } finally { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index 77ef88f0952..2ce3fa6c0f6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.Deployment; import com.yahoo.config.provision.HostLivenessTracker; @@ -187,27 +188,29 @@ public class NodeFailer extends NodeRepositoryMaintainer { * Otherwise we remove any "down" history record. */ private void updateNodeDownState() { - Map<String, Node> activeNodesByHostname = nodeRepository().getNodes(Node.State.active).stream() - .collect(Collectors.toMap(Node::hostname, node -> node)); - - serviceMonitor.getServiceModelSnapshot().getServiceInstancesByHostName() - .forEach((hostName, serviceInstances) -> { - Node node = activeNodesByHostname.get(hostName.s()); - if (node == null) return; - try (var lock = nodeRepository().lock(node.allocation().get().owner())) { - Optional<Node> currentNode = nodeRepository().getNode(node.hostname(), Node.State.active); // re-get inside lock - if (currentNode.isEmpty()) return; // Node disappeared since acquiring lock - node = currentNode.get(); - if (badNode(serviceInstances)) { - recordAsDown(node, lock); - } else { - clearDownRecord(node, lock); - } - } - catch (UncheckedTimeoutException e) { - // Ignore - node may be locked on this round due to deployment - } - }); + NodeList activeNodes = NodeList.copyOf(nodeRepository().getNodes(Node.State.active)); + serviceMonitor.getServiceModelSnapshot().getServiceInstancesByHostName().forEach((hostname, serviceInstances) -> { + Optional<Node> node = activeNodes.matching(n -> n.hostname().equals(hostname.toString())).first(); + if (node.isEmpty()) return; + + // Already correct record, nothing to do + boolean badNode = badNode(serviceInstances); + if (badNode == node.get().history().event(History.Event.Type.down).isPresent()) return; + + // Lock and update status + ApplicationId owner = node.get().allocation().get().owner(); + try (var lock = nodeRepository().lock(owner, Duration.ofSeconds(1))) { + node = getNode(hostname.toString(), owner, lock); // Re-get inside lock + if (node.isEmpty()) return; // Node disappeared or changed allocation + if (badNode) { + recordAsDown(node.get(), lock); + } else { + clearDownRecord(node.get(), lock); + } + } catch (UncheckedTimeoutException ignored) { + // Fine, we'll try updating this node in the next run + } + }); } private Map<Node, String> getActiveNodesByFailureReason(List<Node> activeNodes) { @@ -248,6 +251,13 @@ public class NodeFailer extends NodeRepositoryMaintainer { return reasonsToFailParentHost(hostNode).size() > 0; } + /** Get node by given hostname and application. The applicationLock must be held when calling this */ + private Optional<Node> getNode(String hostname, ApplicationId application, @SuppressWarnings("unused") Mutex applicationLock) { + return nodeRepository().getNode(hostname, Node.State.active) + .filter(node -> node.allocation().isPresent()) + .filter(node -> node.allocation().get().owner().equals(application)); + } + private boolean expectConfigRequests(Node node) { return !node.type().isHost(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 75f3c892571..b2cb7f545ff 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -28,6 +28,8 @@ import java.util.stream.Collectors; */ public class GroupPreparer { + private static final Mutex PROBE_LOCK = () -> {}; + private final NodeRepository nodeRepository; private final Optional<HostProvisioner> hostProvisioner; private final ListFlag<HostCapacity> preprovisionCapacityFlag; @@ -59,28 +61,26 @@ public class GroupPreparer { List<Node> surplusActiveNodes, MutableInteger highestIndex, int wantedGroups) { boolean dynamicProvisioningEnabled = nodeRepository.canProvisionHosts() && nodeRepository.zone().getCloud().dynamicProvisioning(); boolean allocateFully = dynamicProvisioningEnabled && preprovisionCapacityFlag.value().isEmpty(); - try (Mutex lock = nodeRepository.lock(application)) { - // Lock ready pool to ensure that the same nodes are not simultaneously allocated by others - try (Mutex allocationLock = nodeRepository.lockUnallocated()) { + // Try preparing in memory without lock. Most of the time there should be no changes and we can return nodes + // previously allocated. + { + MutableInteger probePrepareHighestIndex = new MutableInteger(highestIndex.get()); + NodeAllocation probeAllocation = prepareAllocation(application, cluster, requestedNodes, surplusActiveNodes, + probePrepareHighestIndex, wantedGroups, allocateFully, PROBE_LOCK); + if (probeAllocation.fulfilledAndNoChanges()) { + List<Node> acceptedNodes = probeAllocation.finalNodes(); + surplusActiveNodes.removeAll(acceptedNodes); + highestIndex.set(probePrepareHighestIndex.get()); + return acceptedNodes; + } + } - // Create a prioritized set of nodes - LockedNodeList allNodes = nodeRepository.list(allocationLock); - NodeAllocation allocation = new NodeAllocation(allNodes, application, cluster, requestedNodes, - highestIndex, nodeRepository); - - NodePrioritizer prioritizer = new NodePrioritizer(allNodes, - application, - cluster, - requestedNodes, - wantedGroups, - allocateFully, - nodeRepository); - prioritizer.addApplicationNodes(); - prioritizer.addSurplusNodes(surplusActiveNodes); - prioritizer.addReadyNodes(); - prioritizer.addNewDockerNodes(); - allocation.offer(prioritizer.prioritize()); + // There were some changes, so re-do the allocation with locks + try (Mutex lock = nodeRepository.lock(application)) { + try (Mutex allocationLock = nodeRepository.lockUnallocated()) { + NodeAllocation allocation = prepareAllocation(application, cluster, requestedNodes, surplusActiveNodes, + highestIndex, wantedGroups, allocateFully, allocationLock); if (dynamicProvisioningEnabled) { Version osVersion = nodeRepository.osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion); @@ -122,4 +122,22 @@ public class GroupPreparer { } } + private NodeAllocation prepareAllocation(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, + List<Node> surplusActiveNodes, MutableInteger highestIndex, int wantedGroups, + boolean allocateFully, Mutex allocationLock) { + LockedNodeList allNodes = nodeRepository.list(allocationLock); + NodeAllocation allocation = new NodeAllocation(allNodes, application, cluster, requestedNodes, + highestIndex, nodeRepository); + NodePrioritizer prioritizer = new NodePrioritizer(allNodes, + application, cluster, requestedNodes, wantedGroups, allocateFully, nodeRepository); + + prioritizer.addApplicationNodes(); + prioritizer.addSurplusNodes(surplusActiveNodes); + prioritizer.addReadyNodes(); + prioritizer.addNewDockerNodes(); + allocation.offer(prioritizer.prioritize()); + + return allocation; + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index b07ce786685..0ebcb34703f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -305,6 +305,11 @@ class NodeAllocation { return requestedNodes.fulfilledBy(accepted); } + /** Returns true this allocation was already fulfilled and resulted in no new changes */ + public boolean fulfilledAndNoChanges() { + return fulfilled() && reservableNodes().isEmpty() && newNodes().isEmpty(); + } + /** * Returns {@link FlavorCount} describing the docker node deficit for the given {@link NodeSpec}. * diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index bd7feec0598..126a7afed4d 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -209,6 +209,10 @@ Domain::getSynced() const void Domain::triggerSyncNow() { + { + vespalib::MonitorGuard guard(_currentChunkMonitor); + commitAndTransferResponses(guard); + } MonitorGuard guard(_syncMonitor); if (!_pendingSync) { _pendingSync = true; @@ -352,12 +356,17 @@ Domain::startCommit(DoneCallback onDone) { void Domain::commitIfFull(const vespalib::MonitorGuard &guard) { if (_currentChunk->sizeBytes() > _config.getChunkSizeLimit()) { - auto completed = std::move(_currentChunk); - _currentChunk = std::make_unique<CommitChunk>(_config.getChunkSizeLimit(), completed->stealCallbacks()); - commitChunk(std::move(completed), guard); + commitAndTransferResponses(guard); } } +void +Domain::commitAndTransferResponses(const vespalib::MonitorGuard &guard) { + auto completed = std::move(_currentChunk); + _currentChunk = std::make_unique<CommitChunk>(_config.getChunkSizeLimit(), completed->stealCallbacks()); + commitChunk(std::move(completed), guard); +} + std::unique_ptr<CommitChunk> Domain::grabCurrentChunk(const vespalib::MonitorGuard & guard) { assert(guard.monitors(_currentChunkMonitor)); diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h index 041ec27cf23..e41ad930840 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h @@ -57,6 +57,7 @@ public: Domain & setConfig(const DomainConfig & cfg); private: void commitIfFull(const vespalib::MonitorGuard & guard); + void commitAndTransferResponses(const vespalib::MonitorGuard & guard); std::unique_ptr<CommitChunk> grabCurrentChunk(const vespalib::MonitorGuard & guard); void commitChunk(std::unique_ptr<CommitChunk> chunk, const vespalib::MonitorGuard & chunkOrderGuard); diff --git a/storage/src/vespa/storage/storageserver/distributornode.cpp b/storage/src/vespa/storage/storageserver/distributornode.cpp index 3d1f9bbaf2e..8c2441b5eed 100644 --- a/storage/src/vespa/storage/storageserver/distributornode.cpp +++ b/storage/src/vespa/storage/storageserver/distributornode.cpp @@ -20,7 +20,8 @@ DistributorNode::DistributorNode( DistributorNodeContext& context, ApplicationGenerationFetcher& generationFetcher, NeedActiveState activeState, - StorageLink::UP communicationManager) + StorageLink::UP communicationManager, + std::unique_ptr<IStorageChainBuilder> storage_chain_builder) : StorageNode(configUri, context, generationFetcher, std::unique_ptr<HostInfo>(new HostInfo()), communicationManager.get() == 0 ? NORMAL @@ -32,6 +33,9 @@ DistributorNode::DistributorNode( _manageActiveBucketCopies(activeState == NEED_ACTIVE_BUCKET_STATES_SET), _retrievedCommunicationManager(std::move(communicationManager)) { + if (storage_chain_builder) { + set_storage_chain_builder(std::move(storage_chain_builder)); + } try{ initialize(); } catch (const vespalib::NetworkSetupFailureException & e) { diff --git a/storage/src/vespa/storage/storageserver/distributornode.h b/storage/src/vespa/storage/storageserver/distributornode.h index 39614674bb5..2a5149aa8ac 100644 --- a/storage/src/vespa/storage/storageserver/distributornode.h +++ b/storage/src/vespa/storage/storageserver/distributornode.h @@ -15,6 +15,8 @@ namespace storage { +class IStorageChainBuilder; + class DistributorNode : public StorageNode, private UniqueTimeCalculator @@ -38,7 +40,8 @@ public: DistributorNodeContext&, ApplicationGenerationFetcher& generationFetcher, NeedActiveState, - std::unique_ptr<StorageLink> communicationManager); + std::unique_ptr<StorageLink> communicationManager, + std::unique_ptr<IStorageChainBuilder> storage_chain_builder); ~DistributorNode() override; const lib::NodeType& getNodeType() const override { return lib::NodeType::DISTRIBUTOR; } diff --git a/storageserver/src/vespa/storageserver/app/distributorprocess.cpp b/storageserver/src/vespa/storageserver/app/distributorprocess.cpp index d9972116559..e448c7c68bc 100644 --- a/storageserver/src/vespa/storageserver/app/distributorprocess.cpp +++ b/storageserver/src/vespa/storageserver/app/distributorprocess.cpp @@ -2,6 +2,7 @@ #include "distributorprocess.h" #include <vespa/storage/common/storagelink.h> +#include <vespa/storage/common/i_storage_chain_builder.h> #include <vespa/config/helper/configgetter.hpp> #include <vespa/log/log.h> @@ -11,7 +12,8 @@ namespace storage { DistributorProcess::DistributorProcess(const config::ConfigUri & configUri) : Process(configUri), - _activeFlag(DistributorNode::NO_NEED_FOR_ACTIVE_STATES) + _activeFlag(DistributorNode::NO_NEED_FOR_ACTIVE_STATES), + _storage_chain_builder() { } @@ -73,9 +75,15 @@ DistributorProcess::configUpdated() void DistributorProcess::createNode() { - _node.reset(new DistributorNode(_configUri, _context, *this, _activeFlag, StorageLink::UP())); + _node = std::make_unique<DistributorNode>(_configUri, _context, *this, _activeFlag, StorageLink::UP(), std::move(_storage_chain_builder)); _node->handleConfigChange(*_distributorConfigHandler->getConfig()); _node->handleConfigChange(*_visitDispatcherConfigHandler->getConfig()); } +void +DistributorProcess::set_storage_chain_builder(std::unique_ptr<IStorageChainBuilder> builder) +{ + _storage_chain_builder = std::move(builder); +} + } // storage diff --git a/storageserver/src/vespa/storageserver/app/distributorprocess.h b/storageserver/src/vespa/storageserver/app/distributorprocess.h index 4a2289e6151..21e7e9b534a 100644 --- a/storageserver/src/vespa/storageserver/app/distributorprocess.h +++ b/storageserver/src/vespa/storageserver/app/distributorprocess.h @@ -12,6 +12,8 @@ namespace storage { +class IStorageChainBuilder; + class DistributorProcess final : public Process { DistributorNodeContext _context; DistributorNode::NeedActiveState _activeFlag; @@ -20,6 +22,7 @@ class DistributorProcess final : public Process { _distributorConfigHandler; config::ConfigHandle<vespa::config::content::core::StorVisitordispatcherConfig>::UP _visitDispatcherConfigHandler; + std::unique_ptr<IStorageChainBuilder> _storage_chain_builder; public: explicit DistributorProcess(const config::ConfigUri & configUri); @@ -35,6 +38,7 @@ public: std::string getComponentName() const override { return "distributor"; } virtual DistributorNodeContext& getDistributorContext() { return _context; } + void set_storage_chain_builder(std::unique_ptr<IStorageChainBuilder> builder); }; } // storage diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java index 2417a4acf71..733e5bca424 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java @@ -47,7 +47,7 @@ public final class ConnectionParams { private int traceEveryXOperation = 0; private boolean printTraceToStdErr = true; private boolean useTlsConfigFromEnvironment = false; - private Duration connectionTimeToLive = Duration.ofSeconds(15); + private Duration connectionTimeToLive = Duration.ofSeconds(30); private Path privateKey; private Path certificate; private Path caCertificates; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java index bd5cf761024..735ca0beb40 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java @@ -23,6 +23,7 @@ import org.apache.http.entity.InputStreamEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.message.BasicHeader; +import org.apache.http.util.EntityUtils; import javax.net.ssl.SSLContext; import java.io.ByteArrayInputStream; @@ -246,24 +247,21 @@ class ApacheGatewayConnection implements GatewayConnection { } private InputStream executePost(HttpPost httpPost) throws ServerResponseException, IOException { - HttpResponse response; - try { - if (httpClient == null) - throw new IOException("Trying to executePost while not having a connection/http client"); - response = httpClient.execute(httpPost); - } catch (Exception e) { - httpPost.abort(); - throw e; - } + if (httpClient == null) + throw new IOException("Trying to executePost while not having a connection/http client"); + HttpResponse response = httpClient.execute(httpPost); try { verifyServerResponseCode(response); verifyServerVersion(response.getFirstHeader(Headers.VERSION)); verifySessionHeader(response.getFirstHeader(Headers.SESSION_ID)); } catch (ServerResponseException e) { - httpPost.abort(); + // Ensure response is consumed to allow connection reuse later on + EntityUtils.consumeQuietly(response.getEntity()); throw e; } - return response.getEntity().getContent(); + // Consume response now to allow connection to be reused immediately + byte[] responseData = EntityUtils.toByteArray(response.getEntity()); + return responseData == null ? null : new ByteArrayInputStream(responseData); } private void verifyServerResponseCode(HttpResponse response) throws ServerResponseException { diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java index 240adc29197..ef90d6853b1 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java @@ -101,36 +101,36 @@ public class IOThreadTest { tester.send("doc1"); tester.tick(1); - tester.clock().advance(Duration.ofSeconds(16)); // Default connection ttl is 15 + tester.clock().advance(Duration.ofSeconds(31)); // Default connection ttl is 30 tester.tick(3); assertEquals(1, ioThread.oldConnections().size()); assertEquals(firstConnection, ioThread.oldConnections().get(0)); assertNotSame(firstConnection, ioThread.currentConnection()); - assertEquals(16, firstConnection.lastPollTime().toEpochMilli() / 1000); + assertEquals(31, firstConnection.lastPollTime().toEpochMilli() / 1000); // Check old connection poll pattern (exponential backoff) - assertLastPollTimeWhenAdvancing(16, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(18, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(18, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(18, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(18, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(22, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(22, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(22, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(22, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(22, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(22, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(22, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(22, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(30, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(30, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(30, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(30, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(30, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(30, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(30, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(30, 1, firstConnection, tester); + assertLastPollTimeWhenAdvancing(31, 1, firstConnection, tester); + assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester); + assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester); + assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester); + assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester); + assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); + assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); + assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); + assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); + assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); + assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); + assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); + assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); + assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); + assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); + assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); + assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); + assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); + assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); + assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); + assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); tester.clock().advance(Duration.ofSeconds(200)); tester.tick(1); |