From acb13c9f0cce8ccd4a529d3c7a3f99fe66ed0003 Mon Sep 17 00:00:00 2001 From: Frode Lundgren Date: Mon, 3 Aug 2020 23:04:09 +0200 Subject: Support multiple levels of directories for flags --- .../api/systemflags/v1/SystemFlagsDataArchive.java | 18 +++++++++++++++++- .../api/systemflags/v1/SystemFlagsDataArchiveTest.java | 13 +++++++++++++ .../flags/group-1/my-test-flag/default.json | 8 ++++++++ .../flags/group-2/my-test-flag/default.json | 8 ++++++++ .../flags/group-1/my-test-flag/default.json | 8 ++++++++ .../flags/group-2/my-other-test-flag/default.json | 8 ++++++++ 6 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-1/my-test-flag/default.json create mode 100644 controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-2/my-test-flag/default.json create mode 100644 controller-api/src/test/resources/system-flags-multi-level/flags/group-1/my-test-flag/default.json create mode 100644 controller-api/src/test/resources/system-flags-multi-level/flags/group-2/my-other-test-flag/default.json diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java index e6310cc6432..a00992da815 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java @@ -45,6 +45,11 @@ import static com.yahoo.yolean.Exceptions.uncheck; * The flag files must reside in a 'flags/' root directory containing a directory for each flag name: * {@code ./flags//*.json} * + * Optionally, there can be an arbitrary number of directories "between" 'flags/' root directory and + * the flag name directory: + * {@code ./flags/onelevel//*.json} + * {@code ./flags/onelevel/anotherlevel//*.json} + * * @author bjorncs */ public class SystemFlagsDataArchive { @@ -155,7 +160,7 @@ public class SystemFlagsDataArchive { if (!filename.endsWith(".json")) { throw new IllegalArgumentException(String.format("Only JSON files are allowed in 'flags/' directory (found '%s')", filePath.toString())); } - FlagId directoryDeducedFlagId = new FlagId(filePath.getName(1).toString()); + FlagId directoryDeducedFlagId = new FlagId(filePath.getName(filePath.getNameCount()-2).toString()); FlagData flagData; if (rawData.isBlank()) { flagData = new FlagData(directoryDeducedFlagId); @@ -178,6 +183,13 @@ public class SystemFlagsDataArchive { "\nSee https://git.ouroath.com/vespa/hosted-feature-flags for more info on the JSON syntax"); } } + + if (builder.hasFile(filename, flagData)) { + throw new IllegalArgumentException( + String.format("Flag data file in '%s' contains redundant flag data for id '%s' already set in another directory!", + filePath, flagData.id())); + } + builder.addFile(filename, flagData); } @@ -236,6 +248,10 @@ public class SystemFlagsDataArchive { return this; } + public boolean hasFile(String filename, FlagData data) { + return files.containsKey(data.id()) && files.get(data.id()).containsKey(filename); + } + public SystemFlagsDataArchive build() { Map> copy = new TreeMap<>(); files.forEach((flagId, map) -> copy.put(flagId, new TreeMap<>(map))); diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java index fad1e944e36..4190c4cb635 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java @@ -84,6 +84,19 @@ public class SystemFlagsDataArchiveTest { assertArchiveReturnsCorrectTestFlagDataForTarget(archive); } + @Test + public void supports_multi_level_flags_directory() { + var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags-multi-level/")); + assertFlagDataHasValue(archive, MY_TEST_FLAG, mainControllerTarget, "default"); + } + + @Test + public void duplicated_flagdata_is_detected() { + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Flag data file in 'flags/group-1/my-test-flag/default.json' contains redundant flag data for id 'my-test-flag' already set in another directory!"); + var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags-multi-level-with-duplicated-flagdata/")); + } + @Test public void empty_files_are_handled_as_no_flag_data_for_target() { var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags/")); diff --git a/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-1/my-test-flag/default.json b/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-1/my-test-flag/default.json new file mode 100644 index 00000000000..5924eb860c0 --- /dev/null +++ b/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-1/my-test-flag/default.json @@ -0,0 +1,8 @@ +{ + "id" : "my-test-flag", + "rules" : [ + { + "value" : "default" + } + ] +} \ No newline at end of file diff --git a/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-2/my-test-flag/default.json b/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-2/my-test-flag/default.json new file mode 100644 index 00000000000..5924eb860c0 --- /dev/null +++ b/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-2/my-test-flag/default.json @@ -0,0 +1,8 @@ +{ + "id" : "my-test-flag", + "rules" : [ + { + "value" : "default" + } + ] +} \ No newline at end of file diff --git a/controller-api/src/test/resources/system-flags-multi-level/flags/group-1/my-test-flag/default.json b/controller-api/src/test/resources/system-flags-multi-level/flags/group-1/my-test-flag/default.json new file mode 100644 index 00000000000..5924eb860c0 --- /dev/null +++ b/controller-api/src/test/resources/system-flags-multi-level/flags/group-1/my-test-flag/default.json @@ -0,0 +1,8 @@ +{ + "id" : "my-test-flag", + "rules" : [ + { + "value" : "default" + } + ] +} \ No newline at end of file diff --git a/controller-api/src/test/resources/system-flags-multi-level/flags/group-2/my-other-test-flag/default.json b/controller-api/src/test/resources/system-flags-multi-level/flags/group-2/my-other-test-flag/default.json new file mode 100644 index 00000000000..e30485b755c --- /dev/null +++ b/controller-api/src/test/resources/system-flags-multi-level/flags/group-2/my-other-test-flag/default.json @@ -0,0 +1,8 @@ +{ + "id" : "my-other-test-flag", + "rules" : [ + { + "value" : "default" + } + ] +} \ No newline at end of file -- cgit v1.2.3 From 9e675e529dd230a7191d27f66efc6f2ddf2d2fda Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Thu, 17 Sep 2020 11:34:44 +0000 Subject: add PackedLabels --- eval/src/vespa/eval/tensor/mixed/packed_labels.cpp | 64 ++++++++++++++++++++++ eval/src/vespa/eval/tensor/mixed/packed_labels.h | 37 +++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 eval/src/vespa/eval/tensor/mixed/packed_labels.cpp create mode 100644 eval/src/vespa/eval/tensor/mixed/packed_labels.h diff --git a/eval/src/vespa/eval/tensor/mixed/packed_labels.cpp b/eval/src/vespa/eval/tensor/mixed/packed_labels.cpp new file mode 100644 index 00000000000..24a1c80a4cb --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_labels.cpp @@ -0,0 +1,64 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "packed_labels.h" +#include + +namespace vespalib::eval { + +int32_t +PackedLabels::find_label(vespalib::stringref to_find) const +{ + uint32_t lo = 0; + uint32_t hi = num_labels(); + while (lo < hi) { + uint32_t mid = (lo + hi) / 2; + if (label_value(mid) < to_find) { + lo = mid + 1; + } else { + hi = mid; + } + } + assert(lo == hi); + if (lo < num_labels() && label_value(lo) == to_find) { + return lo; + } + return -1; +} + +vespalib::stringref +PackedLabels::label_value(uint32_t index) const +{ + assert(index < num_labels()); + + auto p = get_label_start(index); + auto sz = get_label_size(index); + return vespalib::stringref(p, sz); +} + +void +PackedLabels::validate_labels(uint32_t num_labels) +{ + assert(num_labels == _offsets.size()-1); + for (uint32_t i = 0; i < num_labels; ++i) { + assert(_offsets[i] < _offsets[i+1]); + uint32_t last_byte_index = _offsets[i+1] - 1; + assert(_label_store[last_byte_index] == 0); + } + assert(_label_store.size() == _offsets[num_labels]); + for (uint32_t i = 0; i+1 < num_labels; ++i) { + assert(label_value(i) < label_value(i+1)); + } +} + +const char * +PackedLabels::get_label_start(uint32_t index) const { + uint32_t offset = _offsets[index]; + return &_label_store[offset]; +} + +uint32_t +PackedLabels::get_label_size(uint32_t index) const { + return _offsets[index+1] - _offsets[index] - 1; +} + +} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_labels.h b/eval/src/vespa/eval/tensor/mixed/packed_labels.h new file mode 100644 index 00000000000..07db1aac8de --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_labels.h @@ -0,0 +1,37 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include +#include + +namespace vespalib::eval { + +class PackedLabels { +public: + PackedLabels(uint32_t num_labels, + ConstArrayRef offsets, + ConstArrayRef label_store) + : _offsets(offsets), + _label_store(label_store) + { + validate_labels(num_labels); + } + + uint32_t num_labels() const { return _offsets.size() - 1; } + + // returns -1 if the given label value cannot be found + int32_t find_label(vespalib::stringref value) const; + + vespalib::stringref label_value(uint32_t index) const; + +private: + const ConstArrayRef _offsets; + const ConstArrayRef _label_store; + + void validate_labels(uint32_t num_labels); + const char *get_label_start(uint32_t index) const; + uint32_t get_label_size(uint32_t index) const; +}; + +} // namespace -- cgit v1.2.3 From e013f2c2abc23f3e9dfb4819e7a17609417944ac Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Thu, 17 Sep 2020 11:35:10 +0000 Subject: add PackedMappings --- .../vespa/eval/tensor/mixed/packed_mappings.cpp | 180 +++++++++++++++++++++ eval/src/vespa/eval/tensor/mixed/packed_mappings.h | 94 +++++++++++ 2 files changed, 274 insertions(+) create mode 100644 eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp create mode 100644 eval/src/vespa/eval/tensor/mixed/packed_mappings.h diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp new file mode 100644 index 00000000000..b11c8d08097 --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp @@ -0,0 +1,180 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "packed_mappings.h" +#include + +namespace vespalib::eval { + + +int32_t +PackedMappings::subspace_of_address(const Address &address) const +{ + int32_t idx = sortid_of_address(address); + if (idx < 0) { + return -1; + } + uint32_t internal_idx = idx; + assert (internal_idx < _num_mappings); + return subspace_of_sortid(internal_idx); +} + +int32_t +PackedMappings::sortid_of_address(const Address &address) const +{ + if (_num_dims == 0) return 0; + assert(address.size() == _num_dims); + std::vector to_find; + to_find.reserve(_num_dims); + for (const auto & label_value : address) { + int32_t label_idx = _label_store.find_label(label_value); + if (label_idx < 0) { + return -1; + } + to_find.push_back(label_idx); + } + return sortid_of_enums(to_find); +} + +int32_t +PackedMappings::sortid_of_enums(const InternalAddress &address) const +{ + if (_num_dims == 0) return 0; + assert(address.size() == _num_dims); + const uint32_t * to_find = &address[0]; + uint32_t lo = 0; + uint32_t hi = _num_mappings; + while (lo < hi) { + uint32_t mid = (lo + hi) / 2; + if (enums_compare(ptr_of_sortid(mid), to_find) < 0) { + lo = mid + 1; + } else { + hi = mid; + } + } + assert(lo == hi); + if ((lo < _num_mappings) && + (enums_compare(ptr_of_sortid(lo), to_find) == 0)) + { + return lo; + } + return -1; +} + +std::vector +PackedMappings::address_of_sortid(uint32_t internal_index) const +{ + std::vector result; + fill_by_sortid(internal_index, result); + return result; +} + +std::vector +PackedMappings::address_of_subspace(uint32_t subspace_index) const +{ + std::vector result; + fill_by_subspace(subspace_index, result); + return result; +} + +uint32_t +PackedMappings::fill_by_subspace(uint32_t subspace_index, InternalAddress &address) const +{ + assert(subspace_index < _num_mappings); + uint32_t internal_index = sortid_of_subspace(subspace_index); + uint32_t subspace = fill_by_sortid(internal_index, address); + assert(subspace == subspace_index); + return internal_index; +} + +uint32_t +PackedMappings::fill_by_sortid(uint32_t internal_index, InternalAddress &address) const +{ + assert(internal_index < _num_mappings); + uint32_t offset = offset_of_mapping_data(internal_index); + address.resize(_num_dims); + for (uint32_t i = 0; i < _num_dims; ++i) { + address[i] = _int_store[offset++]; + } + return _int_store[offset]; +} + +/** returns subspace_index */ +uint32_t +PackedMappings::fill_by_sortid(uint32_t internal_index, Address &address) const +{ + assert(internal_index < _num_mappings); + uint32_t offset = offset_of_mapping_data(internal_index); + address.resize(_num_dims); + for (uint32_t i = 0; i < _num_dims; ++i) { + uint32_t label_idx = _int_store[offset++]; + address[i] = _label_store.label_value(label_idx); + } + return _int_store[offset]; +} + +/** returns internal_index */ +uint32_t +PackedMappings::fill_by_subspace(uint32_t subspace_index, Address &address) const +{ + assert(subspace_index < _num_mappings); + uint32_t internal_index = sortid_of_subspace(subspace_index); + uint32_t subspace = fill_by_sortid(internal_index, address); + assert(subspace == subspace_index); + return internal_index; +} + +void +PackedMappings::validate() const +{ + assert((_num_mappings * (2 + _num_dims)) == _int_store.size()); + auto iter = _int_store.cbegin(); + for (uint32_t i = 0; i < _num_mappings; ++i) { + uint32_t internal_index = *iter++; + assert(internal_index < _num_mappings); + } + std::vector prev; + std::vector next; + for (uint32_t i = 0; i < _num_mappings; ++i) { + next.clear(); + for (uint32_t j = 0; j < _num_dims; ++j) { + uint32_t label_index = *iter++; + next.push_back(label_index); + assert(label_index < _label_store.num_labels()); + } + if (_num_dims == 0) { + assert(next == prev); + assert(i == 0); + assert(_num_mappings == 1); + } else { + assert(prev < next); + } + std::swap(prev, next); + uint32_t subspace_index = *iter++; + assert(subspace_index < _num_mappings); + } + assert(iter == _int_store.cend()); +} + +std::vector +PackedMappings::enums_of_subspace(uint32_t subspace_index) const +{ + assert(subspace_index < _num_mappings); + uint32_t internal_index = sortid_of_subspace(subspace_index); + return enums_of_sortid(internal_index); +} + +std::vector +PackedMappings::enums_of_sortid(uint32_t internal_index) const +{ + std::vector result; + result.reserve(_num_dims); + assert(internal_index < _num_mappings); + uint32_t offset = offset_of_mapping_data(internal_index); + for (uint32_t i = 0; i < _num_dims; ++i) { + result.push_back(_int_store[offset++]); + } + return result; +} + + +} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings.h b/eval/src/vespa/eval/tensor/mixed/packed_mappings.h new file mode 100644 index 00000000000..18c70f01eb6 --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings.h @@ -0,0 +1,94 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "packed_labels.h" +#include + +namespace vespalib::eval { + +class PackedMappings { +public: + using Address = std::vector; + using InternalAddress = std::vector; + + uint32_t size() const { return _num_mappings; } + uint32_t num_sparse_dims() const { return _num_dims; } + + // returns -1 if mapping does not contain address + int32_t subspace_of_address(const Address &address) const; + int32_t subspace_of_enums(const InternalAddress &address) const; + int32_t sortid_of_address(const Address &address) const; + int32_t sortid_of_enums(const InternalAddress &address) const; + + Address address_of_sortid(uint32_t internal_index) const; + Address address_of_subspace(uint32_t subspace_index) const; + + /** returns sortid */ + uint32_t fill_by_subspace(uint32_t subspace_index, Address &address) const; + uint32_t fill_by_subspace(uint32_t subspace_index, InternalAddress &address) const; + + /** returns subspace_index */ + uint32_t fill_by_sortid(uint32_t sortid, Address &address) const; + uint32_t fill_by_sortid(uint32_t sortid, InternalAddress &address) const; + + InternalAddress enums_of_sortid(uint32_t internal_index) const; + InternalAddress enums_of_subspace(uint32_t subspace_index) const; + + int enums_compare(const uint32_t *a, const uint32_t *b) const { + for (size_t i = 0; i < _num_dims; ++i) { + if (a[i] < b[i]) return -1; + if (a[i] > b[i]) return 1; + } + return 0; + } + + const PackedLabels & label_store() const { return _label_store; } +private: + PackedMappings(uint32_t num_dims, uint32_t num_mappings, + ConstArrayRef int_store, + PackedLabels label_store) + : _num_dims(num_dims), + _num_mappings(num_mappings), + _int_store(int_store), + _label_store(label_store) + { + validate(); + } + friend class PackedMappingsBuilder; + + void validate() const; + + const uint32_t _num_dims; + const uint32_t _num_mappings; + /* + _int_store contains data corresponding to this model: + struct IntStore { + // map to index in next table: + uint32_t index_of_subspace[num_mappings]; + // sorted lexicographically by label_enums: + struct MappingData { + uint32_t label_enums[num_dims]; + uint32_t subspace_index; + } mappings[num_mappings]; + }; + */ + const ConstArrayRef _int_store; + const PackedLabels _label_store; + + uint32_t offset_of_mapping_data(uint32_t idx) const { + return (idx * (1 + _num_dims)) + _num_mappings; + } + uint32_t subspace_of_sortid(uint32_t internal_index) const { + uint32_t offset = offset_of_mapping_data(internal_index); + return _int_store[offset + _num_dims]; + } + uint32_t sortid_of_subspace(uint32_t subspace_index) const { + return _int_store[subspace_index]; + } + const uint32_t * ptr_of_sortid(uint32_t internal_index) const { + return &_int_store[offset_of_mapping_data(internal_index)]; + } +}; + +} // namespace -- cgit v1.2.3 From 291031cf8054e13d7ec6d3cf4655af8638683085 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Thu, 17 Sep 2020 11:35:25 +0000 Subject: add PackedMappingsBuilder --- .../eval/tensor/mixed/packed_mappings_builder.cpp | 118 +++++++++++++++++++++ .../eval/tensor/mixed/packed_mappings_builder.h | 42 ++++++++ 2 files changed, 160 insertions(+) create mode 100644 eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp create mode 100644 eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp new file mode 100644 index 00000000000..d149539bd39 --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp @@ -0,0 +1,118 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "packed_mappings_builder.h" +#include + +namespace vespalib::eval { + +PackedMappingsBuilder::~PackedMappingsBuilder() = default; + +uint32_t +PackedMappingsBuilder::add_mapping_for(SparseAddress address) +{ + assert(address.size() == _num_dims); + for (auto & label_value : address) { + // store label string in our own set: + auto iter = _labels.insert(label_value).first; + label_value = *iter; + } + + uint32_t next_index = _mappings.size(); + IndexMap::value_type new_val(address, next_index); + auto iter = _mappings.insert(new_val).first; + return iter->second; +} + + +size_t +PackedMappingsBuilder::extra_memory() const +{ + size_t int_store_cnt = (2 + _num_dims) * _mappings.size(); + size_t int_store_size = int_store_cnt * sizeof(uint32_t); + size_t label_cnt = _labels.size(); + size_t label_offsets_size = (1 + label_cnt) * sizeof(uint32_t); + size_t label_bytes = 0; + for (const auto & label_value : _labels) { + label_bytes += (label_value.size() + 1); + } + size_t extra_size = int_store_size + label_offsets_size + label_bytes; + return extra_size; +} + +PackedMappings +PackedMappingsBuilder::target_memory(char *mem_start, char *mem_end) const +{ + size_t int_store_cnt = (2 + _num_dims) * _mappings.size(); + size_t int_store_size = int_store_cnt * sizeof(uint32_t); + size_t label_cnt = _labels.size(); + size_t label_offsets_size = (1 + label_cnt) * sizeof(uint32_t); + + size_t label_bytes = 0; + for (const auto & label_value : _labels) { + label_bytes += (label_value.size() + 1); + } + + ssize_t needs_sz = int_store_size + label_offsets_size + label_bytes; + ssize_t avail_sz = mem_end - mem_start; + assert(needs_sz <= avail_sz); + + uint32_t * int_store_mem = (uint32_t *) mem_start; + uint32_t * offsets_mem = (uint32_t *) (mem_start + int_store_size); + char * labels_mem = mem_start + int_store_size + label_offsets_size; + + ArrayRef int_store_data(int_store_mem, int_store_cnt); + ArrayRef label_offsets(offsets_mem, 1 + label_cnt); + ArrayRef labels_data(labels_mem, label_bytes); + + size_t byte_idx = 0; + size_t label_num = 0; + for (const auto & label_value : _labels) { + label_offsets[label_num++] = byte_idx; + size_t len_with_zero = label_value.size() + 1; + memcpy(&labels_data[byte_idx], label_value.data(), len_with_zero); + byte_idx += len_with_zero; + } + assert(label_num == label_cnt); + label_offsets[label_num] = byte_idx; + + PackedLabels stored_labels(label_cnt, label_offsets, labels_data); + + size_t mapping_idx = 0; + size_t int_store_offset = _mappings.size(); + for (const auto & kv : _mappings) { + const SparseAddress & k = kv.first; + uint32_t v = kv.second; + for (const auto & label_value : k) { + int32_t label_idx = stored_labels.find_label(label_value); + assert(label_idx >= 0); + assert(uint32_t(label_idx) < label_num); + int_store_data[int_store_offset++] = label_idx; + } + int_store_data[int_store_offset++] = v; + int_store_data[v] = mapping_idx++; + } + assert(int_store_offset == int_store_cnt); + assert(mapping_idx == _mappings.size()); + + return PackedMappings(_num_dims, _mappings.size(), + int_store_data, stored_labels); +} + +std::unique_ptr +PackedMappingsBuilder::build_mappings() const +{ + size_t meta_size = sizeof(PackedMappings); + size_t total_size = meta_size + extra_memory(); + + char * mem = (char *) operator new(total_size); + auto meta_data = target_memory(mem + meta_size, mem + total_size); + + PackedMappings * built = new (mem) PackedMappings(meta_data); + + return std::unique_ptr(built); +} + +} // namespace + + + diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h new file mode 100644 index 00000000000..b78c1cf17f7 --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h @@ -0,0 +1,42 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "packed_mappings.h" +#include +#include +#include +#include +#include + +namespace vespalib::eval { + +class PackedMappingsBuilder { +public: + using SparseAddress = std::vector; + + PackedMappingsBuilder(uint32_t num_sparse_dims) + : _num_dims(num_sparse_dims), + _labels(), + _mappings() + {} + + ~PackedMappingsBuilder(); + + uint32_t add_mapping_for(SparseAddress address); + + std::unique_ptr build_mappings() const; + + size_t extra_memory() const; + PackedMappings target_memory(char *mem_start, char *mem_end) const; + + uint32_t num_sparse_dims() const { return _num_dims; } + size_t size() const { return _mappings.size(); } +private: + uint32_t _num_dims; + std::set _labels; + using IndexMap = std::map; + IndexMap _mappings; +}; + +} // namespace -- cgit v1.2.3 From af224e39fb79678acef43f83492925551ca9273a Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Thu, 17 Sep 2020 11:36:10 +0000 Subject: add PackedMixedTensor --- .../eval/tensor/mixed/packed_mixed_tensor.cpp | 118 +++++++++++++++++++++ .../vespa/eval/tensor/mixed/packed_mixed_tensor.h | 43 ++++++++ 2 files changed, 161 insertions(+) create mode 100644 eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp create mode 100644 eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp new file mode 100644 index 00000000000..42a9bf78846 --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp @@ -0,0 +1,118 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "packed_mixed_tensor.h" + +namespace vespalib::eval { + +class PackedMixedTensorIndexView : public NewValue::Index::View +{ +private: + const PackedMappings& _mappings; + const std::vector _view_dims; + std::vector _lookup_enums; + std::vector _full_enums; + std::vector _lookup_addr; + std::vector _full_address; + size_t _index; + + size_t num_full_dims() const { return _mappings.num_sparse_dims(); } + size_t num_view_dims() const { return _view_dims.size(); } + size_t num_rest_dims() const { return num_full_dims() - num_view_dims(); } +public: + PackedMixedTensorIndexView(const PackedMappings& mappings, + const std::vector &dims) + : _mappings(mappings), + _view_dims(dims), + _lookup_enums(), + _lookup_addr(), + _full_address(), + _index(0) + { + _lookup_enums.reserve(num_view_dims()); + _lookup_addr.reserve(num_view_dims()); + _full_enums.resize(num_full_dims()); + _full_address.resize(num_full_dims()); + } + + void lookup(const std::vector &addr) override; + bool next_result(const std::vector &addr_out, size_t &idx_out) override; + ~PackedMixedTensorIndexView() override = default; +}; + +void +PackedMixedTensorIndexView::lookup(const std::vector &addr) +{ + _index = 0; + assert(addr.size() == num_view_dims()); + _lookup_enums.clear(); + _lookup_addr.clear(); + // printf("lookup %zu/%zu dims:", num_view_dims(), num_full_dims()); + for (const vespalib::stringref * label_ptr : addr) { + int32_t label_enum = _mappings.label_store().find_label(*label_ptr); + if (label_enum < 0) { + // cannot match + _index = _mappings.size(); + break; + } + _lookup_enums.push_back(label_enum); + _lookup_addr.push_back(*label_ptr); + // printf(" '%s'", label_ptr->data()); + } + // printf(" [in %u mappings]\n", _mappings.size()); +} + +bool +PackedMixedTensorIndexView::next_result(const std::vector &addr_out, size_t &idx_out) +{ + assert(addr_out.size() == num_rest_dims()); + while (_index < _mappings.size()) { + idx_out = _mappings.fill_by_subspace(_index++, _full_enums); + bool couldmatch = true; + size_t vd_idx = 0; + size_t ao_idx = 0; + for (size_t i = 0; i < num_full_dims(); ++i) { + if (vd_idx < num_view_dims()) { + size_t next_view_dim = _view_dims[vd_idx]; + if (i == next_view_dim) { + if (_lookup_enums[vd_idx] == _full_enums[i]) { + // match in this dimension + ++vd_idx; + continue; + } else { + // does not match + couldmatch = false; + break; + } + } + } + // not a view dimension: + uint32_t label_enum = _full_enums[i]; + auto label_value = _mappings.label_store().label_value(label_enum); + _full_address[i] = label_value; + *addr_out[ao_idx] = label_value; + ++ao_idx; + } + if (couldmatch) { + // printf("matches at %zu/%u\n", _index, _mappings.size()); + assert(vd_idx == num_view_dims()); + assert(ao_idx == num_rest_dims()); + return true; + } + } + // printf("no more matches %zu/%u\n", _index, _mappings.size()); + return false; +} + +PackedMixedTensor::~PackedMixedTensor() = default; + +std::unique_ptr +PackedMixedTensor::create_view(const std::vector &dims) const +{ + for (size_t i = 1; i < dims.size(); ++i) { + assert(dims[i-1] < dims[i]); + assert(dims[i] < _mappings.num_sparse_dims()); + } + return std::make_unique(_mappings, dims); +} + +} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h new file mode 100644 index 00000000000..67684097380 --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h @@ -0,0 +1,43 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include +#include +#include + +#include +#include + +namespace vespalib::eval { + +class PackedMixedTensor : public NewValue, public NewValue::Index +{ +private: + const ValueType _type; + const TypedCells _cells; + const PackedMappings _mappings; + + PackedMixedTensor(const ValueType &type, TypedCells cells, + PackedMappings mappings) + : _type(type), + _cells(cells), + _mappings(mappings) + {} + + template friend class PackedMixedBuilder; + +public: + ~PackedMixedTensor() override; + + // NewValue API: + const ValueType &type() const override { return _type; } + const NewValue::Index &index() const override { return *this; } + TypedCells cells() const override { return _cells; } + + // NewValue::Index API: + size_t size() const override { return _mappings.size(); } + std::unique_ptr create_view(const std::vector &dims) const override; +}; + +} // namespace -- cgit v1.2.3 From 97455ac1c5db9036615465505631823d906d2d13 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Thu, 17 Sep 2020 11:36:40 +0000 Subject: add PackedMixedBuilder --- .../eval/tensor/mixed/packed_mixed_builder.cpp | 55 ++++++++++++++++++++++ .../vespa/eval/tensor/mixed/packed_mixed_builder.h | 36 ++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.cpp create mode 100644 eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.cpp new file mode 100644 index 00000000000..cd0310762d2 --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.cpp @@ -0,0 +1,55 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "packed_mixed_builder.h" + +namespace vespalib::eval { + +template +ArrayRef +PackedMixedBuilder::add_subspace(const std::vector &addr) +{ + uint32_t idx = _mappings_builder.add_mapping_for(addr); + size_t offset = idx * _subspace_size; + assert(offset <= _cells.size()); + if (offset == _cells.size()) { + _cells.resize(offset + _subspace_size); + } + return ArrayRef(&_cells[offset], _subspace_size); +} + + +template +std::unique_ptr +PackedMixedBuilder::build(std::unique_ptr> self) +{ + size_t meta_size = sizeof(PackedMixedTensor); + size_t mappings_size = _mappings_builder.extra_memory(); + // align: + mappings_size += 15ul; + mappings_size &= ~15ul; + size_t cells_size = sizeof(T) * _cells.size(); + size_t total_size = sizeof(PackedMixedTensor) + mappings_size + cells_size; + + char *mem = (char *) operator new(total_size); + char *mappings_mem = mem + meta_size; + char *cells_mem = mappings_mem + mappings_size; + + // fill mapping data: + auto mappings = _mappings_builder.target_memory(mappings_mem, cells_mem); + + // copy cells: + memcpy(cells_mem, &_cells[0], cells_size); + ConstArrayRef cells((T *)cells_mem, _cells.size()); + + PackedMixedTensor * built = + new (mem) PackedMixedTensor(_type, TypedCells(cells), mappings); + + // keep "this" alive until this point: + (void) self; + return std::unique_ptr(built); +} + +template class PackedMixedBuilder; +template class PackedMixedBuilder; + +} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h new file mode 100644 index 00000000000..200c8f19482 --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h @@ -0,0 +1,36 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "packed_mixed_tensor.h" + +namespace vespalib::eval { + +template +class PackedMixedBuilder : public ValueBuilder +{ +private: + const ValueType & _type; + size_t _subspace_size; + std::vector _cells; + PackedMappingsBuilder _mappings_builder; +public: + PackedMixedBuilder(const ValueType &type, + size_t num_mapped_in, + size_t subspace_size_in, + size_t expect_subspaces) + : _type(type), + _subspace_size(subspace_size_in), + _cells(), + _mappings_builder(num_mapped_in) + { + _cells.reserve(_subspace_size * expect_subspaces); + } + + ~PackedMixedBuilder() override = default; + + ArrayRef add_subspace(const std::vector &addr) override; + std::unique_ptr build(std::unique_ptr> self) override; +}; + +} // namespace -- cgit v1.2.3 From 14665b8624b56079765eed9070deaa85905852fa Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Thu, 17 Sep 2020 11:37:28 +0000 Subject: add PackedMixedFactory --- .../eval/tensor/mixed/packed_mixed_factory.cpp | 33 ++++++++++++++++++++++ .../vespa/eval/tensor/mixed/packed_mixed_factory.h | 16 +++++++++++ 2 files changed, 49 insertions(+) create mode 100644 eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.cpp create mode 100644 eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.h diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.cpp new file mode 100644 index 00000000000..db09db48a4c --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.cpp @@ -0,0 +1,33 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "packed_mixed_factory.h" +#include "packed_mixed_builder.h" + +#include + +namespace vespalib::eval { + +namespace { + +struct CreatePackedMixedBuilder { + template + static std::unique_ptr invoke(const ValueType &type, Args &&...args) + { + assert(check_cell_type(type.cell_type())); + return std::make_unique>(type, std::forward(args)...); + } +}; + +} // namespace + +std::unique_ptr +PackedMixedFactory::create_value_builder_base(const ValueType &type, + size_t num_mapped_in, + size_t subspace_size_in, + size_t expect_subspaces) const +{ + return typify_invoke<1,TypifyCellType,CreatePackedMixedBuilder>(type.cell_type(), + type, num_mapped_in, subspace_size_in, expect_subspaces); +} + +} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.h new file mode 100644 index 00000000000..3be266541cc --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.h @@ -0,0 +1,16 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include +#include +#include + +namespace vespalib::eval { + +struct PackedMixedFactory : ValueBuilderFactory { + ~PackedMixedFactory() override {} +protected: + std::unique_ptr create_value_builder_base(const ValueType &type, + size_t num_mapped_in, size_t subspace_size_in, size_t expect_subspaces) const override; +}; + +} // namespace -- cgit v1.2.3 From 58c5db1ee90f9472c9ff3fa4554dbcc571d0ee2f Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Thu, 17 Sep 2020 11:38:23 +0000 Subject: add unit tests for mixed/Packed classes --- .../packed_mappings/packed_mappings_test.cpp | 223 +++++++++++++++++++++ .../tensor/packed_mappings/packed_mixed_test.cpp | 152 ++++++++++++++ 2 files changed, 375 insertions(+) create mode 100644 eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp create mode 100644 eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp diff --git a/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp b/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp new file mode 100644 index 00000000000..0b668986cb6 --- /dev/null +++ b/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp @@ -0,0 +1,223 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace vespalib::eval; + +namespace { + +uint32_t random_range(uint32_t from, uint32_t to) { + assert(from + 1 < to); + int unif = rand() % (to - from); + return from + unif; +} + +const char *mixed_tensor_types[] = { + "tensor(x{})", + "tensor(a{},b{},c{},d{},e{},f{})", + "tensor(x{},y{})", + "tensor(x{},z[3])", + "tensor(w[5],x{},y{},z[3])" +}; + +const char *float_tensor_types[] = { + "tensor(x{})", + "tensor(x{},y{})", + "tensor(x{},z[3])", + "tensor(w[5],x{},y{},z[3])", + "tensor(z[2])", + "tensor()" +}; + + vespalib::string label1(""), + label2("foo"), + label3("bar"); + vespalib::string label4("foobar"), + label5("barfoo"), + label6("other"); + vespalib::string label7("long text number one"), + label8("long text number two"), + label9("long text number three"); + +std::vector +generate_random_address(uint32_t dims) +{ + std::vector foo(dims, label1); + for (auto & ref : foo) { + size_t pct = random_range(0, 100); + if (pct < 5) { ref = label1; } + else if (pct < 30) { ref = label2; } + else if (pct < 55) { ref = label3; } + else if (pct < 65) { ref = label4; } + else if (pct < 75) { ref = label5; } + else if (pct < 85) { ref = label6; } + else if (pct < 90) { ref = label7; } + else if (pct < 95) { ref = label8; } + else { ref = label9; } + } + return foo; +} + +} // namespace + +class MappingsBuilderTest : public ::testing::Test { +public: + std::unique_ptr builder; + std::unique_ptr built; + + MappingsBuilderTest() = default; + + virtual ~MappingsBuilderTest() = default; + + void build_and_compare() { + ASSERT_TRUE(builder); + built = builder->build_mappings(); + ASSERT_TRUE(built); + EXPECT_EQ(builder->num_sparse_dims(), built->num_sparse_dims()); + EXPECT_EQ(builder->size(), built->size()); + for (size_t idx = 0; idx < built->size(); ++idx) { + auto got = built->address_of_subspace(idx); + printf("Got address:"); + for (auto ref : got) { + printf(" '%s'", ref.data()); + } + uint32_t original = builder->add_mapping_for(got); + printf(" -> %u\n", original); + EXPECT_EQ(idx, original); + } + } +}; + +TEST_F(MappingsBuilderTest, empty_mapping) +{ + for (uint32_t dims : { 0, 1, 2, 3 }) { + builder = std::make_unique(dims); + build_and_compare(); + } +} + +TEST_F(MappingsBuilderTest, just_one) +{ + vespalib::string label("foobar"); + for (uint32_t dims : { 0, 1, 2, 3, 7 }) { + builder = std::make_unique(dims); + std::vector foo(dims, label); + uint32_t idx = builder->add_mapping_for(foo); + EXPECT_EQ(idx, 0); + build_and_compare(); + } +} + +TEST_F(MappingsBuilderTest, some_random) +{ + for (uint32_t dims : { 1, 2, 5 }) { + builder = std::make_unique(dims); + uint32_t cnt = random_range(dims*5, dims*20); + printf("Generate %u addresses for %u dims\n", cnt, dims); + for (uint32_t i = 0; i < cnt; ++i) { + auto foo = generate_random_address(dims); + uint32_t idx = builder->add_mapping_for(foo); + EXPECT_LE(idx, i); + } + build_and_compare(); + } +} + +class MixedBuilderTest : public ::testing::Test { +public: + std::unique_ptr> builder; + std::unique_ptr built; + + MixedBuilderTest() = default; + + virtual ~MixedBuilderTest() = default; + + size_t expected_value = 0; + + void build_and_compare(size_t expect_size) { + built.reset(nullptr); + EXPECT_FALSE(built); + ASSERT_TRUE(builder); + built = builder->build(std::move(builder)); + EXPECT_FALSE(builder); + ASSERT_TRUE(built); + EXPECT_EQ(built->index().size(), expect_size); + auto cells = built->cells().typify(); + for (float f : cells) { + float expect = ++expected_value; + EXPECT_EQ(f, expect); + } + } +}; + +TEST_F(MixedBuilderTest, empty_mapping) +{ + for (auto type_spec : mixed_tensor_types) { + ValueType type = ValueType::from_spec(type_spec); + size_t dims = type.count_mapped_dimensions(); + size_t dsss = type.dense_subspace_size(); + EXPECT_GT(dims, 0); + EXPECT_GT(dsss, 0); + builder = std::make_unique>(type, dims, dsss, 3); + build_and_compare(0); + } +} + +TEST_F(MixedBuilderTest, just_one) +{ + size_t counter = 0; + for (auto type_spec : float_tensor_types) { + ValueType type = ValueType::from_spec(type_spec); + size_t dims = type.count_mapped_dimensions(); + size_t dsss = type.dense_subspace_size(); + EXPECT_GT(dsss, 0); + builder = std::make_unique>(type, dims, dsss, 3); + auto address = generate_random_address(dims); + auto ref = builder->add_subspace(address); + EXPECT_EQ(ref.size(), dsss); + for (size_t i = 0; i < ref.size(); ++i) { + ref[i] = ++counter; + } + build_and_compare(1); + } +} + +TEST_F(MixedBuilderTest, some_random) +{ + size_t counter = 0; + for (auto type_spec : mixed_tensor_types) { + ValueType type = ValueType::from_spec(type_spec); + uint32_t dims = type.count_mapped_dimensions(); + uint32_t dsss = type.dense_subspace_size(); + EXPECT_GT(dims, 0); + EXPECT_GT(dsss, 0); + builder = std::make_unique>(type, dims, dsss, 3); + + uint32_t cnt = random_range(dims*5, dims*20); + printf("MixBuild: generate %u addresses for %u dims\n", cnt, dims); + std::set> seen; + for (uint32_t i = 0; i < cnt; ++i) { + auto address = generate_random_address(dims); + if (seen.insert(address).second) { + auto ref = builder->add_subspace(address); + EXPECT_EQ(ref.size(), dsss); + for (size_t j = 0; j < ref.size(); ++j) { + ref[j] = ++counter; + } + } + } + printf("MixBuild: generated %zu unique addresses\n", seen.size()); + build_and_compare(seen.size()); + } +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp b/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp new file mode 100644 index 00000000000..fa3597e9858 --- /dev/null +++ b/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp @@ -0,0 +1,152 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include +#include +#include +#include + +using namespace vespalib::eval; +using namespace vespalib::eval::test; + +std::vector layouts = { + {}, + {x(3)}, + {x(3),y(5)}, + {x(3),y(5),z(7)}, + float_cells({x(3),y(5),z(7)}), + {x({"a","b","c"})}, + {x({"a","b","c"}),y({"foo","bar"})}, + {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), + {x(3),y({"foo", "bar"}),z(7)}, + {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) +}; + +TEST(PackedMixedTest, packed_mixed_tensors_can_be_converted_from_and_to_tensor_spec) { + for (const auto &layout: layouts) { + TensorSpec expect = spec(layout, N()); + std::unique_ptr value = new_value_from_spec(expect, PackedMixedFactory()); + TensorSpec actual = spec_from_new_value(*value); + EXPECT_EQ(actual, expect); + } +} + +TEST(PackedMixedTest, packed_mixed_tensors_can_be_built_and_inspected) { + ValueType type = ValueType::from_spec("tensor(x{},y[2],z{})"); + PackedMixedFactory factory; + std::unique_ptr> builder = factory.create_value_builder(type); + float seq = 0.0; + for (vespalib::string x: {"a", "b", "c"}) { + for (vespalib::string y: {"aa", "bb"}) { + auto subspace = builder->add_subspace({x, y}); + EXPECT_EQ(subspace.size(), 2); + subspace[0] = seq + 1.0; + subspace[1] = seq + 5.0; + seq += 10.0; + } + seq += 100.0; + } + std::unique_ptr value = builder->build(std::move(builder)); + EXPECT_EQ(value->index().size(), 6); + auto view = value->index().create_view({0}); + vespalib::stringref query = "b"; + vespalib::stringref label; + size_t subspace; + view->lookup({&query}); + EXPECT_TRUE(view->next_result({&label}, subspace)); + EXPECT_EQ(label, "aa"); + EXPECT_EQ(subspace, 2); + EXPECT_TRUE(view->next_result({&label}, subspace)); + EXPECT_EQ(label, "bb"); + EXPECT_EQ(subspace, 3); + EXPECT_FALSE(view->next_result({&label}, subspace)); + + query = "c"; + view->lookup({&query}); + EXPECT_TRUE(view->next_result({&label}, subspace)); + EXPECT_EQ(label, "aa"); + EXPECT_EQ(subspace, 4); + EXPECT_TRUE(view->next_result({&label}, subspace)); + EXPECT_EQ(label, "bb"); + EXPECT_EQ(subspace, 5); + EXPECT_FALSE(view->next_result({&label}, subspace)); + + query = "notpresent"; + view->lookup({&query}); + EXPECT_FALSE(view->next_result({&label}, subspace)); + + view = value->index().create_view({1}); + query = "aa"; + view->lookup({&query}); + EXPECT_TRUE(view->next_result({&label}, subspace)); + EXPECT_EQ(label, "a"); + EXPECT_EQ(subspace, 0); + EXPECT_TRUE(view->next_result({&label}, subspace)); + EXPECT_EQ(label, "b"); + EXPECT_EQ(subspace, 2); + EXPECT_TRUE(view->next_result({&label}, subspace)); + EXPECT_EQ(label, "c"); + EXPECT_EQ(subspace, 4); + EXPECT_FALSE(view->next_result({&label}, subspace)); + + query = "bb"; + view->lookup({&query}); + EXPECT_TRUE(view->next_result({&label}, subspace)); + EXPECT_EQ(label, "a"); + EXPECT_EQ(subspace, 1); + EXPECT_TRUE(view->next_result({&label}, subspace)); + EXPECT_EQ(label, "b"); + EXPECT_EQ(subspace, 3); + EXPECT_TRUE(view->next_result({&label}, subspace)); + EXPECT_EQ(label, "c"); + EXPECT_EQ(subspace, 5); + EXPECT_FALSE(view->next_result({&label}, subspace)); + + query = "notpresent"; + view->lookup({&query}); + EXPECT_FALSE(view->next_result({&label}, subspace)); + + view = value->index().create_view({0,1}); + vespalib::stringref query_x = "b"; + vespalib::stringref query_y = "bb"; + view->lookup({&query_x, &query_y}); + EXPECT_TRUE(view->next_result({}, subspace)); + EXPECT_EQ(subspace, 3); + EXPECT_FALSE(view->next_result({}, subspace)); + + view = value->index().create_view({}); + vespalib::stringref label_x; + vespalib::stringref label_y; + view->lookup({}); + + const std::vector out({&label_x, &label_y}); + EXPECT_TRUE(view->next_result(out, subspace)); + EXPECT_EQ(label_x, "a"); + EXPECT_EQ(label_y, "aa"); + EXPECT_EQ(subspace, 0); + EXPECT_TRUE(view->next_result(out, subspace)); + EXPECT_EQ(label_x, "a"); + EXPECT_EQ(label_y, "bb"); + EXPECT_EQ(subspace, 1); + EXPECT_TRUE(view->next_result(out, subspace)); + EXPECT_EQ(label_x, "b"); + EXPECT_EQ(label_y, "aa"); + EXPECT_EQ(subspace, 2); + EXPECT_TRUE(view->next_result(out, subspace)); + EXPECT_EQ(label_x, "b"); + EXPECT_EQ(label_y, "bb"); + EXPECT_EQ(subspace, 3); + EXPECT_TRUE(view->next_result(out, subspace)); + EXPECT_EQ(label_x, "c"); + EXPECT_EQ(label_y, "aa"); + EXPECT_EQ(subspace, 4); + EXPECT_TRUE(view->next_result(out, subspace)); + EXPECT_EQ(label_x, "c"); + EXPECT_EQ(label_y, "bb"); + EXPECT_EQ(subspace, 5); + EXPECT_FALSE(view->next_result(out, subspace)); +} + + +GTEST_MAIN_RUN_ALL_TESTS() -- cgit v1.2.3 From 4178231d5f23a5e5e36f117259bde1ac2da99189 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Thu, 17 Sep 2020 11:38:49 +0000 Subject: update cmake lists --- eval/CMakeLists.txt | 2 ++ eval/src/tests/tensor/packed_mappings/CMakeLists.txt | 19 +++++++++++++++++++ eval/src/vespa/eval/CMakeLists.txt | 1 + eval/src/vespa/eval/tensor/mixed/CMakeLists.txt | 11 +++++++++++ 4 files changed, 33 insertions(+) create mode 100644 eval/src/tests/tensor/packed_mappings/CMakeLists.txt create mode 100644 eval/src/vespa/eval/tensor/mixed/CMakeLists.txt diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 76c1a63b881..d4189fb3256 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -54,6 +54,7 @@ vespa_define_module( src/tests/tensor/direct_sparse_tensor_builder src/tests/tensor/index_lookup_table src/tests/tensor/onnx_wrapper + src/tests/tensor/packed_mappings src/tests/tensor/tensor_add_operation src/tests/tensor/tensor_address src/tests/tensor/tensor_conformance @@ -72,6 +73,7 @@ vespa_define_module( src/vespa/eval/gp src/vespa/eval/tensor src/vespa/eval/tensor/dense + src/vespa/eval/tensor/mixed src/vespa/eval/tensor/serialization src/vespa/eval/tensor/sparse ) diff --git a/eval/src/tests/tensor/packed_mappings/CMakeLists.txt b/eval/src/tests/tensor/packed_mappings/CMakeLists.txt new file mode 100644 index 00000000000..2d11755a0c5 --- /dev/null +++ b/eval/src/tests/tensor/packed_mappings/CMakeLists.txt @@ -0,0 +1,19 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +vespa_add_executable(eval_packed_mappings_test_app TEST + SOURCES + packed_mappings_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_packed_mappings_test_app COMMAND eval_packed_mappings_test_app) + +vespa_add_executable(eval_packed_mixed_test_app TEST + SOURCES + packed_mixed_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_packed_mixed_test_app COMMAND eval_packed_mixed_test_app) diff --git a/eval/src/vespa/eval/CMakeLists.txt b/eval/src/vespa/eval/CMakeLists.txt index 04f151f7ced..b01945cf752 100644 --- a/eval/src/vespa/eval/CMakeLists.txt +++ b/eval/src/vespa/eval/CMakeLists.txt @@ -8,6 +8,7 @@ vespa_add_library(vespaeval $ $ $ + $ $ $ INSTALL lib64 diff --git a/eval/src/vespa/eval/tensor/mixed/CMakeLists.txt b/eval/src/vespa/eval/tensor/mixed/CMakeLists.txt new file mode 100644 index 00000000000..2621ec616c0 --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +vespa_add_library(eval_tensor_mixed OBJECT + SOURCES + packed_labels.cpp + packed_mappings.cpp + packed_mappings_builder.cpp + packed_mixed_factory.cpp + packed_mixed_tensor.cpp + packed_mixed_builder.cpp +) -- cgit v1.2.3 From e3ff6a8769b6e09fe2ea29ee2cd4912960fe1664 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Thu, 17 Sep 2020 13:34:16 +0000 Subject: add simpler views for the two special cases --- .../vespa/eval/tensor/mixed/packed_mappings.cpp | 12 +++ .../eval/tensor/mixed/packed_mixed_tensor.cpp | 110 +++++++++++++++++++++ 2 files changed, 122 insertions(+) diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp index b11c8d08097..bb9fe1ceef8 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp @@ -18,6 +18,18 @@ PackedMappings::subspace_of_address(const Address &address) const return subspace_of_sortid(internal_idx); } +int32_t +PackedMappings::subspace_of_enums(const InternalAddress &address) const +{ + int32_t idx = sortid_of_enums(address); + if (idx < 0) { + return -1; + } + uint32_t internal_idx = idx; + assert (internal_idx < _num_mappings); + return subspace_of_sortid(internal_idx); +} + int32_t PackedMappings::sortid_of_address(const Address &address) const { diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp index 42a9bf78846..94496e05e57 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp @@ -4,6 +4,8 @@ namespace vespalib::eval { +/*********************************************************************************/ + class PackedMixedTensorIndexView : public NewValue::Index::View { private: @@ -103,15 +105,123 @@ PackedMixedTensorIndexView::next_result(const std::vector return false; } +/*********************************************************************************/ + +class PackedMixedTensorLookup : public NewValue::Index::View +{ +private: + const PackedMappings& _mappings; + std::vector _lookup_enums; + bool _first_time; + + size_t num_full_dims() const { return _mappings.num_sparse_dims(); } +public: + PackedMixedTensorLookup(const PackedMappings& mappings) + : _mappings(mappings), + _lookup_enums(), + _first_time(false) + { + _lookup_enums.reserve(num_full_dims()); + } + + void lookup(const std::vector &addr) override; + bool next_result(const std::vector &addr_out, size_t &idx_out) override; + ~PackedMixedTensorLookup() override = default; +}; + +void +PackedMixedTensorLookup::lookup(const std::vector &addr) +{ + assert(addr.size() == num_full_dims()); + _lookup_enums.clear(); + for (const vespalib::stringref * label_ptr : addr) { + int32_t label_enum = _mappings.label_store().find_label(*label_ptr); + if (label_enum < 0) { + // cannot match + _first_time = false; + return; + } + _lookup_enums.push_back(label_enum); + } + _first_time = true; +} + +bool +PackedMixedTensorLookup::next_result(const std::vector &addr_out, size_t &idx_out) +{ + assert(addr_out.size() == 0); + if (_first_time) { + _first_time = false; + int32_t subspace = _mappings.subspace_of_enums(_lookup_enums); + if (subspace >= 0) { + idx_out = subspace; + return true; + } + } + return false; +} + +/*********************************************************************************/ + +class PackedMixedTensorAllMappings : public NewValue::Index::View +{ +private: + const PackedMappings& _mappings; + std::vector _full_address; + size_t _index; + +public: + PackedMixedTensorAllMappings(const PackedMappings& mappings) + : _mappings(mappings), + _full_address(), + _index(0) + { + _full_address.resize(_mappings.num_sparse_dims()); + } + + void lookup(const std::vector &addr) override; + bool next_result(const std::vector &addr_out, size_t &idx_out) override; + ~PackedMixedTensorAllMappings() override = default; +}; + +void +PackedMixedTensorAllMappings::lookup(const std::vector &addr) +{ + _index = 0; + assert(addr.size() == 0); +} + +bool +PackedMixedTensorAllMappings::next_result(const std::vector &addr_out, size_t &idx_out) +{ + assert(addr_out.size() == _mappings.num_sparse_dims()); + while (_index < _mappings.size()) { + idx_out = _mappings.fill_by_subspace(_index++, _full_address); + for (size_t i = 0; i < _mappings.num_sparse_dims(); ++i) { + *addr_out[i] = _full_address[i]; + } + return true; + } + return false; +} + +/*********************************************************************************/ + PackedMixedTensor::~PackedMixedTensor() = default; std::unique_ptr PackedMixedTensor::create_view(const std::vector &dims) const { + if (dims.size() == 0) { + return std::make_unique(_mappings); + } for (size_t i = 1; i < dims.size(); ++i) { assert(dims[i-1] < dims[i]); assert(dims[i] < _mappings.num_sparse_dims()); } + if (dims.size() == _mappings.num_sparse_dims()) { + return std::make_unique(_mappings); + } return std::make_unique(_mappings, dims); } -- cgit v1.2.3 From e60c65dbb6a3bbe006ce45c087b855c2eb460821 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Thu, 17 Sep 2020 13:57:12 +0000 Subject: cleanup two unused members --- eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp index 94496e05e57..472cf553c8e 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp @@ -13,8 +13,6 @@ private: const std::vector _view_dims; std::vector _lookup_enums; std::vector _full_enums; - std::vector _lookup_addr; - std::vector _full_address; size_t _index; size_t num_full_dims() const { return _mappings.num_sparse_dims(); } @@ -26,14 +24,10 @@ public: : _mappings(mappings), _view_dims(dims), _lookup_enums(), - _lookup_addr(), - _full_address(), _index(0) { _lookup_enums.reserve(num_view_dims()); - _lookup_addr.reserve(num_view_dims()); _full_enums.resize(num_full_dims()); - _full_address.resize(num_full_dims()); } void lookup(const std::vector &addr) override; @@ -47,7 +41,6 @@ PackedMixedTensorIndexView::lookup(const std::vector _index = 0; assert(addr.size() == num_view_dims()); _lookup_enums.clear(); - _lookup_addr.clear(); // printf("lookup %zu/%zu dims:", num_view_dims(), num_full_dims()); for (const vespalib::stringref * label_ptr : addr) { int32_t label_enum = _mappings.label_store().find_label(*label_ptr); @@ -57,7 +50,6 @@ PackedMixedTensorIndexView::lookup(const std::vector break; } _lookup_enums.push_back(label_enum); - _lookup_addr.push_back(*label_ptr); // printf(" '%s'", label_ptr->data()); } // printf(" [in %u mappings]\n", _mappings.size()); @@ -90,7 +82,6 @@ PackedMixedTensorIndexView::next_result(const std::vector // not a view dimension: uint32_t label_enum = _full_enums[i]; auto label_value = _mappings.label_store().label_value(label_enum); - _full_address[i] = label_value; *addr_out[ao_idx] = label_value; ++ao_idx; } -- cgit v1.2.3 From 79bb7383a54615f6d2f1962683b11e9a738b69f5 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Thu, 17 Sep 2020 14:30:57 +0000 Subject: add class comments --- eval/src/vespa/eval/tensor/mixed/packed_labels.h | 7 +++++++ eval/src/vespa/eval/tensor/mixed/packed_mappings.h | 21 +++++++++++++++++++++ .../eval/tensor/mixed/packed_mappings_builder.h | 7 +++++++ .../vespa/eval/tensor/mixed/packed_mixed_builder.h | 4 ++++ .../vespa/eval/tensor/mixed/packed_mixed_factory.h | 4 ++++ .../vespa/eval/tensor/mixed/packed_mixed_tensor.h | 12 ++++++++++-- 6 files changed, 53 insertions(+), 2 deletions(-) diff --git a/eval/src/vespa/eval/tensor/mixed/packed_labels.h b/eval/src/vespa/eval/tensor/mixed/packed_labels.h index 07db1aac8de..e7fc1d8e33a 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_labels.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_labels.h @@ -7,6 +7,13 @@ namespace vespalib::eval { +/** + * Stores labels for sparse (mapped) tensor dimensions, + * where each unique label value is stored only once, + * and the values are sorted. References data that + * must be constant and owned by some object with + * enclosing lifetime. + **/ class PackedLabels { public: PackedLabels(uint32_t num_labels, diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings.h b/eval/src/vespa/eval/tensor/mixed/packed_mappings.h index 18c70f01eb6..19db3566020 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings.h @@ -7,6 +7,27 @@ namespace vespalib::eval { +/** + * Mappings for sparse tensor dimensions. + * Each address (conceptually "array of string") + * has two indexes: The subspace_index (the + * order that addresses were added to a builder), + * and the internal_index AKA "sortid", which + * indexes into a lexicographically sorted array + * of addresses. + * (Note: we may want to change this so subspaces + * are always sorted by address, making these two + * indexes equivalent). + * + * Has various methods for mapping back and forth + * between addresses and indexes, and also allows + * using the internal label enumerations instead + * of working with strings all the time. + * + * NOTE: Making a copy of PackedMappings will not copy + * the underlying data, these must then stay alive + * and unchanged for the lifetime of the copy as well. + **/ class PackedMappings { public: using Address = std::vector; diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h index b78c1cf17f7..0b96a76f4c8 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h @@ -11,6 +11,13 @@ namespace vespalib::eval { +/** + * Builder for PackedMappings. + * Copies label values in all addresses added + * and packs all resulting data into a block of memory + * held by the built object (or optionally some + * larger aggregating object by target_memory). + **/ class PackedMappingsBuilder { public: using SparseAddress = std::vector; diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h index 200c8f19482..39d8345911d 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h @@ -6,6 +6,10 @@ namespace vespalib::eval { +/** + * A builder for PackedMixedTensor objects + * appropriate for cell type T. + **/ template class PackedMixedBuilder : public ValueBuilder { diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.h index 3be266541cc..ae90e5e243b 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.h @@ -6,6 +6,10 @@ namespace vespalib::eval { +/** + * A factory that can generate PackedMixedBuilder + * objects appropriate for the requested CellType. + */ struct PackedMixedFactory : ValueBuilderFactory { ~PackedMixedFactory() override {} protected: diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h index 67684097380..8956abafcd3 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h @@ -11,6 +11,13 @@ namespace vespalib::eval { +/** + * An implementation of NewValue modeling a mixed tensor, + * where all the data (cells and sparse address mappings) + * can reside in a self-contained, contigous block of memory. + * Currently must be built by a PackedMixedBuilder. + * Immutable (all data always const). + **/ class PackedMixedTensor : public NewValue, public NewValue::Index { private: @@ -18,8 +25,9 @@ private: const TypedCells _cells; const PackedMappings _mappings; - PackedMixedTensor(const ValueType &type, TypedCells cells, - PackedMappings mappings) + PackedMixedTensor(const ValueType &type, + TypedCells cells, + const PackedMappings &mappings) : _type(type), _cells(cells), _mappings(mappings) -- cgit v1.2.3 From 2299f4685c4ea9d0030fed539f1c5e53a09cab5d Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Fri, 18 Sep 2020 12:46:54 +0000 Subject: add assert --- eval/src/vespa/eval/tensor/mixed/packed_labels.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/eval/src/vespa/eval/tensor/mixed/packed_labels.cpp b/eval/src/vespa/eval/tensor/mixed/packed_labels.cpp index 24a1c80a4cb..05986cbc562 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_labels.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_labels.cpp @@ -42,6 +42,7 @@ PackedLabels::validate_labels(uint32_t num_labels) for (uint32_t i = 0; i < num_labels; ++i) { assert(_offsets[i] < _offsets[i+1]); uint32_t last_byte_index = _offsets[i+1] - 1; + assert(last_byte_index < _label_store.size()); assert(_label_store[last_byte_index] == 0); } assert(_label_store.size() == _offsets[num_labels]); -- cgit v1.2.3 From c8cfe0c3c203073729a196df66031228bf418ca4 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Fri, 18 Sep 2020 12:47:23 +0000 Subject: use map::emplace --- eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp index d149539bd39..f7c3ffe4ed4 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp @@ -16,10 +16,8 @@ PackedMappingsBuilder::add_mapping_for(SparseAddress address) auto iter = _labels.insert(label_value).first; label_value = *iter; } - uint32_t next_index = _mappings.size(); - IndexMap::value_type new_val(address, next_index); - auto iter = _mappings.insert(new_val).first; + auto iter = _mappings.emplace(address, next_index).first; return iter->second; } -- cgit v1.2.3 From 3812753b7a030b52c8768ceedd6f9af6eec6c9a2 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Fri, 18 Sep 2020 12:49:03 +0000 Subject: always iterate in sortid order * prepares for simplifying PackedMappings * rename num_sparse_dims -> num_mapped_dims * rename expect_subspaces -> expected_subspaces --- .../packed_mappings/packed_mappings_test.cpp | 8 ++-- eval/src/vespa/eval/tensor/mixed/packed_mappings.h | 45 +++++++++++----------- .../eval/tensor/mixed/packed_mappings_builder.h | 6 +-- .../vespa/eval/tensor/mixed/packed_mixed_builder.h | 4 +- .../eval/tensor/mixed/packed_mixed_factory.cpp | 4 +- .../eval/tensor/mixed/packed_mixed_tensor.cpp | 18 ++++----- 6 files changed, 44 insertions(+), 41 deletions(-) diff --git a/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp b/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp index 0b668986cb6..3c1c6a90521 100644 --- a/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp +++ b/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp @@ -82,17 +82,19 @@ public: ASSERT_TRUE(builder); built = builder->build_mappings(); ASSERT_TRUE(built); - EXPECT_EQ(builder->num_sparse_dims(), built->num_sparse_dims()); + EXPECT_EQ(builder->num_mapped_dims(), built->num_mapped_dims()); EXPECT_EQ(builder->size(), built->size()); for (size_t idx = 0; idx < built->size(); ++idx) { - auto got = built->address_of_subspace(idx); + std::vector got(builder->num_mapped_dims()); + built->fill_by_sortid(idx, got); printf("Got address:"); for (auto ref : got) { printf(" '%s'", ref.data()); } + uint32_t subspace = built->subspace_of_address(got); uint32_t original = builder->add_mapping_for(got); printf(" -> %u\n", original); - EXPECT_EQ(idx, original); + EXPECT_EQ(subspace, original); } } }; diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings.h b/eval/src/vespa/eval/tensor/mixed/packed_mappings.h index 19db3566020..44ab136f8bd 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings.h @@ -34,36 +34,16 @@ public: using InternalAddress = std::vector; uint32_t size() const { return _num_mappings; } - uint32_t num_sparse_dims() const { return _num_dims; } + uint32_t num_mapped_dims() const { return _num_dims; } // returns -1 if mapping does not contain address - int32_t subspace_of_address(const Address &address) const; int32_t subspace_of_enums(const InternalAddress &address) const; - int32_t sortid_of_address(const Address &address) const; - int32_t sortid_of_enums(const InternalAddress &address) const; - - Address address_of_sortid(uint32_t internal_index) const; - Address address_of_subspace(uint32_t subspace_index) const; - - /** returns sortid */ - uint32_t fill_by_subspace(uint32_t subspace_index, Address &address) const; - uint32_t fill_by_subspace(uint32_t subspace_index, InternalAddress &address) const; + int32_t subspace_of_address(const Address &address) const; /** returns subspace_index */ uint32_t fill_by_sortid(uint32_t sortid, Address &address) const; uint32_t fill_by_sortid(uint32_t sortid, InternalAddress &address) const; - InternalAddress enums_of_sortid(uint32_t internal_index) const; - InternalAddress enums_of_subspace(uint32_t subspace_index) const; - - int enums_compare(const uint32_t *a, const uint32_t *b) const { - for (size_t i = 0; i < _num_dims; ++i) { - if (a[i] < b[i]) return -1; - if (a[i] > b[i]) return 1; - } - return 0; - } - const PackedLabels & label_store() const { return _label_store; } private: PackedMappings(uint32_t num_dims, uint32_t num_mappings, @@ -97,6 +77,14 @@ private: const ConstArrayRef _int_store; const PackedLabels _label_store; + int enums_compare(const uint32_t *a, const uint32_t *b) const { + for (size_t i = 0; i < _num_dims; ++i) { + if (a[i] < b[i]) return -1; + if (a[i] > b[i]) return 1; + } + return 0; + } + uint32_t offset_of_mapping_data(uint32_t idx) const { return (idx * (1 + _num_dims)) + _num_mappings; } @@ -110,6 +98,19 @@ private: const uint32_t * ptr_of_sortid(uint32_t internal_index) const { return &_int_store[offset_of_mapping_data(internal_index)]; } + + int32_t sortid_of_address(const Address &address) const; + int32_t sortid_of_enums(const InternalAddress &address) const; + + /** returns sortid */ + uint32_t fill_by_subspace(uint32_t subspace_index, Address &address) const; + uint32_t fill_by_subspace(uint32_t subspace_index, InternalAddress &address) const; + + InternalAddress enums_of_subspace(uint32_t subspace_index) const; + InternalAddress enums_of_sortid(uint32_t internal_index) const; + + Address address_of_sortid(uint32_t internal_index) const; + Address address_of_subspace(uint32_t subspace_index) const; }; } // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h index 0b96a76f4c8..46bed473d06 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h @@ -22,8 +22,8 @@ class PackedMappingsBuilder { public: using SparseAddress = std::vector; - PackedMappingsBuilder(uint32_t num_sparse_dims) - : _num_dims(num_sparse_dims), + PackedMappingsBuilder(uint32_t num_mapped_dims) + : _num_dims(num_mapped_dims), _labels(), _mappings() {} @@ -37,7 +37,7 @@ public: size_t extra_memory() const; PackedMappings target_memory(char *mem_start, char *mem_end) const; - uint32_t num_sparse_dims() const { return _num_dims; } + uint32_t num_mapped_dims() const { return _num_dims; } size_t size() const { return _mappings.size(); } private: uint32_t _num_dims; diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h index 39d8345911d..ea8a6206244 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h @@ -22,13 +22,13 @@ public: PackedMixedBuilder(const ValueType &type, size_t num_mapped_in, size_t subspace_size_in, - size_t expect_subspaces) + size_t expected_subspaces) : _type(type), _subspace_size(subspace_size_in), _cells(), _mappings_builder(num_mapped_in) { - _cells.reserve(_subspace_size * expect_subspaces); + _cells.reserve(_subspace_size * expected_subspaces); } ~PackedMixedBuilder() override = default; diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.cpp index db09db48a4c..2a29cab9ce0 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.cpp @@ -24,10 +24,10 @@ std::unique_ptr PackedMixedFactory::create_value_builder_base(const ValueType &type, size_t num_mapped_in, size_t subspace_size_in, - size_t expect_subspaces) const + size_t expected_subspaces) const { return typify_invoke<1,TypifyCellType,CreatePackedMixedBuilder>(type.cell_type(), - type, num_mapped_in, subspace_size_in, expect_subspaces); + type, num_mapped_in, subspace_size_in, expected_subspaces); } } // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp index 472cf553c8e..0e32e1b06e7 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp @@ -15,7 +15,7 @@ private: std::vector _full_enums; size_t _index; - size_t num_full_dims() const { return _mappings.num_sparse_dims(); } + size_t num_full_dims() const { return _mappings.num_mapped_dims(); } size_t num_view_dims() const { return _view_dims.size(); } size_t num_rest_dims() const { return num_full_dims() - num_view_dims(); } public: @@ -60,7 +60,7 @@ PackedMixedTensorIndexView::next_result(const std::vector { assert(addr_out.size() == num_rest_dims()); while (_index < _mappings.size()) { - idx_out = _mappings.fill_by_subspace(_index++, _full_enums); + idx_out = _mappings.fill_by_sortid(_index++, _full_enums); bool couldmatch = true; size_t vd_idx = 0; size_t ao_idx = 0; @@ -105,7 +105,7 @@ private: std::vector _lookup_enums; bool _first_time; - size_t num_full_dims() const { return _mappings.num_sparse_dims(); } + size_t num_full_dims() const { return _mappings.num_mapped_dims(); } public: PackedMixedTensorLookup(const PackedMappings& mappings) : _mappings(mappings), @@ -167,7 +167,7 @@ public: _full_address(), _index(0) { - _full_address.resize(_mappings.num_sparse_dims()); + _full_address.resize(_mappings.num_mapped_dims()); } void lookup(const std::vector &addr) override; @@ -185,10 +185,10 @@ PackedMixedTensorAllMappings::lookup(const std::vector &addr_out, size_t &idx_out) { - assert(addr_out.size() == _mappings.num_sparse_dims()); + assert(addr_out.size() == _mappings.num_mapped_dims()); while (_index < _mappings.size()) { - idx_out = _mappings.fill_by_subspace(_index++, _full_address); - for (size_t i = 0; i < _mappings.num_sparse_dims(); ++i) { + idx_out = _mappings.fill_by_sortid(_index++, _full_address); + for (size_t i = 0; i < _mappings.num_mapped_dims(); ++i) { *addr_out[i] = _full_address[i]; } return true; @@ -208,9 +208,9 @@ PackedMixedTensor::create_view(const std::vector &dims) const } for (size_t i = 1; i < dims.size(); ++i) { assert(dims[i-1] < dims[i]); - assert(dims[i] < _mappings.num_sparse_dims()); + assert(dims[i] < _mappings.num_mapped_dims()); } - if (dims.size() == _mappings.num_sparse_dims()) { + if (dims.size() == _mappings.num_mapped_dims()) { return std::make_unique(_mappings); } return std::make_unique(_mappings, dims); -- cgit v1.2.3 From 5cc6f25ef2add97bab09f46ce244cdf4abe0bde8 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Fri, 18 Sep 2020 13:07:10 +0000 Subject: use an extra namespace --- eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp | 1 + eval/src/vespa/eval/tensor/mixed/packed_labels.cpp | 2 +- eval/src/vespa/eval/tensor/mixed/packed_labels.h | 2 +- eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp | 2 +- eval/src/vespa/eval/tensor/mixed/packed_mappings.h | 2 +- eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp | 2 +- eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h | 2 +- eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.cpp | 2 +- eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h | 2 +- eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.cpp | 2 +- eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp | 2 +- eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h | 2 +- 12 files changed, 12 insertions(+), 11 deletions(-) diff --git a/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp b/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp index 3c1c6a90521..87a77ab53ad 100644 --- a/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp +++ b/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp @@ -12,6 +12,7 @@ #include using namespace vespalib::eval; +using namespace vespalib::eval::packed_mixed_tensor; namespace { diff --git a/eval/src/vespa/eval/tensor/mixed/packed_labels.cpp b/eval/src/vespa/eval/tensor/mixed/packed_labels.cpp index 05986cbc562..2153a6a0ca9 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_labels.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_labels.cpp @@ -3,7 +3,7 @@ #include "packed_labels.h" #include -namespace vespalib::eval { +namespace vespalib::eval::packed_mixed_tensor { int32_t PackedLabels::find_label(vespalib::stringref to_find) const diff --git a/eval/src/vespa/eval/tensor/mixed/packed_labels.h b/eval/src/vespa/eval/tensor/mixed/packed_labels.h index e7fc1d8e33a..0994ef07a63 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_labels.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_labels.h @@ -5,7 +5,7 @@ #include #include -namespace vespalib::eval { +namespace vespalib::eval::packed_mixed_tensor { /** * Stores labels for sparse (mapped) tensor dimensions, diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp index bb9fe1ceef8..03080772989 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp @@ -3,7 +3,7 @@ #include "packed_mappings.h" #include -namespace vespalib::eval { +namespace vespalib::eval::packed_mixed_tensor { int32_t diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings.h b/eval/src/vespa/eval/tensor/mixed/packed_mappings.h index 44ab136f8bd..3ed554594aa 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings.h @@ -5,7 +5,7 @@ #include "packed_labels.h" #include -namespace vespalib::eval { +namespace vespalib::eval::packed_mixed_tensor { /** * Mappings for sparse tensor dimensions. diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp index f7c3ffe4ed4..b06f826a3e3 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp @@ -3,7 +3,7 @@ #include "packed_mappings_builder.h" #include -namespace vespalib::eval { +namespace vespalib::eval::packed_mixed_tensor { PackedMappingsBuilder::~PackedMappingsBuilder() = default; diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h index 46bed473d06..ce3abb78280 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h @@ -9,7 +9,7 @@ #include #include -namespace vespalib::eval { +namespace vespalib::eval::packed_mixed_tensor { /** * Builder for PackedMappings. diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.cpp index cd0310762d2..1540958fa27 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.cpp @@ -2,7 +2,7 @@ #include "packed_mixed_builder.h" -namespace vespalib::eval { +namespace vespalib::eval::packed_mixed_tensor { template ArrayRef diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h index ea8a6206244..099f531ae38 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h @@ -4,7 +4,7 @@ #include "packed_mixed_tensor.h" -namespace vespalib::eval { +namespace vespalib::eval::packed_mixed_tensor { /** * A builder for PackedMixedTensor objects diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.cpp index 2a29cab9ce0..75e6b1e996e 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.cpp @@ -14,7 +14,7 @@ struct CreatePackedMixedBuilder { static std::unique_ptr invoke(const ValueType &type, Args &&...args) { assert(check_cell_type(type.cell_type())); - return std::make_unique>(type, std::forward(args)...); + return std::make_unique>(type, std::forward(args)...); } }; diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp index 0e32e1b06e7..b10f6586da5 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp @@ -2,7 +2,7 @@ #include "packed_mixed_tensor.h" -namespace vespalib::eval { +namespace vespalib::eval::packed_mixed_tensor { /*********************************************************************************/ diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h index 8956abafcd3..ffef379b37b 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h @@ -9,7 +9,7 @@ #include #include -namespace vespalib::eval { +namespace vespalib::eval::packed_mixed_tensor { /** * An implementation of NewValue modeling a mixed tensor, -- cgit v1.2.3 From e03c0faf3ef22d82f6907e9d880fdfb33a2adbad Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Fri, 18 Sep 2020 13:12:24 +0000 Subject: rename and remove some methods * remove unused methods * rename to be more explicit --- .../packed_mappings/packed_mappings_test.cpp | 2 +- .../vespa/eval/tensor/mixed/packed_mappings.cpp | 63 +--------------------- eval/src/vespa/eval/tensor/mixed/packed_mappings.h | 17 +----- .../eval/tensor/mixed/packed_mixed_tensor.cpp | 4 +- 4 files changed, 7 insertions(+), 79 deletions(-) diff --git a/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp b/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp index 87a77ab53ad..b8072f92b6c 100644 --- a/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp +++ b/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp @@ -87,7 +87,7 @@ public: EXPECT_EQ(builder->size(), built->size()); for (size_t idx = 0; idx < built->size(); ++idx) { std::vector got(builder->num_mapped_dims()); - built->fill_by_sortid(idx, got); + built->fill_address_by_sortid(idx, got); printf("Got address:"); for (auto ref : got) { printf(" '%s'", ref.data()); diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp index 03080772989..01d797a1c6c 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp @@ -72,34 +72,8 @@ PackedMappings::sortid_of_enums(const InternalAddress &address) const return -1; } -std::vector -PackedMappings::address_of_sortid(uint32_t internal_index) const -{ - std::vector result; - fill_by_sortid(internal_index, result); - return result; -} - -std::vector -PackedMappings::address_of_subspace(uint32_t subspace_index) const -{ - std::vector result; - fill_by_subspace(subspace_index, result); - return result; -} - uint32_t -PackedMappings::fill_by_subspace(uint32_t subspace_index, InternalAddress &address) const -{ - assert(subspace_index < _num_mappings); - uint32_t internal_index = sortid_of_subspace(subspace_index); - uint32_t subspace = fill_by_sortid(internal_index, address); - assert(subspace == subspace_index); - return internal_index; -} - -uint32_t -PackedMappings::fill_by_sortid(uint32_t internal_index, InternalAddress &address) const +PackedMappings::fill_enums_by_sortid(uint32_t internal_index, InternalAddress &address) const { assert(internal_index < _num_mappings); uint32_t offset = offset_of_mapping_data(internal_index); @@ -112,7 +86,7 @@ PackedMappings::fill_by_sortid(uint32_t internal_index, InternalAddress &address /** returns subspace_index */ uint32_t -PackedMappings::fill_by_sortid(uint32_t internal_index, Address &address) const +PackedMappings::fill_address_by_sortid(uint32_t internal_index, Address &address) const { assert(internal_index < _num_mappings); uint32_t offset = offset_of_mapping_data(internal_index); @@ -124,17 +98,6 @@ PackedMappings::fill_by_sortid(uint32_t internal_index, Address &address) const return _int_store[offset]; } -/** returns internal_index */ -uint32_t -PackedMappings::fill_by_subspace(uint32_t subspace_index, Address &address) const -{ - assert(subspace_index < _num_mappings); - uint32_t internal_index = sortid_of_subspace(subspace_index); - uint32_t subspace = fill_by_sortid(internal_index, address); - assert(subspace == subspace_index); - return internal_index; -} - void PackedMappings::validate() const { @@ -167,26 +130,4 @@ PackedMappings::validate() const assert(iter == _int_store.cend()); } -std::vector -PackedMappings::enums_of_subspace(uint32_t subspace_index) const -{ - assert(subspace_index < _num_mappings); - uint32_t internal_index = sortid_of_subspace(subspace_index); - return enums_of_sortid(internal_index); -} - -std::vector -PackedMappings::enums_of_sortid(uint32_t internal_index) const -{ - std::vector result; - result.reserve(_num_dims); - assert(internal_index < _num_mappings); - uint32_t offset = offset_of_mapping_data(internal_index); - for (uint32_t i = 0; i < _num_dims; ++i) { - result.push_back(_int_store[offset++]); - } - return result; -} - - } // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings.h b/eval/src/vespa/eval/tensor/mixed/packed_mappings.h index 3ed554594aa..c273725d732 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings.h @@ -41,8 +41,8 @@ public: int32_t subspace_of_address(const Address &address) const; /** returns subspace_index */ - uint32_t fill_by_sortid(uint32_t sortid, Address &address) const; - uint32_t fill_by_sortid(uint32_t sortid, InternalAddress &address) const; + uint32_t fill_address_by_sortid(uint32_t sortid, Address &address) const; + uint32_t fill_enums_by_sortid(uint32_t sortid, InternalAddress &address) const; const PackedLabels & label_store() const { return _label_store; } private: @@ -92,25 +92,12 @@ private: uint32_t offset = offset_of_mapping_data(internal_index); return _int_store[offset + _num_dims]; } - uint32_t sortid_of_subspace(uint32_t subspace_index) const { - return _int_store[subspace_index]; - } const uint32_t * ptr_of_sortid(uint32_t internal_index) const { return &_int_store[offset_of_mapping_data(internal_index)]; } int32_t sortid_of_address(const Address &address) const; int32_t sortid_of_enums(const InternalAddress &address) const; - - /** returns sortid */ - uint32_t fill_by_subspace(uint32_t subspace_index, Address &address) const; - uint32_t fill_by_subspace(uint32_t subspace_index, InternalAddress &address) const; - - InternalAddress enums_of_subspace(uint32_t subspace_index) const; - InternalAddress enums_of_sortid(uint32_t internal_index) const; - - Address address_of_sortid(uint32_t internal_index) const; - Address address_of_subspace(uint32_t subspace_index) const; }; } // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp index b10f6586da5..a9bbdc39889 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp @@ -60,7 +60,7 @@ PackedMixedTensorIndexView::next_result(const std::vector { assert(addr_out.size() == num_rest_dims()); while (_index < _mappings.size()) { - idx_out = _mappings.fill_by_sortid(_index++, _full_enums); + idx_out = _mappings.fill_enums_by_sortid(_index++, _full_enums); bool couldmatch = true; size_t vd_idx = 0; size_t ao_idx = 0; @@ -187,7 +187,7 @@ PackedMixedTensorAllMappings::next_result(const std::vector Date: Fri, 18 Sep 2020 13:55:14 +0000 Subject: remove table for sortid->subspace mapping --- eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp | 7 ++----- eval/src/vespa/eval/tensor/mixed/packed_mappings.h | 4 +--- .../eval/tensor/mixed/packed_mappings_builder.cpp | 18 +++++++++--------- .../vespa/eval/tensor/mixed/packed_mixed_builder.cpp | 10 ++++------ 4 files changed, 16 insertions(+), 23 deletions(-) diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp index 01d797a1c6c..9a50f9ae431 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp @@ -72,6 +72,7 @@ PackedMappings::sortid_of_enums(const InternalAddress &address) const return -1; } +/** returns subspace_index */ uint32_t PackedMappings::fill_enums_by_sortid(uint32_t internal_index, InternalAddress &address) const { @@ -101,12 +102,8 @@ PackedMappings::fill_address_by_sortid(uint32_t internal_index, Address &address void PackedMappings::validate() const { - assert((_num_mappings * (2 + _num_dims)) == _int_store.size()); + assert((_num_mappings * (1 + _num_dims)) == _int_store.size()); auto iter = _int_store.cbegin(); - for (uint32_t i = 0; i < _num_mappings; ++i) { - uint32_t internal_index = *iter++; - assert(internal_index < _num_mappings); - } std::vector prev; std::vector next; for (uint32_t i = 0; i < _num_mappings; ++i) { diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings.h b/eval/src/vespa/eval/tensor/mixed/packed_mappings.h index c273725d732..f43f2ef6832 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings.h @@ -65,8 +65,6 @@ private: /* _int_store contains data corresponding to this model: struct IntStore { - // map to index in next table: - uint32_t index_of_subspace[num_mappings]; // sorted lexicographically by label_enums: struct MappingData { uint32_t label_enums[num_dims]; @@ -86,7 +84,7 @@ private: } uint32_t offset_of_mapping_data(uint32_t idx) const { - return (idx * (1 + _num_dims)) + _num_mappings; + return (idx * (1 + _num_dims)); } uint32_t subspace_of_sortid(uint32_t internal_index) const { uint32_t offset = offset_of_mapping_data(internal_index); diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp index b06f826a3e3..a78a7423520 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp @@ -40,7 +40,7 @@ PackedMappingsBuilder::extra_memory() const PackedMappings PackedMappingsBuilder::target_memory(char *mem_start, char *mem_end) const { - size_t int_store_cnt = (2 + _num_dims) * _mappings.size(); + size_t int_store_cnt = (1 + _num_dims) * _mappings.size(); size_t int_store_size = int_store_cnt * sizeof(uint32_t); size_t label_cnt = _labels.size(); size_t label_offsets_size = (1 + label_cnt) * sizeof(uint32_t); @@ -61,6 +61,7 @@ PackedMappingsBuilder::target_memory(char *mem_start, char *mem_end) const ArrayRef int_store_data(int_store_mem, int_store_cnt); ArrayRef label_offsets(offsets_mem, 1 + label_cnt); ArrayRef labels_data(labels_mem, label_bytes); + assert(labels_data.end() <= mem_end); size_t byte_idx = 0; size_t label_num = 0; @@ -73,10 +74,11 @@ PackedMappingsBuilder::target_memory(char *mem_start, char *mem_end) const assert(label_num == label_cnt); label_offsets[label_num] = byte_idx; + assert(labels_data.begin() + byte_idx == labels_data.end()); + PackedLabels stored_labels(label_cnt, label_offsets, labels_data); - size_t mapping_idx = 0; - size_t int_store_offset = _mappings.size(); + size_t int_store_offset = 0; for (const auto & kv : _mappings) { const SparseAddress & k = kv.first; uint32_t v = kv.second; @@ -87,10 +89,8 @@ PackedMappingsBuilder::target_memory(char *mem_start, char *mem_end) const int_store_data[int_store_offset++] = label_idx; } int_store_data[int_store_offset++] = v; - int_store_data[v] = mapping_idx++; } assert(int_store_offset == int_store_cnt); - assert(mapping_idx == _mappings.size()); return PackedMappings(_num_dims, _mappings.size(), int_store_data, stored_labels); @@ -99,13 +99,13 @@ PackedMappingsBuilder::target_memory(char *mem_start, char *mem_end) const std::unique_ptr PackedMappingsBuilder::build_mappings() const { - size_t meta_size = sizeof(PackedMappings); - size_t total_size = meta_size + extra_memory(); + size_t self_size = sizeof(PackedMappings); + size_t total_size = self_size + extra_memory(); char * mem = (char *) operator new(total_size); - auto meta_data = target_memory(mem + meta_size, mem + total_size); + auto self_data = target_memory(mem + self_size, mem + total_size); - PackedMappings * built = new (mem) PackedMappings(meta_data); + PackedMappings * built = new (mem) PackedMappings(self_data); return std::unique_ptr(built); } diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.cpp index 1540958fa27..e9390c3bd21 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.cpp @@ -20,18 +20,18 @@ PackedMixedBuilder::add_subspace(const std::vector &addr template std::unique_ptr -PackedMixedBuilder::build(std::unique_ptr> self) +PackedMixedBuilder::build(std::unique_ptr>) { - size_t meta_size = sizeof(PackedMixedTensor); + size_t self_size = sizeof(PackedMixedTensor); size_t mappings_size = _mappings_builder.extra_memory(); // align: mappings_size += 15ul; mappings_size &= ~15ul; size_t cells_size = sizeof(T) * _cells.size(); - size_t total_size = sizeof(PackedMixedTensor) + mappings_size + cells_size; + size_t total_size = self_size + mappings_size + cells_size; char *mem = (char *) operator new(total_size); - char *mappings_mem = mem + meta_size; + char *mappings_mem = mem + self_size; char *cells_mem = mappings_mem + mappings_size; // fill mapping data: @@ -44,8 +44,6 @@ PackedMixedBuilder::build(std::unique_ptr> self) PackedMixedTensor * built = new (mem) PackedMixedTensor(_type, TypedCells(cells), mappings); - // keep "this" alive until this point: - (void) self; return std::unique_ptr(built); } -- cgit v1.2.3 From 9db6c09a62822ae1a3e7b6bad70ed214e442c1c6 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Tue, 15 Sep 2020 11:21:29 +0200 Subject: Make PrioritizableNode immutable --- .../provision/provisioning/NodeAllocation.java | 66 ++++++++++++---------- .../provision/provisioning/NodePrioritizer.java | 7 +-- .../provision/provisioning/PrioritizableNode.java | 10 +++- 3 files changed, 45 insertions(+), 38 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index a37da10f5f0..758f46f5113 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -21,8 +21,9 @@ import java.util.Collection; import java.util.Comparator; import java.util.EnumSet; import java.util.HashSet; -import java.util.LinkedHashSet; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Predicate; @@ -49,7 +50,7 @@ class NodeAllocation { private final NodeSpec requestedNodes; /** The nodes this has accepted so far */ - private final Set nodes = new LinkedHashSet<>(); + private final Map nodes = new LinkedHashMap<>(); /** The number of already allocated nodes accepted and not retired */ private int accepted = 0; @@ -127,7 +128,7 @@ class NodeAllocation { ++rejectedDueToInsufficientRealResources; continue; } - if ( violatesParentHostPolicy(this.nodes, offered)) { + if ( violatesParentHostPolicy(offered)) { ++rejectedDueToClashingParentHost; continue; } @@ -142,10 +143,10 @@ class NodeAllocation { if (offered.status().wantToRetire()) { continue; } - node.node = offered.allocate(application, - ClusterMembership.from(cluster, highestIndex.add(1)), - requestedNodes.resources().orElse(node.node.resources()), - nodeRepository.clock().instant()); + node = node.withNode(offered.allocate(application, + ClusterMembership.from(cluster, highestIndex.add(1)), + requestedNodes.resources().orElse(node.node.resources()), + nodeRepository.clock().instant())); accepted.add(acceptNode(node, false, false)); } } @@ -156,15 +157,15 @@ class NodeAllocation { private boolean shouldRetire(PrioritizableNode node) { if ( ! requestedNodes.considerRetiring()) return false; if ( ! nodeResourceLimits.isWithinRealLimits(node.node, cluster)) return true; - if (violatesParentHostPolicy(this.nodes, node.node)) return true; + if (violatesParentHostPolicy(node.node)) return true; if ( ! hasCompatibleFlavor(node)) return true; if (node.node.status().wantToRetire()) return true; if (requestedNodes.isExclusive() && ! hostsOnly(application, node.node.parentHostname())) return true; return false; } - private boolean violatesParentHostPolicy(Collection accepted, Node offered) { - return checkForClashingParentHost() && offeredNodeHasParentHostnameAlreadyAccepted(accepted, offered); + private boolean violatesParentHostPolicy(Node offered) { + return checkForClashingParentHost() && offeredNodeHasParentHostnameAlreadyAccepted(offered); } private boolean checkForClashingParentHost() { @@ -173,8 +174,8 @@ class NodeAllocation { ! application.instance().isTester(); } - private boolean offeredNodeHasParentHostnameAlreadyAccepted(Collection accepted, Node offered) { - for (PrioritizableNode acceptedNode : accepted) { + private boolean offeredNodeHasParentHostnameAlreadyAccepted(Node offered) { + for (PrioritizableNode acceptedNode : nodes.values()) { if (acceptedNode.node.parentHostname().isPresent() && offered.parentHostname().isPresent() && acceptedNode.node.parentHostname().get().equals(offered.parentHostname().get())) { return true; @@ -271,13 +272,17 @@ class NodeAllocation { // group may be different node = setCluster(cluster, node); } - prioritizableNode.node = node; + prioritizableNode = prioritizableNode.withNode(node); indexes.add(node.allocation().get().membership().index()); highestIndex.set(Math.max(highestIndex.get(), node.allocation().get().membership().index())); - nodes.add(prioritizableNode); + update(prioritizableNode); return node; } + private void update(PrioritizableNode node) { + nodes.put(node.node.hostname(), node); + } + private Node resize(Node node) { NodeResources hostResources = allNodes.parentOf(node).get().flavor().resources(); return node.with(new Flavor(requestedNodes.resources().get() @@ -323,36 +328,39 @@ class NodeAllocation { * @return the final list of nodes */ List finalNodes() { - int currentRetiredCount = (int) nodes.stream().filter(node -> node.node.allocation().get().membership().retired()).count(); + int currentRetiredCount = (int) nodes.values().stream().filter(node -> node.node.allocation().get().membership().retired()).count(); int deltaRetiredCount = requestedNodes.idealRetiredCount(nodes.size(), currentRetiredCount) - currentRetiredCount; if (deltaRetiredCount > 0) { // retire until deltaRetiredCount is 0 - for (PrioritizableNode node : byRetiringPriority(nodes)) { + for (PrioritizableNode node : byRetiringPriority(nodes.values())) { if ( ! node.node.allocation().get().membership().retired() && node.node.state() == Node.State.active) { - node.node = node.node.retire(Agent.application, nodeRepository.clock().instant()); + PrioritizableNode newNode = node.withNode(node.node.retire(Agent.application, nodeRepository.clock().instant())); + update(newNode); if (--deltaRetiredCount == 0) break; } } } else if (deltaRetiredCount < 0) { // unretire until deltaRetiredCount is 0 - for (PrioritizableNode node : byUnretiringPriority(nodes)) { + for (PrioritizableNode node : byUnretiringPriority(nodes.values())) { if ( node.node.allocation().get().membership().retired() && hasCompatibleFlavor(node) ) { if (node.isResizable) - node.node = resize(node.node); - node.node = node.node.unretire(); + node = node.withNode(resize(node.node)); + node = node.withNode(node.node.unretire()); + update(node); if (++deltaRetiredCount == 0) break; } } } - for (PrioritizableNode node : nodes) { + for (PrioritizableNode node : nodes.values()) { // Set whether the node is exclusive Allocation allocation = node.node.allocation().get(); - node.node = node.node.with(allocation.with(allocation.membership() - .with(allocation.membership().cluster().exclusive(requestedNodes.isExclusive())))); + node = node.withNode(node.node.with(allocation.with(allocation.membership() + .with(allocation.membership().cluster().exclusive(requestedNodes.isExclusive()))))); + update(node); } - return nodes.stream().map(n -> n.node).collect(Collectors.toList()); + return nodes.values().stream().map(n -> n.node).collect(Collectors.toList()); } List reservableNodes() { @@ -361,28 +369,24 @@ class NodeAllocation { return nodesFilter(n -> !n.isNewNode && reservableStates.contains(n.node.state())); } - List surplusNodes() { - return nodesFilter(n -> n.isSurplusNode); - } - List newNodes() { return nodesFilter(n -> n.isNewNode); } private List nodesFilter(Predicate predicate) { - return nodes.stream() + return nodes.values().stream() .filter(predicate) .map(n -> n.node) .collect(Collectors.toList()); } /** Prefer to retire nodes we want the least */ - private List byRetiringPriority(Set nodes) { + private List byRetiringPriority(Collection nodes) { return nodes.stream().sorted(Comparator.reverseOrder()).collect(Collectors.toList()); } /** Prefer to unretire nodes we don't want to retire, and otherwise those with lower index */ - private List byUnretiringPriority(Set nodes) { + private List byUnretiringPriority(Collection nodes) { return nodes.stream() .sorted(Comparator.comparing((PrioritizableNode n) -> n.node.status().wantToRetire()) .thenComparing(n -> n.node.allocation().get().membership().index())) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index 3dc7eefa277..d5951490729 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -5,21 +5,20 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; - -import java.util.ArrayList; -import java.util.logging.Level; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.IP; +import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -81,7 +80,7 @@ public class NodePrioritizer { this.isDocker = resources(requestedNodes) != null; } - /** Returns the list of nodes sorted by PrioritizableNode::compare */ + /** Returns the list of nodes sorted by {@link PrioritizableNode#compareTo(PrioritizableNode)} */ List prioritize() { return nodes.values().stream().sorted().collect(Collectors.toList()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java index 9955d75a742..dd1ecb453db 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java @@ -8,7 +8,7 @@ import java.util.List; import java.util.Optional; /** - * A node with additional information required to prioritize it for allocation. + * A node with additional information required to prioritize it for allocation. This is immutable. * * @author smorgrav */ @@ -21,8 +21,7 @@ class PrioritizableNode implements Comparable { private static final NodeResources zeroResources = new NodeResources(0, 0, 0, 0, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); - // TODO: Make immutable - Node node; + final Node node; /** The free capacity on the parent of this node, before adding this node to it */ private final NodeResources freeParentCapacity; @@ -138,6 +137,11 @@ class PrioritizableNode implements Comparable { /** Returns the allocation skew of the parent of this after adding this node to it */ double skewWithThis() { return skewWith(node.resources()); } + /** Returns a copy of this with node set to given value */ + PrioritizableNode withNode(Node node) { + return new PrioritizableNode(node, freeParentCapacity, parent, violatesSpares, isSurplusNode, isNewNode, isResizable); + } + private boolean lessThanHalfTheHost(PrioritizableNode node) { var n = node.node.resources(); var h = node.parent.get().resources(); -- cgit v1.2.3 From 65d3b3c3d953e5b97785e7e0452525eb7888f74c Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Tue, 15 Sep 2020 11:31:01 +0200 Subject: Rename PrioritizableNode -> NodeCandidate --- .../provision/provisioning/GroupPreparer.java | 14 +- .../provision/provisioning/NodeAllocation.java | 112 +++++----- .../provision/provisioning/NodeCandidate.java | 234 +++++++++++++++++++++ .../provision/provisioning/NodePrioritizer.java | 33 ++- .../provision/provisioning/PrioritizableNode.java | 234 --------------------- .../provision/provisioning/NodeCandidateTest.java | 149 +++++++++++++ .../provisioning/PrioritizableNodeTest.java | 149 ------------- 7 files changed, 461 insertions(+), 464 deletions(-) create mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java delete mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index d3faa4d80f5..75f3c892571 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -99,13 +99,13 @@ public class GroupPreparer { nodeRepository.addNodes(hosts, Agent.application); // Offer the nodes on the newly provisioned hosts, this should be enough to cover the deficit - List nodes = provisionedHosts.stream() - .map(provisionedHost -> new PrioritizableNode.Builder(provisionedHost.generateNode()) - .parent(provisionedHost.generateHost()) - .newNode(true) - .build()) - .collect(Collectors.toList()); - allocation.offer(nodes); + List candidates = provisionedHosts.stream() + .map(provisionedHost -> new NodeCandidate.Builder(provisionedHost.generateNode()) + .parent(provisionedHost.generateHost()) + .newNode(true) + .build()) + .collect(Collectors.toList()); + allocation.offer(candidates); } if (! allocation.fulfilled() && requestedNodes.canFail()) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index 758f46f5113..b07ce786685 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -49,8 +49,8 @@ class NodeAllocation { /** The requested nodes of this list */ private final NodeSpec requestedNodes; - /** The nodes this has accepted so far */ - private final Map nodes = new LinkedHashMap<>(); + /** The node candidates this has accepted so far, keyed on hostname */ + private final Map nodes = new LinkedHashMap<>(); /** The number of already allocated nodes accepted and not retired */ private int accepted = 0; @@ -99,31 +99,31 @@ class NodeAllocation { * @param nodesPrioritized the nodes which are potentially on offer. These may belong to a different application etc. * @return the subset of offeredNodes which was accepted, with the correct allocation assigned */ - List offer(List nodesPrioritized) { + List offer(List nodesPrioritized) { List accepted = new ArrayList<>(); - for (PrioritizableNode node : nodesPrioritized) { - Node offered = node.node; + for (NodeCandidate candidate : nodesPrioritized) { + Node offered = candidate.node; if (offered.allocation().isPresent()) { Allocation allocation = offered.allocation().get(); ClusterMembership membership = allocation.membership(); if ( ! allocation.owner().equals(application)) continue; // wrong application if ( ! membership.cluster().satisfies(cluster)) continue; // wrong cluster id/type - if ((! node.isSurplusNode || saturated()) && ! membership.cluster().group().equals(cluster.group())) continue; // wrong group and we can't or have no reason to change it + if ((! candidate.isSurplusNode || saturated()) && ! membership.cluster().group().equals(cluster.group())) continue; // wrong group and we can't or have no reason to change it if ( offered.state() == Node.State.active && allocation.isRemovable()) continue; // don't accept; causes removal if ( indexes.contains(membership.index())) continue; // duplicate index (just to be sure) boolean resizeable = false; boolean acceptToRetire = false; if (requestedNodes.considerRetiring()) { - resizeable = node.isResizable; - acceptToRetire = acceptToRetire(node); + resizeable = candidate.isResizable; + acceptToRetire = acceptToRetire(candidate); } - if ((! saturated() && hasCompatibleFlavor(node) && requestedNodes.acceptable(offered)) || acceptToRetire) - accepted.add(acceptNode(node, shouldRetire(node), resizeable)); + if ((! saturated() && hasCompatibleFlavor(candidate) && requestedNodes.acceptable(offered)) || acceptToRetire) + accepted.add(acceptNode(candidate, shouldRetire(candidate), resizeable)); } - else if (! saturated() && hasCompatibleFlavor(node)) { + else if (! saturated() && hasCompatibleFlavor(candidate)) { if ( ! nodeResourceLimits.isWithinRealLimits(offered, cluster)) { ++rejectedDueToInsufficientRealResources; continue; @@ -143,24 +143,24 @@ class NodeAllocation { if (offered.status().wantToRetire()) { continue; } - node = node.withNode(offered.allocate(application, + candidate = candidate.withNode(offered.allocate(application, ClusterMembership.from(cluster, highestIndex.add(1)), - requestedNodes.resources().orElse(node.node.resources()), + requestedNodes.resources().orElse(candidate.node.resources()), nodeRepository.clock().instant())); - accepted.add(acceptNode(node, false, false)); + accepted.add(acceptNode(candidate, false, false)); } } return accepted; } - private boolean shouldRetire(PrioritizableNode node) { + private boolean shouldRetire(NodeCandidate candidate) { if ( ! requestedNodes.considerRetiring()) return false; - if ( ! nodeResourceLimits.isWithinRealLimits(node.node, cluster)) return true; - if (violatesParentHostPolicy(node.node)) return true; - if ( ! hasCompatibleFlavor(node)) return true; - if (node.node.status().wantToRetire()) return true; - if (requestedNodes.isExclusive() && ! hostsOnly(application, node.node.parentHostname())) return true; + if ( ! nodeResourceLimits.isWithinRealLimits(candidate.node, cluster)) return true; + if (violatesParentHostPolicy(candidate.node)) return true; + if ( ! hasCompatibleFlavor(candidate)) return true; + if (candidate.node.status().wantToRetire()) return true; + if (requestedNodes.isExclusive() && ! hostsOnly(application, candidate.node.parentHostname())) return true; return false; } @@ -175,7 +175,7 @@ class NodeAllocation { } private boolean offeredNodeHasParentHostnameAlreadyAccepted(Node offered) { - for (PrioritizableNode acceptedNode : nodes.values()) { + for (NodeCandidate acceptedNode : nodes.values()) { if (acceptedNode.node.parentHostname().isPresent() && offered.parentHostname().isPresent() && acceptedNode.node.parentHostname().get().equals(offered.parentHostname().get())) { return true; @@ -231,21 +231,21 @@ class NodeAllocation { * initialized. (In the other case, where a container node is not desired because we have enough nodes we * do want to remove it immediately to get immediate feedback on how the size reduction works out.) */ - private boolean acceptToRetire(PrioritizableNode node) { - if (node.node.state() != Node.State.active) return false; - if (! node.node.allocation().get().membership().cluster().group().equals(cluster.group())) return false; - if (node.node.allocation().get().membership().retired()) return true; // don't second-guess if already retired + private boolean acceptToRetire(NodeCandidate candidate) { + if (candidate.node.state() != Node.State.active) return false; + if (! candidate.node.allocation().get().membership().cluster().group().equals(cluster.group())) return false; + if (candidate.node.allocation().get().membership().retired()) return true; // don't second-guess if already retired return cluster.type().isContent() || - (cluster.type() == ClusterSpec.Type.container && !hasCompatibleFlavor(node)); + (cluster.type() == ClusterSpec.Type.container && !hasCompatibleFlavor(candidate)); } - private boolean hasCompatibleFlavor(PrioritizableNode node) { - return requestedNodes.isCompatible(node.node.flavor(), nodeRepository.flavors()) || node.isResizable; + private boolean hasCompatibleFlavor(NodeCandidate candidate) { + return requestedNodes.isCompatible(candidate.node.flavor(), nodeRepository.flavors()) || candidate.isResizable; } - private Node acceptNode(PrioritizableNode prioritizableNode, boolean wantToRetire, boolean resizeable) { - Node node = prioritizableNode.node; + private Node acceptNode(NodeCandidate candidate, boolean wantToRetire, boolean resizeable) { + Node node = candidate.node; if (node.allocation().isPresent()) // Record the currently requested resources node = node.with(node.allocation().get().withRequestedResources(requestedNodes.resources().orElse(node.resources()))); @@ -272,15 +272,15 @@ class NodeAllocation { // group may be different node = setCluster(cluster, node); } - prioritizableNode = prioritizableNode.withNode(node); + candidate = candidate.withNode(node); indexes.add(node.allocation().get().membership().index()); highestIndex.set(Math.max(highestIndex.get(), node.allocation().get().membership().index())); - update(prioritizableNode); + put(candidate); return node; } - private void update(PrioritizableNode node) { - nodes.put(node.node.hostname(), node); + private void put(NodeCandidate candidate) { + nodes.put(candidate.node.hostname(), candidate); } private Node resize(Node node) { @@ -332,32 +332,32 @@ class NodeAllocation { int deltaRetiredCount = requestedNodes.idealRetiredCount(nodes.size(), currentRetiredCount) - currentRetiredCount; if (deltaRetiredCount > 0) { // retire until deltaRetiredCount is 0 - for (PrioritizableNode node : byRetiringPriority(nodes.values())) { - if ( ! node.node.allocation().get().membership().retired() && node.node.state() == Node.State.active) { - PrioritizableNode newNode = node.withNode(node.node.retire(Agent.application, nodeRepository.clock().instant())); - update(newNode); + for (NodeCandidate candidate : byRetiringPriority(nodes.values())) { + if ( ! candidate.node.allocation().get().membership().retired() && candidate.node.state() == Node.State.active) { + candidate = candidate.withNode(candidate.node.retire(Agent.application, nodeRepository.clock().instant())); + put(candidate); if (--deltaRetiredCount == 0) break; } } } else if (deltaRetiredCount < 0) { // unretire until deltaRetiredCount is 0 - for (PrioritizableNode node : byUnretiringPriority(nodes.values())) { - if ( node.node.allocation().get().membership().retired() && hasCompatibleFlavor(node) ) { - if (node.isResizable) - node = node.withNode(resize(node.node)); - node = node.withNode(node.node.unretire()); - update(node); + for (NodeCandidate candidate : byUnretiringPriority(nodes.values())) { + if ( candidate.node.allocation().get().membership().retired() && hasCompatibleFlavor(candidate) ) { + if (candidate.isResizable) + candidate = candidate.withNode(resize(candidate.node)); + candidate = candidate.withNode(candidate.node.unretire()); + put(candidate); if (++deltaRetiredCount == 0) break; } } } - for (PrioritizableNode node : nodes.values()) { + for (NodeCandidate candidate : nodes.values()) { // Set whether the node is exclusive - Allocation allocation = node.node.allocation().get(); - node = node.withNode(node.node.with(allocation.with(allocation.membership() + Allocation allocation = candidate.node.allocation().get(); + candidate = candidate.withNode(candidate.node.with(allocation.with(allocation.membership() .with(allocation.membership().cluster().exclusive(requestedNodes.isExclusive()))))); - update(node); + put(candidate); } return nodes.values().stream().map(n -> n.node).collect(Collectors.toList()); @@ -373,7 +373,7 @@ class NodeAllocation { return nodesFilter(n -> n.isNewNode); } - private List nodesFilter(Predicate predicate) { + private List nodesFilter(Predicate predicate) { return nodes.values().stream() .filter(predicate) .map(n -> n.node) @@ -381,16 +381,16 @@ class NodeAllocation { } /** Prefer to retire nodes we want the least */ - private List byRetiringPriority(Collection nodes) { - return nodes.stream().sorted(Comparator.reverseOrder()).collect(Collectors.toList()); + private List byRetiringPriority(Collection candidates) { + return candidates.stream().sorted(Comparator.reverseOrder()).collect(Collectors.toList()); } /** Prefer to unretire nodes we don't want to retire, and otherwise those with lower index */ - private List byUnretiringPriority(Collection nodes) { - return nodes.stream() - .sorted(Comparator.comparing((PrioritizableNode n) -> n.node.status().wantToRetire()) - .thenComparing(n -> n.node.allocation().get().membership().index())) - .collect(Collectors.toList()); + private List byUnretiringPriority(Collection candidates) { + return candidates.stream() + .sorted(Comparator.comparing((NodeCandidate n) -> n.node.status().wantToRetire()) + .thenComparing(n -> n.node.allocation().get().membership().index())) + .collect(Collectors.toList()); } public String outOfCapacityDetails() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java new file mode 100644 index 00000000000..651ab9b1e09 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java @@ -0,0 +1,234 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.NodeResources; +import com.yahoo.vespa.hosted.provision.Node; + +import java.util.List; +import java.util.Optional; + +/** + * A node candidate containing the details required to prioritize it for allocation. This is immutable. + * + * @author smorgrav + */ +class NodeCandidate implements Comparable { + + /** List of host states ordered by preference (ascending) */ + private static final List HOST_STATE_PRIORITY = + List.of(Node.State.provisioned, Node.State.ready, Node.State.active); + + private static final NodeResources zeroResources = + new NodeResources(0, 0, 0, 0, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); + + final Node node; + + /** The free capacity on the parent of this node, before adding this node to it */ + private final NodeResources freeParentCapacity; + + /** The parent host (docker or hypervisor) */ + final Optional parent; + + /** True if the node is allocated to a host that should be dedicated as a spare */ + final boolean violatesSpares; + + /** True if this node belongs to a group which will not be needed after this deployment */ + final boolean isSurplusNode; + + /** This node does not exist in the node repository yet */ + final boolean isNewNode; + + /** This node can be resized to the new NodeResources */ + final boolean isResizable; + + NodeCandidate(Node node, NodeResources freeParentCapacity, Optional parent, boolean violatesSpares, boolean isSurplusNode, boolean isNewNode, boolean isResizeable) { + if (isResizeable && isNewNode) + throw new IllegalArgumentException("A new node cannot be resizable"); + + this.node = node; + this.freeParentCapacity = freeParentCapacity; + this.parent = parent; + this.violatesSpares = violatesSpares; + this.isSurplusNode = isSurplusNode; + this.isNewNode = isNewNode; + this.isResizable = isResizeable; + } + + /** + * Compare this candidate to another + * + * @return negative if first priority is higher than second node + */ + @Override + public int compareTo(NodeCandidate other) { + // First always pick nodes without violation above nodes with violations + if (!this.violatesSpares && other.violatesSpares) return -1; + if (!other.violatesSpares && this.violatesSpares) return 1; + + // Choose active nodes + if (this.node.state() == Node.State.active && other.node.state() != Node.State.active) return -1; + if (other.node.state() == Node.State.active && this.node.state() != Node.State.active) return 1; + + // Choose active node that is not retired first (surplus is active but retired) + if (!this.isSurplusNode && other.isSurplusNode) return -1; + if (!other.isSurplusNode && this.isSurplusNode) return 1; + + // Choose reserved nodes from a previous allocation attempt (the exist in node repo) + if (this.isInNodeRepoAndReserved() && ! other.isInNodeRepoAndReserved()) return -1; + if (other.isInNodeRepoAndReserved() && ! this.isInNodeRepoAndReserved()) return 1; + + // Choose inactive nodes + if (this.node.state() == Node.State.inactive && other.node.state() != Node.State.inactive) return -1; + if (other.node.state() == Node.State.inactive && this.node.state() != Node.State.inactive) return 1; + + // Choose ready nodes + if (this.node.state() == Node.State.ready && other.node.state() != Node.State.ready) return -1; + if (other.node.state() == Node.State.ready && this.node.state() != Node.State.ready) return 1; + + if (this.node.state() != other.node.state()) + throw new IllegalStateException("Nodes " + this.node + " and " + other.node + " have different states"); + + if (this.parent.isPresent() && other.parent.isPresent()) { + // Prefer reserved hosts (that they are reserved to the right tenant is ensured elsewhere) + if ( this.parent.get().reservedTo().isPresent() && ! other.parent.get().reservedTo().isPresent()) return -1; + if ( ! this.parent.get().reservedTo().isPresent() && other.parent.get().reservedTo().isPresent()) return 1; + + int diskCostDifference = NodeResources.DiskSpeed.compare(this.parent.get().flavor().resources().diskSpeed(), + other.parent.get().flavor().resources().diskSpeed()); + if (diskCostDifference != 0) + return diskCostDifference; + + int storageCostDifference = NodeResources.StorageType.compare(this.parent.get().flavor().resources().storageType(), + other.parent.get().flavor().resources().storageType()); + if (storageCostDifference != 0) + return storageCostDifference; + + // Prefer hosts that are at least twice the size of this node + // (utilization is more even if one application does not dominate the host) + if ( lessThanHalfTheHost(this) && ! lessThanHalfTheHost(other)) return -1; + if ( ! lessThanHalfTheHost(this) && lessThanHalfTheHost(other)) return 1; + } + + int hostPriority = Double.compare(this.skewWithThis() - this.skewWithoutThis(), + other.skewWithThis() - other.skewWithoutThis()); + if (hostPriority != 0) return hostPriority; + + // Choose cheapest node + if (this.node.flavor().cost() < other.node.flavor().cost()) return -1; + if (other.node.flavor().cost() < this.node.flavor().cost()) return 1; + + // Choose nodes where host is in more desirable state + int thisHostStatePri = this.parent.map(host -> HOST_STATE_PRIORITY.indexOf(host.state())).orElse(-2); + int otherHostStatePri = other.parent.map(host -> HOST_STATE_PRIORITY.indexOf(host.state())).orElse(-2); + if (thisHostStatePri != otherHostStatePri) return otherHostStatePri - thisHostStatePri; + + // Prefer lower indexes to minimize redistribution + if (this.node.allocation().isPresent() && other.node.allocation().isPresent()) + return Integer.compare(this.node.allocation().get().membership().index(), + other.node.allocation().get().membership().index()); + + // All else equal choose hostname alphabetically + return this.node.hostname().compareTo(other.node.hostname()); + } + + /** Returns the allocation skew of the parent of this before adding this node to it */ + double skewWithoutThis() { return skewWith(zeroResources); } + + /** Returns the allocation skew of the parent of this after adding this node to it */ + double skewWithThis() { return skewWith(node.resources()); } + + /** Returns a copy of this with node set to given value */ + NodeCandidate withNode(Node node) { + return new NodeCandidate(node, freeParentCapacity, parent, violatesSpares, isSurplusNode, isNewNode, isResizable); + } + + private boolean lessThanHalfTheHost(NodeCandidate node) { + var n = node.node.resources(); + var h = node.parent.get().resources(); + if (h.vcpu() < n.vcpu() * 2) return false; + if (h.memoryGb() < n.memoryGb() * 2) return false; + if (h.diskGb() < n.diskGb() * 2) return false; + return true; + } + + private double skewWith(NodeResources resources) { + if (parent.isEmpty()) return 0; + + NodeResources free = freeParentCapacity.justNumbers().subtract(resources.justNumbers()); + return Node.skew(parent.get().flavor().resources(), free); + } + + private boolean isInNodeRepoAndReserved() { + if (isNewNode) return false; + return node.state().equals(Node.State.reserved); + } + + @Override + public String toString() { + return node.id(); + } + + @Override + public int hashCode() { + return node.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == this) return true; + if ( ! (other instanceof NodeCandidate)) return false; + return this.node.equals(((NodeCandidate)other).node); + } + + static class Builder { + + public final Node node; + private NodeResources freeParentCapacity; + private Optional parent = Optional.empty(); + private boolean violatesSpares; + private boolean isSurplusNode; + private boolean isNewNode; + private boolean isResizable; + + Builder(Node node) { + this.node = node; + this.freeParentCapacity = node.flavor().resources(); + } + + /** The free capacity of the parent, before adding this node to it */ + Builder freeParentCapacity(NodeResources freeParentCapacity) { + this.freeParentCapacity = freeParentCapacity; + return this; + } + + Builder parent(Node parent) { + this.parent = Optional.of(parent); + return this; + } + + Builder violatesSpares(boolean violatesSpares) { + this.violatesSpares = violatesSpares; + return this; + } + + Builder surplusNode(boolean surplusNode) { + isSurplusNode = surplusNode; + return this; + } + + Builder newNode(boolean newNode) { + isNewNode = newNode; + return this; + } + + Builder resizable(boolean resizable) { + isResizable = resizable; + return this; + } + + NodeCandidate build() { + return new NodeCandidate(node, freeParentCapacity, parent, violatesSpares, isSurplusNode, isNewNode, isResizable); + } + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index d5951490729..226079e43fe 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -24,9 +24,9 @@ import java.util.stream.Collectors; /** * Builds up data structures necessary for node prioritization. It wraps each node - * up in a PrioritizableNode object with attributes used in sorting. + * up in a {@link NodeCandidate} object with attributes used in sorting. * - * The actual sorting/prioritization is implemented in the PrioritizableNode class as a compare method. + * The prioritization logic is implemented by {@link NodeCandidate}. * * @author smorgrav */ @@ -34,7 +34,7 @@ public class NodePrioritizer { private final static Logger log = Logger.getLogger(NodePrioritizer.class.getName()); - private final Map nodes = new HashMap<>(); + private final Map nodes = new HashMap<>(); private final LockedNodeList allNodes; private final HostCapacity capacity; private final NodeSpec requestedNodes; @@ -80,8 +80,8 @@ public class NodePrioritizer { this.isDocker = resources(requestedNodes) != null; } - /** Returns the list of nodes sorted by {@link PrioritizableNode#compareTo(PrioritizableNode)} */ - List prioritize() { + /** Returns the list of nodes sorted by {@link NodeCandidate#compareTo(NodeCandidate)} */ + List prioritize() { return nodes.values().stream().sorted().collect(Collectors.toList()); } @@ -91,7 +91,7 @@ public class NodePrioritizer { */ void addSurplusNodes(List surplusNodes) { for (Node node : surplusNodes) { - PrioritizableNode nodePri = toPrioritizable(node, true, false); + NodeCandidate nodePri = candidateFrom(node, true, false); if (!nodePri.violatesSpares || isAllocatingForReplacement) { nodes.put(node, nodePri); } @@ -142,7 +142,7 @@ public class NodePrioritizer { resources(requestedNodes).with(host.flavor().resources().diskSpeed()) .with(host.flavor().resources().storageType()), NodeType.tenant); - PrioritizableNode nodePri = toPrioritizable(newNode, false, true); + NodeCandidate nodePri = candidateFrom(newNode, false, true); if ( ! nodePri.violatesSpares || isAllocatingForReplacement) { log.log(Level.FINE, "Adding new Docker node " + newNode); nodes.put(newNode, nodePri); @@ -158,7 +158,7 @@ public class NodePrioritizer { .filter(node -> legalStates.contains(node.state())) .filter(node -> node.allocation().isPresent()) .filter(node -> node.allocation().get().owner().equals(application)) - .map(node -> toPrioritizable(node, false, false)) + .map(node -> candidateFrom(node, false, false)) .forEach(prioritizableNode -> nodes.put(prioritizableNode.node, prioritizableNode)); } @@ -167,20 +167,17 @@ public class NodePrioritizer { allNodes.asList().stream() .filter(node -> node.type() == requestedNodes.type()) .filter(node -> node.state() == Node.State.ready) - .map(node -> toPrioritizable(node, false, false)) + .map(node -> candidateFrom(node, false, false)) .filter(n -> !n.violatesSpares || isAllocatingForReplacement) - .forEach(prioritizableNode -> nodes.put(prioritizableNode.node, prioritizableNode)); + .forEach(candidate -> nodes.put(candidate.node, candidate)); } - public List nodes() { return new ArrayList<>(nodes.values()); } + public List nodes() { return new ArrayList<>(nodes.values()); } - /** - * Convert a list of nodes to a list of node priorities. This includes finding, calculating - * parameters to the priority sorting procedure. - */ - private PrioritizableNode toPrioritizable(Node node, boolean isSurplusNode, boolean isNewNode) { - PrioritizableNode.Builder builder = new PrioritizableNode.Builder(node).surplusNode(isSurplusNode) - .newNode(isNewNode); + /** Create a candidate from given node */ + private NodeCandidate candidateFrom(Node node, boolean isSurplusNode, boolean isNewNode) { + NodeCandidate.Builder builder = new NodeCandidate.Builder(node).surplusNode(isSurplusNode) + .newNode(isNewNode); allNodes.parentOf(node).ifPresent(parent -> { NodeResources parentCapacity = capacity.freeCapacityOf(parent, false); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java deleted file mode 100644 index dd1ecb453db..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java +++ /dev/null @@ -1,234 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.provisioning; - -import com.yahoo.config.provision.NodeResources; -import com.yahoo.vespa.hosted.provision.Node; - -import java.util.List; -import java.util.Optional; - -/** - * A node with additional information required to prioritize it for allocation. This is immutable. - * - * @author smorgrav - */ -class PrioritizableNode implements Comparable { - - /** List of host states ordered by preference (ascending) */ - private static final List HOST_STATE_PRIORITY = - List.of(Node.State.provisioned, Node.State.ready, Node.State.active); - - private static final NodeResources zeroResources = - new NodeResources(0, 0, 0, 0, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); - - final Node node; - - /** The free capacity on the parent of this node, before adding this node to it */ - private final NodeResources freeParentCapacity; - - /** The parent host (docker or hypervisor) */ - final Optional parent; - - /** True if the node is allocated to a host that should be dedicated as a spare */ - final boolean violatesSpares; - - /** True if this node belongs to a group which will not be needed after this deployment */ - final boolean isSurplusNode; - - /** This node does not exist in the node repository yet */ - final boolean isNewNode; - - /** This node can be resized to the new NodeResources */ - final boolean isResizable; - - PrioritizableNode(Node node, NodeResources freeParentCapacity, Optional parent, boolean violatesSpares, boolean isSurplusNode, boolean isNewNode, boolean isResizeable) { - if (isResizeable && isNewNode) - throw new IllegalArgumentException("A new node cannot be resizable"); - - this.node = node; - this.freeParentCapacity = freeParentCapacity; - this.parent = parent; - this.violatesSpares = violatesSpares; - this.isSurplusNode = isSurplusNode; - this.isNewNode = isNewNode; - this.isResizable = isResizeable; - } - - /** - * Compares two prioritizable nodes - * - * @return negative if first priority is higher than second node - */ - @Override - public int compareTo(PrioritizableNode other) { - // First always pick nodes without violation above nodes with violations - if (!this.violatesSpares && other.violatesSpares) return -1; - if (!other.violatesSpares && this.violatesSpares) return 1; - - // Choose active nodes - if (this.node.state() == Node.State.active && other.node.state() != Node.State.active) return -1; - if (other.node.state() == Node.State.active && this.node.state() != Node.State.active) return 1; - - // Choose active node that is not retired first (surplus is active but retired) - if (!this.isSurplusNode && other.isSurplusNode) return -1; - if (!other.isSurplusNode && this.isSurplusNode) return 1; - - // Choose reserved nodes from a previous allocation attempt (the exist in node repo) - if (this.isInNodeRepoAndReserved() && ! other.isInNodeRepoAndReserved()) return -1; - if (other.isInNodeRepoAndReserved() && ! this.isInNodeRepoAndReserved()) return 1; - - // Choose inactive nodes - if (this.node.state() == Node.State.inactive && other.node.state() != Node.State.inactive) return -1; - if (other.node.state() == Node.State.inactive && this.node.state() != Node.State.inactive) return 1; - - // Choose ready nodes - if (this.node.state() == Node.State.ready && other.node.state() != Node.State.ready) return -1; - if (other.node.state() == Node.State.ready && this.node.state() != Node.State.ready) return 1; - - if (this.node.state() != other.node.state()) - throw new IllegalStateException("Nodes " + this.node + " and " + other.node + " have different states"); - - if (this.parent.isPresent() && other.parent.isPresent()) { - // Prefer reserved hosts (that they are reserved to the right tenant is ensured elsewhere) - if ( this.parent.get().reservedTo().isPresent() && ! other.parent.get().reservedTo().isPresent()) return -1; - if ( ! this.parent.get().reservedTo().isPresent() && other.parent.get().reservedTo().isPresent()) return 1; - - int diskCostDifference = NodeResources.DiskSpeed.compare(this.parent.get().flavor().resources().diskSpeed(), - other.parent.get().flavor().resources().diskSpeed()); - if (diskCostDifference != 0) - return diskCostDifference; - - int storageCostDifference = NodeResources.StorageType.compare(this.parent.get().flavor().resources().storageType(), - other.parent.get().flavor().resources().storageType()); - if (storageCostDifference != 0) - return storageCostDifference; - - // Prefer hosts that are at least twice the size of this node - // (utilization is more even if one application does not dominate the host) - if ( lessThanHalfTheHost(this) && ! lessThanHalfTheHost(other)) return -1; - if ( ! lessThanHalfTheHost(this) && lessThanHalfTheHost(other)) return 1; - } - - int hostPriority = Double.compare(this.skewWithThis() - this.skewWithoutThis(), - other.skewWithThis() - other.skewWithoutThis()); - if (hostPriority != 0) return hostPriority; - - // Choose cheapest node - if (this.node.flavor().cost() < other.node.flavor().cost()) return -1; - if (other.node.flavor().cost() < this.node.flavor().cost()) return 1; - - // Choose nodes where host is in more desirable state - int thisHostStatePri = this.parent.map(host -> HOST_STATE_PRIORITY.indexOf(host.state())).orElse(-2); - int otherHostStatePri = other.parent.map(host -> HOST_STATE_PRIORITY.indexOf(host.state())).orElse(-2); - if (thisHostStatePri != otherHostStatePri) return otherHostStatePri - thisHostStatePri; - - // Prefer lower indexes to minimize redistribution - if (this.node.allocation().isPresent() && other.node.allocation().isPresent()) - return Integer.compare(this.node.allocation().get().membership().index(), - other.node.allocation().get().membership().index()); - - // All else equal choose hostname alphabetically - return this.node.hostname().compareTo(other.node.hostname()); - } - - /** Returns the allocation skew of the parent of this before adding this node to it */ - double skewWithoutThis() { return skewWith(zeroResources); } - - /** Returns the allocation skew of the parent of this after adding this node to it */ - double skewWithThis() { return skewWith(node.resources()); } - - /** Returns a copy of this with node set to given value */ - PrioritizableNode withNode(Node node) { - return new PrioritizableNode(node, freeParentCapacity, parent, violatesSpares, isSurplusNode, isNewNode, isResizable); - } - - private boolean lessThanHalfTheHost(PrioritizableNode node) { - var n = node.node.resources(); - var h = node.parent.get().resources(); - if (h.vcpu() < n.vcpu() * 2) return false; - if (h.memoryGb() < n.memoryGb() * 2) return false; - if (h.diskGb() < n.diskGb() * 2) return false; - return true; - } - - private double skewWith(NodeResources resources) { - if (parent.isEmpty()) return 0; - - NodeResources free = freeParentCapacity.justNumbers().subtract(resources.justNumbers()); - return Node.skew(parent.get().flavor().resources(), free); - } - - private boolean isInNodeRepoAndReserved() { - if (isNewNode) return false; - return node.state().equals(Node.State.reserved); - } - - @Override - public String toString() { - return node.id(); - } - - @Override - public int hashCode() { - return node.hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == this) return true; - if ( ! (other instanceof PrioritizableNode)) return false; - return this.node.equals(((PrioritizableNode)other).node); - } - - static class Builder { - - public final Node node; - private NodeResources freeParentCapacity; - private Optional parent = Optional.empty(); - private boolean violatesSpares; - private boolean isSurplusNode; - private boolean isNewNode; - private boolean isResizable; - - Builder(Node node) { - this.node = node; - this.freeParentCapacity = node.flavor().resources(); - } - - /** The free capacity of the parent, before adding this node to it */ - Builder freeParentCapacity(NodeResources freeParentCapacity) { - this.freeParentCapacity = freeParentCapacity; - return this; - } - - Builder parent(Node parent) { - this.parent = Optional.of(parent); - return this; - } - - Builder violatesSpares(boolean violatesSpares) { - this.violatesSpares = violatesSpares; - return this; - } - - Builder surplusNode(boolean surplusNode) { - isSurplusNode = surplusNode; - return this; - } - - Builder newNode(boolean newNode) { - isNewNode = newNode; - return this; - } - - Builder resizable(boolean resizable) { - isResizable = resizable; - return this; - } - - PrioritizableNode build() { - return new PrioritizableNode(node, freeParentCapacity, parent, violatesSpares, isSurplusNode, isNewNode, isResizable); - } - } - -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java new file mode 100644 index 00000000000..95b9f334bb4 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java @@ -0,0 +1,149 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.History; +import com.yahoo.vespa.hosted.provision.node.IP; +import com.yahoo.vespa.hosted.provision.node.Reports; +import com.yahoo.vespa.hosted.provision.node.Status; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class NodeCandidateTest { + + @Test + public void test_order() { + List expected = List.of( + new NodeCandidate(node("01", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.empty(), false, true, false, false), + new NodeCandidate(node("02", Node.State.active), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, false, false), + new NodeCandidate(node("04", Node.State.reserved), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, false, false), + new NodeCandidate(node("03", Node.State.inactive), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, false, false), + new NodeCandidate(node("05", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.active)), true, false, true, false), + new NodeCandidate(node("06", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.ready)), true, false, true, false), + new NodeCandidate(node("07", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.provisioned)), true, false, true, false), + new NodeCandidate(node("08", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.failed)), true, false, true, false), + new NodeCandidate(node("09", Node.State.ready), new NodeResources(1, 1, 1, 1), Optional.empty(), true, false, true, false), + new NodeCandidate(node("10", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, true, false), + new NodeCandidate(node("11", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, true, false) + ); + assertOrder(expected); + } + + @Test + public void testOrderingByAllocationSkew1() { + List expected = List.of( + node("1", node(4, 4), host(20, 20), host(40, 40)), + node("2", node(4, 4), host(21, 20), host(40, 40)), + node("3", node(4, 4), host(22, 20), host(40, 40)), + node("4", node(4, 4), host(21, 22), host(40, 40)), + node("5", node(4, 4), host(21, 21), host(40, 80)) + ); + assertOrder(expected); + } + + @Test + public void testOrderingByAllocationSkew2() { + // The same as testOrderingByAllocationSkew1, but deviating from mean (20) in the other direction. + // Since we don't choose the node with the lowest skew, but with the largest skew *reduction* + // this causes the opposite order. + List expected = List.of( + node("4", node(4, 4), host(19, 18), host(40, 40)), + node("3", node(4, 4), host(18, 20), host(40, 40)), + node("2", node(4, 4), host(19, 20), host(40, 40)), + node("1", node(4, 4), host(20, 20), host(40, 40)), + node("5", node(4, 4), host(19, 19), host(40, 80)) + ); + assertOrder(expected); + } + + @Test + public void testOrderingByAllocationSkew3() { + // The same as testOrderingByAllocationSkew1, but allocating skewed towards cpu + List expected = List.of( + node("1", node(4, 2), host(20, 20), host(40, 40)), + node("2", node(4, 2), host(21, 20), host(40, 40)), + node("4", node(4, 2), host(21, 22), host(40, 40)), + node("3", node(4, 2), host(22, 20), host(40, 40)), + node("5", node(4, 2), host(21, 21), host(40, 80)) + ); + assertOrder(expected); + } + + @Test + public void testOrderingByAllocationSkew4() { + // The same as testOrderingByAllocationSkew1, but allocating skewed towards memory + List expected = List.of( + node("5", node(2, 10), host(21, 21), host(40, 80)), + node("3", node(2, 10), host(22, 20), host(40, 40)), + node("2", node(2, 10), host(21, 20), host(40, 40)), + node("1", node(2, 10), host(20, 20), host(40, 40)), + node("4", node(2, 10), host(21, 22), host(40, 40)) + ); + assertOrder(expected); + } + + @Test + public void testOrderingByAllocationSkew5() { + // node1 is skewed towards cpu (without this allocation), allocation is skewed towards memory, therefore + // node 1 is preferred (even though it is still most skewed) + List expected = List.of( + node("1", node(1, 5), host(21, 10), host(40, 40)), + node("2", node(1, 5), host(21, 20), host(40, 40)), + node("3", node(1, 5), host(20, 20), host(40, 40)), + node("4", node(1, 5), host(20, 22), host(40, 40)) + ); + assertOrder(expected); + } + + private void assertOrder(List expected) { + List copy = new ArrayList<>(expected); + Collections.shuffle(copy); + Collections.sort(copy); + assertEquals(expected, copy); + } + + private static NodeResources node(double vcpu, double mem) { + return new NodeResources(vcpu, mem, 0, 0); + } + + private static NodeResources host(double vcpu, double mem) { + return new NodeResources(vcpu, mem, 10, 10); + } + + private static Node node(String hostname, Node.State state) { + return new Node(hostname, new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.empty(), + new Flavor(new NodeResources(2, 2, 2, 2)), + Status.initial(), state, Optional.empty(), History.empty(), NodeType.tenant, new Reports(), + Optional.empty(), Optional.empty()); + } + + private static NodeCandidate node(String hostname, + NodeResources nodeResources, + NodeResources allocatedHostResources, // allocated before adding nodeResources + NodeResources totalHostResources) { + Node node = new Node(hostname, new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.of(hostname + "parent"), + new Flavor(nodeResources), + Status.initial(), Node.State.ready, Optional.empty(), History.empty(), NodeType.tenant, + new Reports(), Optional.empty(), Optional.empty()); + Node parent = new Node(hostname + "parent", new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.empty(), + new Flavor(totalHostResources), + Status.initial(), Node.State.ready, Optional.empty(), History.empty(), NodeType.host, + new Reports(), Optional.empty(), Optional.empty()); + return new NodeCandidate(node, totalHostResources.subtract(allocatedHostResources), Optional.of(parent), + false, false, true, false); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java deleted file mode 100644 index 3865baa51c1..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.provisioning; - -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.node.History; -import com.yahoo.vespa.hosted.provision.node.IP; -import com.yahoo.vespa.hosted.provision.node.Reports; -import com.yahoo.vespa.hosted.provision.node.Status; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.Set; - -import static org.junit.Assert.assertEquals; - -/** - * @author bratseth - */ -public class PrioritizableNodeTest { - - @Test - public void test_order() { - List expected = List.of( - new PrioritizableNode(node("01", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.empty(), false, true, false, false), - new PrioritizableNode(node("02", Node.State.active), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, false, false), - new PrioritizableNode(node("04", Node.State.reserved), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, false, false), - new PrioritizableNode(node("03", Node.State.inactive), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, false, false), - new PrioritizableNode(node("05", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.active)), true, false, true, false), - new PrioritizableNode(node("06", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.ready)), true, false, true, false), - new PrioritizableNode(node("07", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.provisioned)), true, false, true, false), - new PrioritizableNode(node("08", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.failed)), true, false, true, false), - new PrioritizableNode(node("09", Node.State.ready), new NodeResources(1, 1, 1, 1), Optional.empty(), true, false, true, false), - new PrioritizableNode(node("10", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, true, false), - new PrioritizableNode(node("11", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, true, false) - ); - assertOrder(expected); - } - - @Test - public void testOrderingByAllocationSkew1() { - List expected = List.of( - node("1", node(4, 4), host(20, 20), host(40, 40)), - node("2", node(4, 4), host(21, 20), host(40, 40)), - node("3", node(4, 4), host(22, 20), host(40, 40)), - node("4", node(4, 4), host(21, 22), host(40, 40)), - node("5", node(4, 4), host(21, 21), host(40, 80)) - ); - assertOrder(expected); - } - - @Test - public void testOrderingByAllocationSkew2() { - // The same as testOrderingByAllocationSkew1, but deviating from mean (20) in the other direction. - // Since we don't choose the node with the lowest skew, but with the largest skew *reduction* - // this causes the opposite order. - List expected = List.of( - node("4", node(4, 4), host(19, 18), host(40, 40)), - node("3", node(4, 4), host(18, 20), host(40, 40)), - node("2", node(4, 4), host(19, 20), host(40, 40)), - node("1", node(4, 4), host(20, 20), host(40, 40)), - node("5", node(4, 4), host(19, 19), host(40, 80)) - ); - assertOrder(expected); - } - - @Test - public void testOrderingByAllocationSkew3() { - // The same as testOrderingByAllocationSkew1, but allocating skewed towards cpu - List expected = List.of( - node("1", node(4, 2), host(20, 20), host(40, 40)), - node("2", node(4, 2), host(21, 20), host(40, 40)), - node("4", node(4, 2), host(21, 22), host(40, 40)), - node("3", node(4, 2), host(22, 20), host(40, 40)), - node("5", node(4, 2), host(21, 21), host(40, 80)) - ); - assertOrder(expected); - } - - @Test - public void testOrderingByAllocationSkew4() { - // The same as testOrderingByAllocationSkew1, but allocating skewed towards memory - List expected = List.of( - node("5", node(2, 10), host(21, 21), host(40, 80)), - node("3", node(2, 10), host(22, 20), host(40, 40)), - node("2", node(2, 10), host(21, 20), host(40, 40)), - node("1", node(2, 10), host(20, 20), host(40, 40)), - node("4", node(2, 10), host(21, 22), host(40, 40)) - ); - assertOrder(expected); - } - - @Test - public void testOrderingByAllocationSkew5() { - // node1 is skewed towards cpu (without this allocation), allocation is skewed towards memory, therefore - // node 1 is preferred (even though it is still most skewed) - List expected = List.of( - node("1", node(1, 5), host(21, 10), host(40, 40)), - node("2", node(1, 5), host(21, 20), host(40, 40)), - node("3", node(1, 5), host(20, 20), host(40, 40)), - node("4", node(1, 5), host(20, 22), host(40, 40)) - ); - assertOrder(expected); - } - - private void assertOrder(List expected) { - List copy = new ArrayList<>(expected); - Collections.shuffle(copy); - Collections.sort(copy); - assertEquals(expected, copy); - } - - private static NodeResources node(double vcpu, double mem) { - return new NodeResources(vcpu, mem, 0, 0); - } - - private static NodeResources host(double vcpu, double mem) { - return new NodeResources(vcpu, mem, 10, 10); - } - - private static Node node(String hostname, Node.State state) { - return new Node(hostname, new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.empty(), - new Flavor(new NodeResources(2, 2, 2, 2)), - Status.initial(), state, Optional.empty(), History.empty(), NodeType.tenant, new Reports(), - Optional.empty(), Optional.empty()); - } - - private static PrioritizableNode node(String hostname, - NodeResources nodeResources, - NodeResources allocatedHostResources, // allocated before adding nodeResources - NodeResources totalHostResources) { - Node node = new Node(hostname, new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.of(hostname + "parent"), - new Flavor(nodeResources), - Status.initial(), Node.State.ready, Optional.empty(), History.empty(), NodeType.tenant, - new Reports(), Optional.empty(), Optional.empty()); - Node parent = new Node(hostname + "parent", new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.empty(), - new Flavor(totalHostResources), - Status.initial(), Node.State.ready, Optional.empty(), History.empty(), NodeType.host, - new Reports(), Optional.empty(), Optional.empty()); - return new PrioritizableNode(node, totalHostResources.subtract(allocatedHostResources), Optional.of(parent), - false, false, true, false); - } - -} -- cgit v1.2.3 From cca8ef0d7b033475d2e32b211f136dbdeb8fee11 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Wed, 16 Sep 2020 12:36:39 +0200 Subject: Rename method --- .../provision/autoscale/AutoscalingTester.java | 5 +- .../provision/autoscale/NodeMetricsDbTest.java | 10 ++-- .../autoscale/NodeMetricsFetcherTest.java | 3 +- .../maintenance/AutoscalingMaintainerTest.java | 2 +- .../DynamicProvisioningMaintainerTest.java | 2 +- .../provision/maintenance/RebalancerTest.java | 10 ++-- .../ScalingSuggestionsMaintainerTest.java | 2 +- .../provisioning/AclProvisioningTest.java | 2 +- .../provision/provisioning/DockerImagesTest.java | 2 +- ...ckerProvisioningCompleteHostCalculatorTest.java | 2 +- .../provisioning/DockerProvisioningTest.java | 13 +++-- .../provisioning/DynamicDockerAllocationTest.java | 26 +++++----- .../provisioning/DynamicDockerProvisionTest.java | 14 +++--- .../provisioning/MultigroupProvisioningTest.java | 6 +-- .../provision/provisioning/ProvisioningTest.java | 58 +++++++++++----------- .../provision/provisioning/ProvisioningTester.java | 4 +- 16 files changed, 79 insertions(+), 82 deletions(-) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index 0a127eacae1..19911076e69 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -32,7 +32,6 @@ import java.util.List; import java.util.Optional; import java.util.Set; -import static com.yahoo.config.provision.NodeResources.StorageType.local; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -51,7 +50,7 @@ class AutoscalingTester { public AutoscalingTester(NodeResources hostResources, HostResourcesCalculator resourcesCalculator) { this(new Zone(Environment.prod, RegionName.from("us-east")), List.of(new Flavor("hostFlavor", hostResources)), resourcesCalculator); provisioningTester.makeReadyNodes(20, "hostFlavor", NodeType.host, 8); - provisioningTester.deployZoneApp(); + provisioningTester.activateTenantHosts(); } public AutoscalingTester(Zone zone, List flavors) { @@ -87,7 +86,7 @@ class AutoscalingTester { List hosts = provisioningTester.prepare(application, cluster, Capacity.from(new ClusterResources(nodes, groups, resources))); for (HostSpec host : hosts) makeReady(host.hostname()); - provisioningTester.deployZoneApp(); + provisioningTester.activateTenantHosts(); provisioningTester.activate(application, hosts); return hosts; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java index 976aeb2346a..eaad4526591 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java @@ -6,10 +6,7 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; import com.yahoo.test.ManualClock; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.NodeRepositoryTester; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import org.junit.Test; @@ -19,14 +16,17 @@ import java.util.List; import static org.junit.Assert.assertEquals; +/** + * @author bratseth + */ public class NodeMetricsDbTest { @Test public void testNodeMetricsDb() { ProvisioningTester tester = new ProvisioningTester.Builder().build(); - tester.makeReadyHosts(10, new NodeResources(10, 100, 1000, 10)).deployZoneApp(); + tester.makeReadyHosts(10, new NodeResources(10, 100, 1000, 10)) + .activateTenantHosts(); ApplicationId app1 = tester.makeApplicationId("app1"); - tester.deployZoneApp(); var hosts = tester.activate(app1, ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("7.0").build(), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java index d418d818ef3..fea3e8da70a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.NodeResources; -import com.yahoo.vdslib.state.NodeState; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock; @@ -29,7 +28,7 @@ public class NodeMetricsFetcherTest { NodeMetricsFetcher fetcher = new NodeMetricsFetcher(tester.nodeRepository(), orchestrator, httpClient); tester.makeReadyNodes(4, resources); // Creates (in order) host-1.yahoo.com, host-2.yahoo.com, host-3.yahoo.com, host-4.yahoo.com - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId application1 = tester.makeApplicationId(); ApplicationId application2 = tester.makeApplicationId(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java index 88195cf0ed9..833daebc37a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java @@ -65,7 +65,7 @@ public class AutoscalingMaintainerTest { assertTrue(deployer.lastDeployTime(app2).isEmpty()); tester.makeReadyNodes(20, "flt", NodeType.host, 8); - tester.deployZoneApp(); + tester.activateTenantHosts(); tester.deploy(app1, cluster1, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index 45da1f1d3ee..12cf114b2d2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -179,7 +179,7 @@ public class DynamicProvisioningMaintainerTest { tester.maintainer.maintain(); // Resume provisioning of new hosts List provisioned = tester.nodeRepository.list().state(Node.State.provisioned).asList(); tester.nodeRepository.setReady(provisioned, Agent.system, this.getClass().getSimpleName()); - tester.provisioningTester.deployZoneApp(); + tester.provisioningTester.activateTenantHosts(); // Allocating nodes to a host does not result in provisioning of additional capacity ApplicationId application = tester.provisioningTester.makeApplicationId(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java index 05e5b4829e9..65b79c2df04 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java @@ -54,7 +54,7 @@ public class RebalancerTest { // --- Making a more suitable node configuration available causes rebalancing Node newCpuHost = tester.makeReadyNode("cpu"); - tester.deployZoneApp(); + tester.activateTenantHosts(); tester.maintain(); assertTrue("Rebalancer retired the node we wanted to move away from", tester.isNodeRetired(cpuSkewedNode)); @@ -74,7 +74,7 @@ public class RebalancerTest { // --- Adding a more suitable node reconfiguration causes no action as the system is not stable Node memSkewedNode = tester.getNode(memoryApp); Node newMemHost = tester.makeReadyNode("mem"); - tester.deployZoneApp(); + tester.activateTenantHosts(); tester.maintain(); assertFalse("No rebalancing happens because cpuSkewedNode is still retired", tester.isNodeRetired(memSkewedNode)); @@ -117,7 +117,7 @@ public class RebalancerTest { // --- Making a more suitable node configuration available causes rebalancing Node newCpuHost = tester.makeReadyNode("cpu"); - tester.deployZoneApp(); + tester.activateTenantHosts(); tester.deployApp(cpuApp, false /* skip advancing clock after deployment */); tester.maintain(); @@ -152,7 +152,7 @@ public class RebalancerTest { deployer = new MockDeployer(tester.provisioner(), tester.clock(), apps); rebalancer = new Rebalancer(deployer, tester.nodeRepository(), metric, tester.clock(), Duration.ofMinutes(1)); tester.makeReadyNodes(3, "flat", NodeType.host, 8); - tester.deployZoneApp(); + tester.activateTenantHosts(); } void maintain() { rebalancer.maintain(); } @@ -163,7 +163,7 @@ public class RebalancerTest { NodeRepository nodeRepository() { return tester.nodeRepository(); } - void deployZoneApp() { tester.deployZoneApp(); } + void activateTenantHosts() { tester.activateTenantHosts(); } void deployApp(ApplicationId id) { deployApp(id, true); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java index 31a9fcb8999..bd2afb5d1c8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -47,7 +47,7 @@ public class ScalingSuggestionsMaintainerTest { NodeMetricsDb nodeMetricsDb = new NodeMetricsDb(tester.nodeRepository()); tester.makeReadyNodes(20, "flt", NodeType.host, 8); - tester.deployZoneApp(); + tester.activateTenantHosts(); tester.deploy(app1, cluster1, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java index d165f865432..505e53c9195 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java @@ -165,7 +165,7 @@ public class AclProvisioningTest { public void trusted_nodes_for_application_with_load_balancer() { // Provision hosts and containers var hosts = tester.makeReadyNodes(2, "default", NodeType.host); - tester.deployZoneApp(); + tester.activateTenantHosts(); for (var host : hosts) { tester.makeReadyVirtualDockerNodes(2, new NodeResources(2, 8, 50, 1), host.hostname()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java index cd6ae587b04..d5437296620 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java @@ -27,7 +27,7 @@ public class DockerImagesTest { // Host uses tenant default image (for preload purposes) var defaultImage = DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"); var hosts = tester.makeReadyNodes(2, "default", NodeType.host); - tester.deployZoneApp(); + tester.activateTenantHosts(); for (var host : hosts) { assertEquals(defaultImage, tester.nodeRepository().dockerImages().dockerImageFor(host.type())); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningCompleteHostCalculatorTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningCompleteHostCalculatorTest.java index 24cdc5c8fd0..2588818a9d3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningCompleteHostCalculatorTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningCompleteHostCalculatorTest.java @@ -30,7 +30,7 @@ public class DockerProvisioningCompleteHostCalculatorTest { .resourcesCalculator(new CompleteResourcesCalculator(hostFlavor)) .flavors(List.of(hostFlavor)) .build(); - tester.makeReadyHosts(9, hostFlavor.resources()).deployZoneApp(); + tester.makeReadyHosts(9, hostFlavor.resources()).activateTenantHosts(); ApplicationId app1 = tester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java index e566172b524..6ae78f9019c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java @@ -20,7 +20,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; -import com.yahoo.vespa.hosted.provision.NodeRepository; import org.junit.Test; import java.util.HashSet; @@ -123,7 +122,7 @@ public class DockerProvisioningTest { tester.makeReadyNodes(10, resources, Optional.of(tenant1), NodeType.host, 1); tester.makeReadyNodes(10, resources, Optional.empty(), NodeType.host, 1); - tester.deployZoneApp(); + tester.activateTenantHosts(); Version wantedVespaVersion = Version.fromString("6.39"); List nodes = tester.prepare(application2_1, @@ -310,7 +309,7 @@ public class DockerProvisioningTest { .resourcesCalculator(3, 0) .flavors(List.of(hostFlavor)) .build(); - tester.makeReadyHosts(2, hostFlavor.resources()).deployZoneApp(); + tester.makeReadyHosts(2, hostFlavor.resources()).activateTenantHosts(); ApplicationId app1 = tester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -330,7 +329,7 @@ public class DockerProvisioningTest { .resourcesCalculator(3, 0) .flavors(List.of(hostFlavor)) .build(); - tester.makeReadyHosts(9, hostFlavor.resources()).deployZoneApp(); + tester.makeReadyHosts(9, hostFlavor.resources()).activateTenantHosts(); ApplicationId app1 = tester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -365,7 +364,7 @@ public class DockerProvisioningTest { .resourcesCalculator(3, 0) .flavors(List.of(hostFlavor)) .build(); - tester.makeReadyHosts(2, hostFlavor.resources()).deployZoneApp(); + tester.makeReadyHosts(2, hostFlavor.resources()).activateTenantHosts(); ApplicationId app1 = tester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -387,7 +386,7 @@ public class DockerProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) .flavors(List.of(new Flavor(r))) .build(); - tester.makeReadyHosts(5, r).deployZoneApp(); + tester.makeReadyHosts(5, r).activateTenantHosts(); ApplicationId app1 = tester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.container, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -420,7 +419,7 @@ public class DockerProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) .flavors(List.of(new Flavor(r))) .build(); - tester.makeReadyHosts(4, r).deployZoneApp(); + tester.makeReadyHosts(4, r).activateTenantHosts(); ApplicationId app1 = tester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(clusterType, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java index 29e371dd937..ab62972df30 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java @@ -66,7 +66,7 @@ public class DynamicDockerAllocationTest { .spareCount(spareCount) .build(); tester.makeReadyNodes(4, "host-small", NodeType.host, 32); - tester.deployZoneApp(); + tester.activateTenantHosts(); List dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); NodeResources flavor = new NodeResources(1, 4, 100, 1); @@ -109,7 +109,7 @@ public class DynamicDockerAllocationTest { public void relocate_failed_nodes() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(5, "host-small", NodeType.host, 32); - tester.deployZoneApp(); + tester.activateTenantHosts(); List dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); NodeResources resources = new NodeResources(1, 4, 100, 0.3); @@ -158,7 +158,7 @@ public class DynamicDockerAllocationTest { tester.makeReadyNodes(3, "flt", NodeType.host, 8); // cpu: 30, mem: 30 tester.makeReadyNodes(3, "cpu", NodeType.host, 8); // cpu: 40, mem: 20 tester.makeReadyNodes(3, "mem", NodeType.host, 8); // cpu: 20, mem: 40 - tester.deployZoneApp(); + tester.activateTenantHosts(); NodeResources fltResources = new NodeResources(6, 6, 10, 0.1); NodeResources cpuResources = new NodeResources(8, 4, 10, 0.1); NodeResources memResources = new NodeResources(4, 8, 10, 0.1); @@ -201,7 +201,7 @@ public class DynamicDockerAllocationTest { public void do_not_relocate_nodes_from_spare_if_no_where_to_relocate_them() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, "host-small", NodeType.host, 32); - tester.deployZoneApp(); + tester.activateTenantHosts(); List dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); NodeResources flavor = new NodeResources(1, 4, 100, 1); @@ -228,7 +228,7 @@ public class DynamicDockerAllocationTest { public void multiple_groups_are_on_separate_parent_hosts() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(5, "host-small", NodeType.host, 32); - tester.deployZoneApp(); + tester.activateTenantHosts(); //Deploy an application having 6 nodes (3 nodes in 2 groups). We only have 5 docker hosts available ApplicationId application1 = tester.makeApplicationId(); @@ -249,7 +249,7 @@ public class DynamicDockerAllocationTest { // Setup test ApplicationId application1 = tester.makeApplicationId(); tester.makeReadyNodes(5, "host-small", NodeType.host, 32); - tester.deployZoneApp(); + tester.activateTenantHosts(); NodeResources flavor = new NodeResources(1, 4, 100, 1); // Deploy initial state (can max deploy 3 nodes due to redundancy requirements) @@ -278,7 +278,7 @@ public class DynamicDockerAllocationTest { public void non_prod_zones_do_not_have_spares() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.perf, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(3, "host-small", NodeType.host, 32); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId application1 = tester.makeApplicationId(); List hosts = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 3, 1, new NodeResources(1, 4, 100, 1)); tester.activate(application1, ImmutableSet.copyOf(hosts)); @@ -291,7 +291,7 @@ public class DynamicDockerAllocationTest { public void cd_uses_slow_disk_nodes_for_docker_hosts() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(SystemName.cd, Environment.test, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(4, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId application1 = tester.makeApplicationId(); List hosts = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 3, 1, new NodeResources(1, 4, 100, 1)); tester.activate(application1, ImmutableSet.copyOf(hosts)); @@ -311,7 +311,7 @@ public class DynamicDockerAllocationTest { public void provision_dual_stack_containers() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, "host-large", NodeType.host, 10, true); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId application = tester.makeApplicationId(); List hosts = tester.prepare(application, clusterSpec("myContent.t1.a1"), 2, 1, new NodeResources(1, 4, 100, 1)); @@ -342,7 +342,7 @@ public class DynamicDockerAllocationTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.fast)), NodeType.host, 10, true); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId application = tester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("1").build(); @@ -359,7 +359,7 @@ public class DynamicDockerAllocationTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.fast)), NodeType.host, 10, true); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId application = tester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("1").build(); @@ -381,7 +381,7 @@ public class DynamicDockerAllocationTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.fast)), NodeType.host, 10, true); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId application = tester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("1").build(); @@ -399,7 +399,7 @@ public class DynamicDockerAllocationTest { public void testSwitchingFromLegacyFlavorSyntaxToResourcesDoesNotCauseReallocation() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, new Flavor(new NodeResources(5, 20, 1400, 3)), NodeType.host, 10, true); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId application = tester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("1").build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java index db6d75d724e..411283abf33 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java @@ -81,7 +81,7 @@ public class DynamicDockerProvisionTest { @Test public void does_not_allocate_to_available_empty_hosts() { tester.makeReadyNodes(3, "small", NodeType.host, 10); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId application = tester.makeApplicationId(); NodeResources flavor = new NodeResources(1, 4, 10, 1); @@ -109,7 +109,7 @@ public class DynamicDockerProvisionTest { tester.nodeRepository().setReady(List.of(host), Agent.system, getClass().getSimpleName()); nameResolver.addRecord(hostname + "-2", "::" + i + ":2"); } - tester.deployZoneApp(); + tester.activateTenantHosts(); mockHostProvisioner(hostProvisioner, tester.nodeRepository().flavors().getFlavorOrThrow("small")); tester.prepare(application, clusterSpec("another-id"), 2, 1, flavor); @@ -136,7 +136,7 @@ public class DynamicDockerProvisionTest { // Allocate 10 hosts tester.makeReadyNodes(10, resources, NodeType.host, 1); - tester.deployZoneApp(); + tester.activateTenantHosts(); // Prepare & activate an application with 8 nodes and 2 groups tester.activate(app, tester.prepare(app, clusterSpec("content"), 8, 2, resources)); @@ -170,7 +170,7 @@ public class DynamicDockerProvisionTest { .resourcesCalculator(memoryTax, 0) .build(); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId app1 = tester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -216,7 +216,7 @@ public class DynamicDockerProvisionTest { .resourcesCalculator(memoryTax, 0) .build(); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId app1 = tester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -291,7 +291,7 @@ public class DynamicDockerProvisionTest { .resourcesCalculator(memoryTax, 0) .build(); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId app1 = tester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -326,7 +326,7 @@ public class DynamicDockerProvisionTest { .resourcesCalculator(memoryTax, localDiskTax) .build(); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId app1 = tester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java index 6e609d13d3b..09d1600e1d7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java @@ -134,7 +134,7 @@ public class MultigroupProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) .flavors(List.of(hostFlavor)) .build(); - tester.makeReadyHosts(6, hostFlavor.resources()).deployZoneApp(); + tester.makeReadyHosts(6, hostFlavor.resources()).activateTenantHosts(); ApplicationId app1 = tester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -160,7 +160,7 @@ public class MultigroupProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) .flavors(List.of(hostFlavor)) .build(); - tester.makeReadyHosts(6, hostFlavor.resources()).deployZoneApp(); + tester.makeReadyHosts(6, hostFlavor.resources()).activateTenantHosts(); ApplicationId app1 = tester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -190,7 +190,7 @@ public class MultigroupProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) .flavors(List.of(hostFlavor)) .build(); - tester.makeReadyHosts(12, hostFlavor.resources()).deployZoneApp(); + tester.makeReadyHosts(12, hostFlavor.resources()).activateTenantHosts(); ApplicationId app1 = tester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index ff2f0ffca96..d53a0732c89 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -62,7 +62,7 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); ApplicationId application2 = tester.makeApplicationId(); - tester.makeReadyHosts(21, defaultResources).deployZoneApp(); + tester.makeReadyHosts(21, defaultResources).activateTenantHosts(); // deploy SystemState state1 = prepare(application1, 2, 2, 3, 3, defaultResources, tester); @@ -196,7 +196,7 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); tester.makeReadyHosts(24, defaultResources); - tester.deployZoneApp(); + tester.activateTenantHosts(); // deploy SystemState state1 = prepare(application1, 2, 2, 3, 3, defaultResources, tester); @@ -267,7 +267,7 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); tester.makeReadyHosts(12, small); - tester.deployZoneApp(); + tester.activateTenantHosts(); // deploy SystemState state1 = prepare(application1, 2, 2, 4, 4, small, tester); @@ -278,7 +278,7 @@ public class ProvisioningTest { tester.activate(application1, state2.allHosts); tester.makeReadyHosts(16, large); - tester.deployZoneApp(); + tester.activateTenantHosts(); // redeploy with increased sizes and new flavor SystemState state3 = prepare(application1, 3, 4, 4, 5, large, tester); @@ -303,7 +303,7 @@ public class ProvisioningTest { tester.makeReadyHosts(12, small); tester.makeReadyHosts(12, large); - tester.deployZoneApp(); + tester.activateTenantHosts(); // deploy SystemState state1 = prepare(application1, 2, 2, 4, 4, small, tester); @@ -319,7 +319,7 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); - tester.makeReadyHosts(5, defaultResources).deployZoneApp(); + tester.makeReadyHosts(5, defaultResources).activateTenantHosts(); // deploy SystemState state1 = prepare(application1, 2, 0, 3, 0, defaultResources, tester); @@ -354,7 +354,7 @@ public class ProvisioningTest { @Test public void requested_resources_info_is_retained() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyHosts(13, defaultResources).deployZoneApp(); + tester.makeReadyHosts(13, defaultResources).activateTenantHosts(); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); ApplicationId application = tester.makeApplicationId(); @@ -398,7 +398,7 @@ public class ProvisioningTest { public void deploy_specific_vespa_version() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - tester.makeReadyHosts(4, defaultResources).deployZoneApp(); + tester.makeReadyHosts(4, defaultResources).activateTenantHosts(); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); ApplicationId application = tester.makeApplicationId(); @@ -411,7 +411,7 @@ public class ProvisioningTest { public void deploy_specific_vespa_version_and_docker_image() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - tester.makeReadyHosts(4, defaultResources).deployZoneApp(); + tester.makeReadyHosts(4, defaultResources).activateTenantHosts(); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); ApplicationId application = tester.makeApplicationId(); @@ -426,7 +426,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.test, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyHosts(4, defaultResources).deployZoneApp(); + tester.makeReadyHosts(4, defaultResources).activateTenantHosts(); SystemState state = prepare(application, 2, 2, 3, 3, defaultResources, tester); assertEquals(4, state.allHosts.size()); tester.activate(application, state.allHosts); @@ -438,7 +438,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) .flavors(List.of(hostFlavor)) .build(); - tester.makeReadyHosts(4, hostFlavor.resources()).deployZoneApp(); + tester.makeReadyHosts(4, hostFlavor.resources()).activateTenantHosts(); ApplicationId app1 = tester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -456,7 +456,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) .flavors(List.of(hostFlavor)) .build(); - tester.makeReadyHosts(31, hostFlavor.resources()).deployZoneApp(); + tester.makeReadyHosts(31, hostFlavor.resources()).activateTenantHosts(); ApplicationId app1 = tester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -516,7 +516,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyHosts(10, defaultResources).deployZoneApp(); + tester.makeReadyHosts(10, defaultResources).activateTenantHosts(); prepare(application, 1, 2, 3, 3, defaultResources, tester); } @@ -525,7 +525,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyHosts(10, defaultResources).deployZoneApp(); + tester.makeReadyHosts(10, defaultResources).activateTenantHosts(); try { prepare(application, 2, 2, 3, 3, new NodeResources(2, 2, 10, 2), tester); @@ -540,7 +540,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyHosts(10, defaultResources).deployZoneApp(); + tester.makeReadyHosts(10, defaultResources).activateTenantHosts(); try { prepare(application, 2, 2, 3, 3, new NodeResources(0.4, 4, 10, 2), tester); @@ -573,7 +573,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.test, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyHosts(4, large).deployZoneApp(); + tester.makeReadyHosts(4, large).activateTenantHosts(); SystemState state = prepare(application, 2, 2, 3, 3, large, tester); assertEquals(4, state.allHosts.size()); tester.activate(application, state.allHosts); @@ -584,7 +584,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.staging, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyHosts(14, defaultResources).deployZoneApp(); + tester.makeReadyHosts(14, defaultResources).activateTenantHosts(); SystemState state = prepare(application, 1, 1, 1, 64, defaultResources, tester); // becomes 1, 1, 1, 1, 6 assertEquals(9, state.allHosts.size()); tester.activate(application, state.allHosts); @@ -594,7 +594,7 @@ public class ProvisioningTest { public void activate_after_reservation_timeout() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyHosts(10, defaultResources).deployZoneApp(); + tester.makeReadyHosts(10, defaultResources).activateTenantHosts(); ApplicationId application = tester.makeApplicationId(); SystemState state = prepare(application, 2, 2, 3, 3, defaultResources, tester); @@ -616,7 +616,7 @@ public class ProvisioningTest { public void out_of_capacity() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyHosts(9, defaultResources).deployZoneApp(); // need 2+2+3+3=10 + tester.makeReadyHosts(9, defaultResources).activateTenantHosts(); // need 2+2+3+3=10 ApplicationId application = tester.makeApplicationId(); try { prepare(application, 2, 2, 3, 3, defaultResources, tester); @@ -633,7 +633,7 @@ public class ProvisioningTest { Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyHosts(13, defaultResources).deployZoneApp(); + tester.makeReadyHosts(13, defaultResources).activateTenantHosts(); ApplicationId application = tester.makeApplicationId(); try { prepare(application, 2, 2, 6, 3, defaultResources, tester); @@ -651,7 +651,7 @@ public class ProvisioningTest { Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyHosts(13, defaultResources).deployZoneApp(); + tester.makeReadyHosts(13, defaultResources).activateTenantHosts(); ApplicationId application = tester.makeApplicationId(); prepare(application, 2, 2, 6, 3, defaultResources, tester); } @@ -659,7 +659,7 @@ public class ProvisioningTest { @Test public void out_of_capacity_but_cannot_fail() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyHosts(4, defaultResources).deployZoneApp(); + tester.makeReadyHosts(4, defaultResources).activateTenantHosts(); ApplicationId application = tester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("4.5.6").build(); tester.prepare(application, cluster, Capacity.from(new ClusterResources(5, 1, NodeResources.unspecified()), false, false)); @@ -693,7 +693,7 @@ public class ProvisioningTest { ApplicationId application = tester.makeApplicationId(); // Create 10 nodes - tester.makeReadyHosts(10, defaultResources).deployZoneApp(); + tester.makeReadyHosts(10, defaultResources).activateTenantHosts(); // Allocate 5 nodes ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("4.5.6").build(); tester.activate(application, tester.prepare(application, cluster, capacity)); @@ -724,7 +724,7 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); - tester.makeReadyHosts(14, defaultResources).deployZoneApp(); + tester.makeReadyHosts(14, defaultResources).activateTenantHosts(); // deploy SystemState state1 = prepare(application1, 3, 3, 4, 4, defaultResources, tester); @@ -751,7 +751,7 @@ public class ProvisioningTest { public void node_on_spare_host_retired_first() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) .spareCount(1).build(); - tester.makeReadyHosts(7, defaultResources).deployZoneApp(); + tester.makeReadyHosts(7, defaultResources).activateTenantHosts(); ApplicationId application = tester.makeApplicationId(); ClusterSpec spec = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content1")).vespaVersion("7.1.2").build(); @@ -774,7 +774,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyHosts(10, defaultResources).deployZoneApp(); + tester.makeReadyHosts(10, defaultResources).activateTenantHosts(); // Deploy application { @@ -802,7 +802,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyHosts(2, defaultResources).deployZoneApp(); + tester.makeReadyHosts(2, defaultResources).activateTenantHosts(); // Deploy fails with out of capacity try { @@ -813,7 +813,7 @@ public class ProvisioningTest { tester.getNodes(application, Node.State.reserved).size()); // Enough nodes become available - tester.makeReadyHosts(2, defaultResources).deployZoneApp(); + tester.makeReadyHosts(2, defaultResources).activateTenantHosts(); // Deploy is retried after a few minutes tester.clock().advance(Duration.ofMinutes(2)); @@ -883,7 +883,7 @@ public class ProvisioningTest { var tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); var application = tester.makeApplicationId(); - tester.makeReadyHosts(4, defaultResources).deployZoneApp(); + tester.makeReadyHosts(4, defaultResources).activateTenantHosts(); // Application allocates two content nodes initially, with cluster type content ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("1.2.3").build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index d56cae799b2..0a3c85d3702 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -320,7 +320,7 @@ public class ProvisioningTester { return makeReadyNodes(n, flavor, NodeType.tenant); } - /** Call deployZoneApp() after this before deploying applications */ + /** Call {@link this#activateTenantHosts()} after this before deploying applications */ public ProvisioningTester makeReadyHosts(int n, NodeResources resources) { makeReadyNodes(n, resources, NodeType.host, 5); return this; @@ -494,7 +494,7 @@ public class ProvisioningTester { return nodes; } - public void deployZoneApp() { + public void activateTenantHosts() { ApplicationId applicationId = makeApplicationId(); List list = prepare(applicationId, ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin")).vespaVersion("6.42").build(), -- cgit v1.2.3 From fa9ed314a0aa788f6ecac1b7c75d397406bbfba6 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Fri, 18 Sep 2020 14:13:29 +0000 Subject: PackedLabels::label_value -> get_label * and inline its functions --- eval/src/vespa/eval/tensor/mixed/packed_labels.cpp | 25 +++++++--------------- eval/src/vespa/eval/tensor/mixed/packed_labels.h | 4 +--- .../vespa/eval/tensor/mixed/packed_mappings.cpp | 2 +- .../eval/tensor/mixed/packed_mixed_tensor.cpp | 8 +------ 4 files changed, 11 insertions(+), 28 deletions(-) diff --git a/eval/src/vespa/eval/tensor/mixed/packed_labels.cpp b/eval/src/vespa/eval/tensor/mixed/packed_labels.cpp index 2153a6a0ca9..a05497f6b68 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_labels.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_labels.cpp @@ -12,26 +12,28 @@ PackedLabels::find_label(vespalib::stringref to_find) const uint32_t hi = num_labels(); while (lo < hi) { uint32_t mid = (lo + hi) / 2; - if (label_value(mid) < to_find) { + if (get_label(mid) < to_find) { lo = mid + 1; } else { hi = mid; } } assert(lo == hi); - if (lo < num_labels() && label_value(lo) == to_find) { + if (lo < num_labels() && get_label(lo) == to_find) { return lo; } return -1; } vespalib::stringref -PackedLabels::label_value(uint32_t index) const +PackedLabels::get_label(uint32_t index) const { assert(index < num_labels()); - auto p = get_label_start(index); - auto sz = get_label_size(index); + uint32_t this_offset = _offsets[index]; + uint32_t next_offset = _offsets[index+1]; + auto p = &_label_store[this_offset]; + size_t sz = next_offset - this_offset - 1; return vespalib::stringref(p, sz); } @@ -47,19 +49,8 @@ PackedLabels::validate_labels(uint32_t num_labels) } assert(_label_store.size() == _offsets[num_labels]); for (uint32_t i = 0; i+1 < num_labels; ++i) { - assert(label_value(i) < label_value(i+1)); + assert(get_label(i) < get_label(i+1)); } } -const char * -PackedLabels::get_label_start(uint32_t index) const { - uint32_t offset = _offsets[index]; - return &_label_store[offset]; -} - -uint32_t -PackedLabels::get_label_size(uint32_t index) const { - return _offsets[index+1] - _offsets[index] - 1; -} - } // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_labels.h b/eval/src/vespa/eval/tensor/mixed/packed_labels.h index 0994ef07a63..dac338448fb 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_labels.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_labels.h @@ -30,15 +30,13 @@ public: // returns -1 if the given label value cannot be found int32_t find_label(vespalib::stringref value) const; - vespalib::stringref label_value(uint32_t index) const; + vespalib::stringref get_label(uint32_t index) const; private: const ConstArrayRef _offsets; const ConstArrayRef _label_store; void validate_labels(uint32_t num_labels); - const char *get_label_start(uint32_t index) const; - uint32_t get_label_size(uint32_t index) const; }; } // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp index 9a50f9ae431..226ac01ef71 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp @@ -94,7 +94,7 @@ PackedMappings::fill_address_by_sortid(uint32_t internal_index, Address &address address.resize(_num_dims); for (uint32_t i = 0; i < _num_dims; ++i) { uint32_t label_idx = _int_store[offset++]; - address[i] = _label_store.label_value(label_idx); + address[i] = _label_store.get_label(label_idx); } return _int_store[offset]; } diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp index a9bbdc39889..2587878f16f 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp @@ -41,7 +41,6 @@ PackedMixedTensorIndexView::lookup(const std::vector _index = 0; assert(addr.size() == num_view_dims()); _lookup_enums.clear(); - // printf("lookup %zu/%zu dims:", num_view_dims(), num_full_dims()); for (const vespalib::stringref * label_ptr : addr) { int32_t label_enum = _mappings.label_store().find_label(*label_ptr); if (label_enum < 0) { @@ -50,9 +49,7 @@ PackedMixedTensorIndexView::lookup(const std::vector break; } _lookup_enums.push_back(label_enum); - // printf(" '%s'", label_ptr->data()); } - // printf(" [in %u mappings]\n", _mappings.size()); } bool @@ -81,18 +78,15 @@ PackedMixedTensorIndexView::next_result(const std::vector } // not a view dimension: uint32_t label_enum = _full_enums[i]; - auto label_value = _mappings.label_store().label_value(label_enum); - *addr_out[ao_idx] = label_value; + *addr_out[ao_idx] = _mappings.label_store().get_label(label_enum); ++ao_idx; } if (couldmatch) { - // printf("matches at %zu/%u\n", _index, _mappings.size()); assert(vd_idx == num_view_dims()); assert(ao_idx == num_rest_dims()); return true; } } - // printf("no more matches %zu/%u\n", _index, _mappings.size()); return false; } -- cgit v1.2.3 From 952a161e168ebe640eecf7a5a126c6bfb4acd161 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Mon, 21 Sep 2020 10:08:15 +0200 Subject: Create remote session in addLocalSession() only when it does not exist --- .../config/server/session/SessionRepository.java | 22 +++++++++++--------- .../config/server/session/SessionStateWatcher.java | 24 +++++++++++----------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 4f577d8f62c..c198f1dd0fc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -123,10 +123,11 @@ public class SessionRepository { // ---------------- Local sessions ---------------------------------------------------------------- public synchronized void addLocalSession(LocalSession session) { - localSessionCache.put(session.getSessionId(), session); long sessionId = session.getSessionId(); - RemoteSession remoteSession = createRemoteSession(sessionId); - addSessionStateWatcher(sessionId, remoteSession); + localSessionCache.put(sessionId, session); + if (remoteSessionCache.get(sessionId) == null) { + createRemoteSession(sessionId); + } } public LocalSession getLocalSession(long sessionId) { @@ -347,13 +348,10 @@ public class SessionRepository { SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); if (sessionZKClient.readStatus().equals(Session.Status.DELETE)) return; - log.log(Level.FINE, () -> "Adding remote session to SessionRepository: " + sessionId); - RemoteSession remoteSession = createRemoteSession(sessionId); - loadSessionIfActive(remoteSession); - addRemoteSession(remoteSession); + log.log(Level.FINE, () -> "Adding remote session " + sessionId); + createRemoteSession(sessionId); if (distributeApplicationPackage()) createLocalSessionUsingDistributedApplicationPackage(sessionId); - addSessionStateWatcher(sessionId, remoteSession); } void activate(RemoteSession session) { @@ -463,9 +461,13 @@ public class SessionRepository { return create(applicationDirectory, applicationId, activeSessionId, false, timeoutBudget); } - public RemoteSession createRemoteSession(long sessionId) { + public synchronized RemoteSession createRemoteSession(long sessionId) { SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); - return new RemoteSession(tenantName, sessionId, componentRegistry, sessionZKClient); + RemoteSession session = new RemoteSession(tenantName, sessionId, componentRegistry, sessionZKClient); + remoteSessionCache.put(sessionId, session); + loadSessionIfActive(session); + addSessionStateWatcher(sessionId, session); + return session; } private void ensureSessionPathDoesNotExist(long sessionId) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java index c6c08beea17..d6d08aaac6c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java @@ -24,18 +24,18 @@ public class SessionStateWatcher { private static final Logger log = Logger.getLogger(SessionStateWatcher.class.getName()); private final Curator.FileCache fileCache; - private final RemoteSession remoteSession; + private final RemoteSession session; private final MetricUpdater metrics; private final Executor zkWatcherExecutor; private final SessionRepository sessionRepository; SessionStateWatcher(Curator.FileCache fileCache, - RemoteSession remoteSession, + RemoteSession session, MetricUpdater metrics, Executor zkWatcherExecutor, SessionRepository sessionRepository) { this.fileCache = fileCache; - this.remoteSession = remoteSession; + this.session = session; this.metrics = metrics; this.fileCache.addListener(this::nodeChanged); this.fileCache.start(); @@ -44,24 +44,24 @@ public class SessionStateWatcher { } private void sessionStatusChanged(Status newStatus) { - long sessionId = remoteSession.getSessionId(); + long sessionId = session.getSessionId(); switch (newStatus) { case NEW: case NONE: break; case PREPARE: createLocalSession(sessionId); - sessionRepository.prepare(remoteSession); + sessionRepository.prepare(session); break; case ACTIVATE: createLocalSession(sessionId); - sessionRepository.activate(remoteSession); + sessionRepository.activate(session); break; case DEACTIVATE: - sessionRepository.deactivate(remoteSession); + sessionRepository.deactivate(session); break; case DELETE: - sessionRepository.delete(remoteSession); + sessionRepository.delete(session); break; default: throw new IllegalStateException("Unknown status " + newStatus); @@ -75,7 +75,7 @@ public class SessionStateWatcher { } public long getSessionId() { - return remoteSession.getSessionId(); + return session.getSessionId(); } public void close() { @@ -93,12 +93,12 @@ public class SessionStateWatcher { ChildData node = fileCache.getCurrentData(); if (node != null) { newStatus = Status.parse(Utf8.toString(node.getData())); - log.log(Level.FINE, remoteSession.logPre() + "Session change: Session " - + remoteSession.getSessionId() + " changed status to " + newStatus.name()); + log.log(Level.FINE, session.logPre() + "Session change: Session " + + session.getSessionId() + " changed status to " + newStatus.name()); sessionStatusChanged(newStatus); } } catch (Exception e) { - log.log(Level.WARNING, remoteSession.logPre() + "Error handling session change to " + + log.log(Level.WARNING, session.logPre() + "Error handling session change to " + newStatus.name() + " for session " + getSessionId(), e); metrics.incSessionChangeErrors(); } -- cgit v1.2.3 From 4e275f0c7d3ecebbba80ff831635815af4fcea69 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Mon, 21 Sep 2020 11:47:42 +0000 Subject: more and updated comments --- eval/src/vespa/eval/tensor/mixed/packed_mappings.h | 28 ++++++++++++---------- .../eval/tensor/mixed/packed_mappings_builder.h | 22 +++++++++++++---- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings.h b/eval/src/vespa/eval/tensor/mixed/packed_mappings.h index f43f2ef6832..ce9ff066271 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings.h @@ -9,20 +9,21 @@ namespace vespalib::eval::packed_mixed_tensor { /** * Mappings for sparse tensor dimensions. + * * Each address (conceptually "array of string") - * has two indexes: The subspace_index (the - * order that addresses were added to a builder), - * and the internal_index AKA "sortid", which - * indexes into a lexicographically sorted array - * of addresses. - * (Note: we may want to change this so subspaces - * are always sorted by address, making these two - * indexes equivalent). + * maps to a "subspace" (currently in the + * order that addresses were added to a builder). * - * Has various methods for mapping back and forth - * between addresses and indexes, and also allows - * using the internal label enumerations instead - * of working with strings all the time. + * Internally addresses are lexicographically + * sorted, and you can iterate over them in sort + * order with the fill_*() methods. + * + * (Note: we may want to change this so subspaces + * are always sorted by address, so the "subspace" + * index and the "sortid" index become equivalent). + * + * Allows using the internal label enumerations + * instead of working with strings all the time. * * NOTE: Making a copy of PackedMappings will not copy * the underlying data, these must then stay alive @@ -40,10 +41,11 @@ public: int32_t subspace_of_enums(const InternalAddress &address) const; int32_t subspace_of_address(const Address &address) const; - /** returns subspace_index */ + /** returns "subspace" index */ uint32_t fill_address_by_sortid(uint32_t sortid, Address &address) const; uint32_t fill_enums_by_sortid(uint32_t sortid, InternalAddress &address) const; + // mapping from label enum to stringref (and vice versa) const PackedLabels & label_store() const { return _label_store; } private: PackedMappings(uint32_t num_dims, uint32_t num_mappings, diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h index ce3abb78280..bf2ae434275 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h @@ -15,8 +15,8 @@ namespace vespalib::eval::packed_mixed_tensor { * Builder for PackedMappings. * Copies label values in all addresses added * and packs all resulting data into a block of memory - * held by the built object (or optionally some - * larger aggregating object by target_memory). + * held by the built object, usually part of a larger + * aggregating object by using target_memory() method. **/ class PackedMappingsBuilder { public: @@ -30,15 +30,29 @@ public: ~PackedMappingsBuilder(); + // returns a new index for new addresses + // may be called multiple times with same address, + // will then return the same index for that address. uint32_t add_mapping_for(SparseAddress address); - std::unique_ptr build_mappings() const; - + // how much extra memory is needed by target_memory + // not including sizeof(PackedMappings) size_t extra_memory() const; + + // put data that PackedMappings can refer to in the given + // memory block, and return an object referring to it. PackedMappings target_memory(char *mem_start, char *mem_end) const; + // number of dimensions uint32_t num_mapped_dims() const { return _num_dims; } + + // how many unique addresses have been added? size_t size() const { return _mappings.size(); } + + // build a self-contained PackedMappings object; + // used for unit testing. + std::unique_ptr build_mappings() const; + private: uint32_t _num_dims; std::set _labels; -- cgit v1.2.3 From 97535ec41f52e4c6319495ced85933ead42b0a2b Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Mon, 21 Sep 2020 18:28:56 +0200 Subject: Revert "Revert "Jonmv/async document v1"" This reverts commit 7b03effca945dea607ed7a3c3debebda303992ab. --- .../vespa/model/container/ContainerCluster.java | 1 + .../MessageBusDocumentAccessProvider.java | 55 ++++++++++++++++++++++ .../container/core/documentapi/package-info.java | 5 ++ documentapi/abi-spec.json | 2 + .../java/com/yahoo/documentapi/DocumentAccess.java | 6 +-- .../java/com/yahoo/documentapi/RemoveResponse.java | 6 ++- .../java/com/yahoo/documentapi/UpdateResponse.java | 6 ++- 7 files changed, 76 insertions(+), 5 deletions(-) create mode 100644 container-core/src/main/java/com/yahoo/container/core/documentapi/MessageBusDocumentAccessProvider.java create mode 100644 container-core/src/main/java/com/yahoo/container/core/documentapi/package-info.java diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 87e8f16f88c..d63987f6c07 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -181,6 +181,7 @@ public abstract class ContainerCluster addSimpleComponent(com.yahoo.metrics.simple.jdisc.JdiscMetricsFactory.class.getName(), null, MetricProperties.BUNDLE_SYMBOLIC_NAME); addSimpleComponent("com.yahoo.container.jdisc.state.StateMonitor"); addSimpleComponent("com.yahoo.container.jdisc.ContainerThreadFactory"); + addSimpleComponent(com.yahoo.container.core.documentapi.MessageBusDocumentAccessProvider.class.getName()); addSimpleComponent("com.yahoo.container.handler.VipStatus"); addSimpleComponent(com.yahoo.container.handler.ClustersStatus.class.getName()); addJaxProviders(); diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/MessageBusDocumentAccessProvider.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/MessageBusDocumentAccessProvider.java new file mode 100644 index 00000000000..c3cd78e4da9 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/core/documentapi/MessageBusDocumentAccessProvider.java @@ -0,0 +1,55 @@ +package com.yahoo.container.core.documentapi; + +import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; +import com.yahoo.container.di.componentgraph.Provider; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.documentapi.DocumentAccess; +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; +import com.yahoo.documentapi.messagebus.MessageBusParams; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.vespa.config.content.LoadTypeConfig; + +/** + * Has a lazily populated reference to a {@link MessageBusDocumentAccess}. + * + * @author jonmv + */ +public class MessageBusDocumentAccessProvider extends AbstractComponent implements Provider { + + private final DocumentmanagerConfig documentmanagerConfig; + private final LoadTypeConfig loadTypeConfig; + private final Object monitor = new Object(); + private boolean shutDown = false; + private DocumentAccess access = null; + + @Inject + public MessageBusDocumentAccessProvider(DocumentmanagerConfig documentmanagerConfig, LoadTypeConfig loadTypeConfig) { + this.documentmanagerConfig = documentmanagerConfig; + this.loadTypeConfig = loadTypeConfig; + } + + @Override + public DocumentAccess get() { + synchronized (monitor) { + if (access == null) { + access = new MessageBusDocumentAccess((MessageBusParams) new MessageBusParams(new LoadTypeSet(loadTypeConfig)).setDocumentmanagerConfig(documentmanagerConfig)); + if (shutDown) + access.shutdown(); + } + return access; + } + } + + @Override + public void deconstruct() { + synchronized (monitor) { + if ( ! shutDown) { + shutDown = true; + if (access != null) + access.shutdown(); + } + } + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/package-info.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/package-info.java new file mode 100644 index 00000000000..de524237499 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/core/documentapi/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.container.core.documentapi; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/documentapi/abi-spec.json b/documentapi/abi-spec.json index a28ce1ee6db..6f5e6d66e2a 100644 --- a/documentapi/abi-spec.json +++ b/documentapi/abi-spec.json @@ -354,6 +354,7 @@ "methods": [ "public void (long, boolean)", "public boolean wasFound()", + "public boolean isSuccess()", "public int hashCode()", "public boolean equals(java.lang.Object)", "public java.lang.String toString()" @@ -561,6 +562,7 @@ "methods": [ "public void (long, boolean)", "public boolean wasFound()", + "public boolean isSuccess()", "public int hashCode()", "public boolean equals(java.lang.Object)", "public java.lang.String toString()" diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java index 308eafcd596..7a9818ba4fd 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java @@ -27,9 +27,9 @@ import com.yahoo.config.subscription.ConfigSubscriber; *

This class is the factory for creating the four session types mentioned above.

* *

There may be multiple implementations of the document api classes. If - * default configuration is sufficient, use the {@link #createDefault} method to - * return a running document access. Note that there are running threads within - * an access object, so you must shut it down when done.

+ * default configuration is sufficient, simply inject a {@code DocumentAccess} to + * obtain a running document access. If you instead create a concrete implementation, note that + * there are running threads within an access object, so you must shut it down when done.

* *

An implementation of the Document Api may support just a subset of the * access types defined in this interface. For example, some document diff --git a/documentapi/src/main/java/com/yahoo/documentapi/RemoveResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/RemoveResponse.java index 502588a3d5f..2a7c6f45d95 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/RemoveResponse.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/RemoveResponse.java @@ -12,7 +12,7 @@ public class RemoveResponse extends Response { private final boolean wasFound; public RemoveResponse(long requestId, boolean wasFound) { - super(requestId); + super(requestId, null, wasFound ? Outcome.SUCCESS : Outcome.NOT_FOUND); this.wasFound = wasFound; } @@ -20,6 +20,10 @@ public class RemoveResponse extends Response { return wasFound; } + @Override + // TODO: fix this when/if NOT_FOUND is no longer a success. + public boolean isSuccess() { return super.isSuccess() || outcome() == Outcome.NOT_FOUND; } + @Override public int hashCode() { return super.hashCode() + Boolean.valueOf(wasFound).hashCode(); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/UpdateResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/UpdateResponse.java index 96bf58c1e64..aca34a92a30 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/UpdateResponse.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/UpdateResponse.java @@ -12,7 +12,7 @@ public class UpdateResponse extends Response { private final boolean wasFound; public UpdateResponse(long requestId, boolean wasFound) { - super(requestId); + super(requestId, null, wasFound ? Outcome.SUCCESS : Outcome.NOT_FOUND); this.wasFound = wasFound; } @@ -20,6 +20,10 @@ public class UpdateResponse extends Response { return wasFound; } + @Override + // TODO: fix this when/if NOT_FOUND is no longer a success. + public boolean isSuccess() { return super.isSuccess() || outcome() == Outcome.NOT_FOUND; } + @Override public int hashCode() { return super.hashCode() + Boolean.valueOf(wasFound).hashCode(); -- cgit v1.2.3 From edae2586a4b7927ab9a1771441f46b81b779b0dd Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Mon, 21 Sep 2020 18:31:31 +0200 Subject: Less defensive shutdown --- .../core/documentapi/MessageBusDocumentAccessProvider.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/MessageBusDocumentAccessProvider.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/MessageBusDocumentAccessProvider.java index c3cd78e4da9..cd6a2fc8b12 100644 --- a/container-core/src/main/java/com/yahoo/container/core/documentapi/MessageBusDocumentAccessProvider.java +++ b/container-core/src/main/java/com/yahoo/container/core/documentapi/MessageBusDocumentAccessProvider.java @@ -33,9 +33,10 @@ public class MessageBusDocumentAccessProvider extends AbstractComponent implemen public DocumentAccess get() { synchronized (monitor) { if (access == null) { - access = new MessageBusDocumentAccess((MessageBusParams) new MessageBusParams(new LoadTypeSet(loadTypeConfig)).setDocumentmanagerConfig(documentmanagerConfig)); if (shutDown) - access.shutdown(); + throw new IllegalStateException("This document access has been shut down"); + + access = new MessageBusDocumentAccess((MessageBusParams) new MessageBusParams(new LoadTypeSet(loadTypeConfig)).setDocumentmanagerConfig(documentmanagerConfig)); } return access; } @@ -44,11 +45,9 @@ public class MessageBusDocumentAccessProvider extends AbstractComponent implemen @Override public void deconstruct() { synchronized (monitor) { - if ( ! shutDown) { - shutDown = true; - if (access != null) - access.shutdown(); - } + shutDown = true; + if (access != null) + access.shutdown(); } } -- cgit v1.2.3 From 8197457af4b0c0cfc88dc3c26b8667e7b004a60a Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Mon, 21 Sep 2020 18:46:50 +0200 Subject: Move laziness down to avoid initialising MessageBusDocumentAccess unless _used_ --- .../MessageBusDocumentAccessProvider.java | 109 +++++++++++++++++---- 1 file changed, 88 insertions(+), 21 deletions(-) diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/MessageBusDocumentAccessProvider.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/MessageBusDocumentAccessProvider.java index cd6a2fc8b12..78653a81e58 100644 --- a/container-core/src/main/java/com/yahoo/container/core/documentapi/MessageBusDocumentAccessProvider.java +++ b/container-core/src/main/java/com/yahoo/container/core/documentapi/MessageBusDocumentAccessProvider.java @@ -4,51 +4,118 @@ import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.documentapi.AsyncParameters; +import com.yahoo.documentapi.AsyncSession; import com.yahoo.documentapi.DocumentAccess; +import com.yahoo.documentapi.DocumentAccessParams; +import com.yahoo.documentapi.SubscriptionParameters; +import com.yahoo.documentapi.SubscriptionSession; +import com.yahoo.documentapi.SyncParameters; +import com.yahoo.documentapi.SyncSession; +import com.yahoo.documentapi.VisitorDestinationParameters; +import com.yahoo.documentapi.VisitorDestinationSession; +import com.yahoo.documentapi.VisitorParameters; +import com.yahoo.documentapi.VisitorSession; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.vespa.config.content.LoadTypeConfig; /** - * Has a lazily populated reference to a {@link MessageBusDocumentAccess}. + * Lets a lazily initialised DocumentAccess forwarding to a real MessageBusDocumentAccess be injected in containers. * * @author jonmv */ public class MessageBusDocumentAccessProvider extends AbstractComponent implements Provider { - private final DocumentmanagerConfig documentmanagerConfig; - private final LoadTypeConfig loadTypeConfig; - private final Object monitor = new Object(); - private boolean shutDown = false; - private DocumentAccess access = null; + private final DocumentAccess access; @Inject + // TODO jonmv: Have Slobrok and RPC config injected as well. public MessageBusDocumentAccessProvider(DocumentmanagerConfig documentmanagerConfig, LoadTypeConfig loadTypeConfig) { - this.documentmanagerConfig = documentmanagerConfig; - this.loadTypeConfig = loadTypeConfig; + this.access = new LazyForwardingMessageBusDocumentAccess(documentmanagerConfig, loadTypeConfig); } @Override public DocumentAccess get() { - synchronized (monitor) { - if (access == null) { - if (shutDown) - throw new IllegalStateException("This document access has been shut down"); - - access = new MessageBusDocumentAccess((MessageBusParams) new MessageBusParams(new LoadTypeSet(loadTypeConfig)).setDocumentmanagerConfig(documentmanagerConfig)); - } - return access; - } + return access; } @Override public void deconstruct() { - synchronized (monitor) { - shutDown = true; - if (access != null) - access.shutdown(); + access.shutdown(); + } + + + private static class LazyForwardingMessageBusDocumentAccess extends DocumentAccess { + + private final DocumentmanagerConfig documentmanagerConfig; + private final LoadTypeConfig loadTypeConfig; + private final Object monitor = new Object(); + + private DocumentAccess delegate = null; + private boolean shutDown = false; + + public LazyForwardingMessageBusDocumentAccess(DocumentmanagerConfig documentmanagerConfig, + LoadTypeConfig loadTypeConfig) { + super(new DocumentAccessParams().setDocumentmanagerConfig(documentmanagerConfig)); + this.documentmanagerConfig = documentmanagerConfig; + this.loadTypeConfig = loadTypeConfig; } + + private DocumentAccess delegate() { + synchronized (monitor) { + if (delegate == null) { + if (shutDown) + throw new IllegalStateException("This document access has been shut down"); + + delegate = new MessageBusDocumentAccess((MessageBusParams) new MessageBusParams(new LoadTypeSet(loadTypeConfig)).setDocumentmanagerConfig(documentmanagerConfig)); + } + return delegate; + } + } + + @Override + public void shutdown() { + synchronized (monitor) { + super.shutdown(); + shutDown = true; + if (delegate != null) + delegate.shutdown(); + } + } + + @Override + public SyncSession createSyncSession(SyncParameters parameters) { + return delegate().createSyncSession(parameters); + } + + @Override + public AsyncSession createAsyncSession(AsyncParameters parameters) { + return delegate().createAsyncSession(parameters); + } + + @Override + public VisitorSession createVisitorSession(VisitorParameters parameters) throws ParseException { + return delegate().createVisitorSession(parameters); + } + + @Override + public VisitorDestinationSession createVisitorDestinationSession(VisitorDestinationParameters parameters) { + return delegate().createVisitorDestinationSession(parameters); + } + + @Override + public SubscriptionSession createSubscription(SubscriptionParameters parameters) { + return delegate().createSubscription(parameters); + } + + @Override + public SubscriptionSession openSubscription(SubscriptionParameters parameters) { + return delegate().openSubscription(parameters); + } + } } -- cgit v1.2.3 From 31b179d62e3dec85d9af5058857ed16fa81a4002 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Mon, 21 Sep 2020 18:23:22 +0000 Subject: Add CommitResult. --- .../src/vespa/searchlib/transactionlog/common.cpp | 26 ++++++++++++++++++--- .../src/vespa/searchlib/transactionlog/common.h | 27 +++++++++++++++++++--- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/searchlib/src/vespa/searchlib/transactionlog/common.cpp b/searchlib/src/vespa/searchlib/transactionlog/common.cpp index 556ebca06ec..3308f3182dc 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/common.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/common.cpp @@ -121,11 +121,26 @@ Packet::add(const Packet::Entry & e) _range.to(e.serial()); } +Writer::CommitResult::CommitResult() + : _callBacks() +{} +Writer::CommitResult::CommitResult( CommitPayload commitPayLoad) + : _callBacks(std::move(commitPayLoad)) +{} + +Writer::CommitResult::~CommitResult() = default; + CommitChunk::CommitChunk(size_t reserveBytes, size_t reserveCount) : _data(reserveBytes), - _callBacks() + _callBacks(std::make_shared()) +{ + _callBacks->reserve(reserveCount); +} + +CommitChunk::CommitChunk(size_t reserveBytes, Writer::CommitPayload postponed) + : _data(reserveBytes), + _callBacks(std::move(postponed)) { - _callBacks.reserve(reserveCount); } CommitChunk::~CommitChunk() = default; @@ -133,7 +148,12 @@ CommitChunk::~CommitChunk() = default; void CommitChunk::add(const Packet &packet, Writer::DoneCallback onDone) { _data.merge(packet); - _callBacks.emplace_back(std::move(onDone)); + _callBacks->emplace_back(std::move(onDone)); +} + +Writer::CommitResult +CommitChunk::createCommitResult() const { + return Writer::CommitResult(_callBacks); } } diff --git a/searchlib/src/vespa/searchlib/transactionlog/common.h b/searchlib/src/vespa/searchlib/transactionlog/common.h index 7cdfad44b87..c8f7a81ac7d 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/common.h +++ b/searchlib/src/vespa/searchlib/transactionlog/common.h @@ -84,6 +84,21 @@ int makeDirectory(const char * dir); class Writer { public: using DoneCallback = std::shared_ptr; + using DoneCallbacksList = std::vector; + using CommitPayload = std::shared_ptr; + class CommitResult { + public: + CommitResult(); + CommitResult(CommitPayload callBacks); + CommitResult(CommitResult &&) noexcept = default; + CommitResult & operator = (CommitResult &&) noexcept = default; + CommitResult(const CommitResult &) = delete; + CommitResult & operator = (const CommitResult &) = delete; + ~CommitResult(); + size_t getNumOperations() const { return _callBacks->size(); } + private: + CommitPayload _callBacks; + }; virtual ~Writer() = default; virtual void commit(const Packet & packet, DoneCallback done) = 0; }; @@ -106,14 +121,20 @@ public: class CommitChunk { public: CommitChunk(size_t reserveBytes, size_t reserveCount); + CommitChunk(size_t reserveBytes, Writer::CommitPayload postponed); ~CommitChunk(); + bool empty() const { return _callBacks->empty(); } void add(const Packet & packet, Writer::DoneCallback onDone); size_t sizeBytes() const { return _data.sizeBytes(); } const Packet & getPacket() const { return _data; } - size_t getNumCallBacks() const { return _callBacks.size(); } + size_t getNumCallBacks() const { return _callBacks->size(); } + Writer::CommitResult createCommitResult() const; + void setCommitDoneCallback(Writer::DoneCallback onDone) { _onCommitDone = std::move(onDone); } + Writer::CommitPayload stealCallbacks() { return std::move(_callBacks); } private: - Packet _data; - std::vector _callBacks; + Packet _data; + Writer::CommitPayload _callBacks; + Writer::DoneCallback _onCommitDone; }; } -- cgit v1.2.3 From 6eab511ac74f9c60427d3510a55003e4c7d380ac Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Mon, 21 Sep 2020 19:07:55 +0000 Subject: Add startCommit method --- .../documentdb/feedhandler/feedhandler_test.cpp | 1 + .../lid_space_compaction_test.cpp | 3 ++ .../maintenancecontroller_test.cpp | 3 ++ .../vespa/searchcore/proton/server/feedhandler.cpp | 42 ++++++++++++++++++---- .../vespa/searchcore/proton/server/feedhandler.h | 1 + .../searchcore/proton/server/i_operation_storer.h | 6 ++++ .../tests/transactionlog/translogclient_test.cpp | 5 +-- .../src/vespa/searchlib/transactionlog/common.h | 3 +- .../src/vespa/searchlib/transactionlog/domain.cpp | 8 ++++- .../src/vespa/searchlib/transactionlog/domain.h | 3 +- .../searchlib/transactionlog/translogserver.cpp | 3 +- 11 files changed, 65 insertions(+), 13 deletions(-) diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index ad0ce0b26c4..72592cca681 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -434,6 +434,7 @@ struct MyTlsWriter : TlsWriter { MyTlsWriter() : store_count(0), erase_count(0), erase_return(true) {} void appendOperation(const FeedOperation &, DoneCallback) override { ++store_count; } + CommitResult startCommit(DoneCallback) override { return CommitResult(); } bool erase(SerialNum) override { ++erase_count; return erase_return; } SerialNum sync(SerialNum syncTo) override { diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp index 31882061b1c..04444647b5d 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp @@ -172,6 +172,9 @@ struct MyStorer : public IOperationStorer { ++_compactCnt; } } + CommitResult startCommit(DoneCallback) override { + return CommitResult(); + } }; struct MyFrozenBucketHandler : public IFrozenBucketHandler { diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index 8df62705cb3..f033dfd50a8 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -235,6 +235,9 @@ public: // Implements IOperationStorer void appendOperation(const FeedOperation &op, DoneCallback) override; + CommitResult startCommit(DoneCallback) override { + return CommitResult(); + } uint32_t getHeartBeats() const { return _heartBeats; diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 734ef01d33a..c45d216a631 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -58,17 +58,20 @@ class TlsMgrWriter : public TlsWriter { std::shared_ptr _writer; public: TlsMgrWriter(TransactionLogManager &tls_mgr, - const search::transactionlog::WriterFactory & factory) : - _tls_mgr(tls_mgr), - _writer(factory.getWriter(tls_mgr.getDomainName())) + const search::transactionlog::WriterFactory & factory) + : _tls_mgr(tls_mgr), + _writer(factory.getWriter(tls_mgr.getDomainName())) { } void appendOperation(const FeedOperation &op, DoneCallback onDone) override; + CommitResult startCommit(DoneCallback onDone) override { + return _writer->startCommit(std::move(onDone)); + } bool erase(SerialNum oldest_to_keep) override; SerialNum sync(SerialNum syncTo) override; }; - -void TlsMgrWriter::appendOperation(const FeedOperation &op, DoneCallback onDone) { +void +TlsMgrWriter::appendOperation(const FeedOperation &op, DoneCallback onDone) { using Packet = search::transactionlog::Packet; vespalib::nbostream stream; op.serialize(stream); @@ -77,9 +80,11 @@ void TlsMgrWriter::appendOperation(const FeedOperation &op, DoneCallback onDone) Packet::Entry entry(op.getSerialNum(), op.getType(), vespalib::ConstBufferRef(stream.data(), stream.size())); Packet packet(entry.serializedSize()); packet.add(entry); - _writer->commit(packet, std::move(onDone)); + _writer->append(packet, std::move(onDone)); } -bool TlsMgrWriter::erase(SerialNum oldest_to_keep) { + +bool +TlsMgrWriter::erase(SerialNum oldest_to_keep) { return _tls_mgr.getSession()->erase(oldest_to_keep); } @@ -104,6 +109,24 @@ TlsMgrWriter::sync(SerialNum syncTo) throw IllegalStateException(make_string("Failed to sync TLS to token %" PRIu64 ".", syncTo)); } +class OnCommitDone : public search::IDestructorCallback { +public: + OnCommitDone(Executor & executor, std::unique_ptr task) + : _executor(executor), + _task(std::move(task)) + {} + ~OnCommitDone() override { _executor.execute(std::move(_task)); } +private: + Executor & _executor; + std::unique_ptr _task; +}; + +template +struct KeepAlive : public search::IDestructorCallback { + explicit KeepAlive(T toKeep) : _toKeep(std::move(toKeep)) { } + ~KeepAlive() override = default; + T _toKeep; +}; } // namespace void @@ -479,6 +502,11 @@ FeedHandler::appendOperation(const FeedOperation &op, TlsWriter::DoneCallback on _tlsWriter->appendOperation(op, std::move(onDone)); } +FeedHandler::CommitResult +FeedHandler::startCommit(DoneCallback onDone) { + return _tlsWriter->startCommit(std::move(onDone)); +} + void FeedHandler::storeOperationSync(const FeedOperation &op) { vespalib::Gate gate; diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index 29961f4a6cc..6a5fa318959 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -226,6 +226,7 @@ public: void performPruneRemovedDocuments(PruneRemovedDocumentsOperation &pruneOp) override; void syncTls(SerialNum syncTo); void appendOperation(const FeedOperation &op, DoneCallback onDone) override; + CommitResult startCommit(DoneCallback onDone) override; void storeOperationSync(const FeedOperation & op); void considerDelayedPrune(); }; diff --git a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h index 47b81a9a17f..481a70564cc 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h @@ -14,12 +14,18 @@ class FeedOperation; struct IOperationStorer { using DoneCallback = search::transactionlog::Writer::DoneCallback; + using CommitResult = search::transactionlog::Writer::CommitResult; virtual ~IOperationStorer() = default; /** * Assign serial number to (if not set) and store the given operation. */ virtual void appendOperation(const FeedOperation &op, DoneCallback onDone) = 0; + void storeOperation(const FeedOperation &op, DoneCallback onDone) { + appendOperation(op, onDone); + startCommit(std::move(onDone)); + } + virtual CommitResult startCommit(DoneCallback onDone) = 0; }; } // namespace proton diff --git a/searchlib/src/tests/transactionlog/translogclient_test.cpp b/searchlib/src/tests/transactionlog/translogclient_test.cpp index 9e5021b4778..478bc594368 100644 --- a/searchlib/src/tests/transactionlog/translogclient_test.cpp +++ b/searchlib/src/tests/transactionlog/translogclient_test.cpp @@ -329,11 +329,12 @@ fillDomainTest(TransLogServer & s1, const vespalib::string & domain, size_t numP Packet::Entry e(value+1, j+1, vespalib::ConstBufferRef((const char *)&value, sizeof(value))); p->add(e); if ( p->sizeBytes() > DEFAULT_PACKET_SIZE ) { - domainWriter->commit(*p, std::make_shared(inFlight)); + domainWriter->append(*p, std::make_shared(inFlight)); p = std::make_unique(DEFAULT_PACKET_SIZE); } } - domainWriter->commit(*p, std::make_shared(inFlight)); + domainWriter->append(*p, std::make_shared(inFlight)); + domainWriter->startCommit(Writer::DoneCallback()); LOG(info, "Inflight %ld", inFlight.load()); } while (inFlight.load() != 0) { diff --git a/searchlib/src/vespa/searchlib/transactionlog/common.h b/searchlib/src/vespa/searchlib/transactionlog/common.h index c8f7a81ac7d..43b20b57045 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/common.h +++ b/searchlib/src/vespa/searchlib/transactionlog/common.h @@ -100,7 +100,8 @@ public: CommitPayload _callBacks; }; virtual ~Writer() = default; - virtual void commit(const Packet & packet, DoneCallback done) = 0; + virtual void append(const Packet & packet, DoneCallback done) = 0; + virtual CommitResult startCommit(DoneCallback onDone) = 0; }; class WriterFactory { diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index 415ccafda70..7a7be3ab86b 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -310,8 +310,14 @@ Domain::optionallyRotateFile(SerialNum serialNum) { return dp; } +Domain::CommitResult +Domain::startCommit(DoneCallback onDone) { + (void) onDone; + return CommitResult(); +} + void -Domain::commit(const Packet & packet, Writer::DoneCallback onDone) +Domain::append(const Packet & packet, Writer::DoneCallback onDone) { (void) onDone; vespalib::nbostream_longlivedbuf is(packet.getHandle().data(), packet.getHandle().size()); diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h index 9adff564cc8..86e3681f0a1 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h @@ -27,7 +27,8 @@ public: const vespalib::string & name() const { return _name; } bool erase(SerialNum to); - void commit(const Packet & packet, Writer::DoneCallback onDone) override; + void append(const Packet & packet, Writer::DoneCallback onDone) override; + CommitResult startCommit(DoneCallback onDone) override; int visit(const Domain::SP & self, SerialNum from, SerialNum to, std::unique_ptr dest); SerialNum begin() const; diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index 269ed7e9380..9b7d328d9cb 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -572,7 +572,8 @@ TransLogServer::domainCommit(FRT_RPCRequest *req) Packet packet(params[1]._data._buf, params[1]._data._len); try { vespalib::Gate gate; - domain->commit(packet, make_shared(gate)); + domain->append(packet, make_shared(gate)); + domain->startCommit(make_shared()); gate.await(); ret.AddInt32(0); ret.AddString("ok"); -- cgit v1.2.3 From eb29b8d356638d0e9e122c7d7c4ffb400d4c1745 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Tue, 22 Sep 2020 09:03:08 +0200 Subject: Add flag controlling whether to use config server VIP --- flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index b704516bd2d..35b2f05f30b 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -368,6 +368,13 @@ public class Flags { APPLICATION_ID ); + public static final UnboundBooleanFlag USE_CONFIG_SERVER_VIP = defineFeatureFlag( + "use-config-server-vip", + false, + "Whether the controller should use a config server VIP or not", + "Takes effect immediately" + ); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { -- cgit v1.2.3 From 8b8cc004b24ba480711161fee9c0c847a5447fc9 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Tue, 22 Sep 2020 09:07:43 +0200 Subject: Change default value of configserver-distribute-application-package to true --- flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index b704516bd2d..3a11da758ed 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -278,7 +278,7 @@ public class Flags { ZONE_ID); public static final UnboundBooleanFlag CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE = defineFeatureFlag( - "configserver-distribute-application-package", false, + "configserver-distribute-application-package", true, "Whether the application package should be distributed to other config servers during a deployment", "Takes effect immediately"); -- cgit v1.2.3 From 78cb9942157ab26bd7d2166eac9a9f75b7338664 Mon Sep 17 00:00:00 2001 From: kkraune Date: Tue, 22 Sep 2020 11:26:04 +0200 Subject: output values used for test runtime --- .../java/ai/vespa/hosted/cd/cloud/impl/VespaTestRuntime.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/cloud/impl/VespaTestRuntime.java b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/cloud/impl/VespaTestRuntime.java index 5f6cc252d85..91f08fb9943 100644 --- a/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/cloud/impl/VespaTestRuntime.java +++ b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/cloud/impl/VespaTestRuntime.java @@ -54,7 +54,14 @@ public class VespaTestRuntime implements TestRuntime { private static TestConfig configFromPropertyOrController() { String configPath = System.getProperty("vespa.test.config"); - return configPath != null ? fromFile(configPath) : fromController(); + if (configPath != null) { + System.out.println("TestRuntime: Using test config from " + configPath); + return fromFile(configPath); + } + else { + System.out.println("TestRuntime: Using test config from Vespa Cloud"); + return fromController(); + } } private static TestConfig fromFile(String path) { @@ -76,6 +83,8 @@ public class VespaTestRuntime implements TestRuntime { Environment environment = Properties.environment().orElse(Environment.dev); ZoneId zone = Properties.region().map(region -> ZoneId.from(environment, region)) .orElseGet(() -> controller.defaultZone(environment)); + System.out.println("TestRuntime: Requesting endpoint config for tenant.application.instance: " + id.toFullString()); + System.out.println("TestRuntime: Zone: " + zone.toString()); return controller.testConfig(id, zone); } } -- cgit v1.2.3 From 20d0f96d2e6f276669293780adc67421aea0dee2 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 22 Sep 2020 10:05:23 +0000 Subject: Add [[nodiscard]] attribute to startCommit. --- .../vespa/searchcore/proton/server/feedhandler.cpp | 2 +- .../vespa/searchcore/proton/server/feedhandler.h | 2 +- .../searchcore/proton/server/i_operation_storer.h | 6 +--- .../tests/transactionlog/translogclient_test.cpp | 2 +- .../src/vespa/searchlib/common/fileheadercontext.h | 34 +++++------------- .../src/vespa/searchlib/transactionlog/common.h | 6 ++-- .../src/vespa/searchlib/transactionlog/domain.cpp | 3 +- .../src/vespa/searchlib/transactionlog/domain.h | 40 +++++++++++----------- .../searchlib/transactionlog/translogserver.cpp | 2 +- 9 files changed, 37 insertions(+), 60 deletions(-) diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index c45d216a631..37dfddf0c2c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -63,7 +63,7 @@ public: _writer(factory.getWriter(tls_mgr.getDomainName())) { } void appendOperation(const FeedOperation &op, DoneCallback onDone) override; - CommitResult startCommit(DoneCallback onDone) override { + [[nodiscard]] CommitResult startCommit(DoneCallback onDone) override { return _writer->startCommit(std::move(onDone)); } bool erase(SerialNum oldest_to_keep) override; diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index 6a5fa318959..4807c596130 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -226,7 +226,7 @@ public: void performPruneRemovedDocuments(PruneRemovedDocumentsOperation &pruneOp) override; void syncTls(SerialNum syncTo); void appendOperation(const FeedOperation &op, DoneCallback onDone) override; - CommitResult startCommit(DoneCallback onDone) override; + [[nodiscard]] CommitResult startCommit(DoneCallback onDone) override; void storeOperationSync(const FeedOperation & op); void considerDelayedPrune(); }; diff --git a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h index 481a70564cc..b276779c2ee 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h @@ -21,11 +21,7 @@ struct IOperationStorer * Assign serial number to (if not set) and store the given operation. */ virtual void appendOperation(const FeedOperation &op, DoneCallback onDone) = 0; - void storeOperation(const FeedOperation &op, DoneCallback onDone) { - appendOperation(op, onDone); - startCommit(std::move(onDone)); - } - virtual CommitResult startCommit(DoneCallback onDone) = 0; + [[nodiscard]] virtual CommitResult startCommit(DoneCallback onDone) = 0; }; } // namespace proton diff --git a/searchlib/src/tests/transactionlog/translogclient_test.cpp b/searchlib/src/tests/transactionlog/translogclient_test.cpp index 478bc594368..fffb70467a3 100644 --- a/searchlib/src/tests/transactionlog/translogclient_test.cpp +++ b/searchlib/src/tests/transactionlog/translogclient_test.cpp @@ -334,7 +334,7 @@ fillDomainTest(TransLogServer & s1, const vespalib::string & domain, size_t numP } } domainWriter->append(*p, std::make_shared(inFlight)); - domainWriter->startCommit(Writer::DoneCallback()); + auto keep = domainWriter->startCommit(Writer::DoneCallback()); LOG(info, "Inflight %ld", inFlight.load()); } while (inFlight.load() != 0) { diff --git a/searchlib/src/vespa/searchlib/common/fileheadercontext.h b/searchlib/src/vespa/searchlib/common/fileheadercontext.h index 6f76fe1717d..8bb3d6a56a6 100644 --- a/searchlib/src/vespa/searchlib/common/fileheadercontext.h +++ b/searchlib/src/vespa/searchlib/common/fileheadercontext.h @@ -3,40 +3,22 @@ #include -namespace vespalib -{ - -class GenericHeader; - +namespace vespalib { + class GenericHeader; } -namespace search -{ - -namespace common -{ +namespace search::common { class FileHeaderContext { public: FileHeaderContext(); + virtual ~FileHeaderContext(); - virtual - ~FileHeaderContext(); + virtual void addTags(vespalib::GenericHeader &header, const vespalib::string &name) const = 0; - virtual void - addTags(vespalib::GenericHeader &header, - const vespalib::string &name) const = 0; - - static void - addCreateAndFreezeTime(vespalib::GenericHeader &header); - - static void - setFreezeTime(vespalib::GenericHeader &header); + static void addCreateAndFreezeTime(vespalib::GenericHeader &header); + static void setFreezeTime(vespalib::GenericHeader &header); }; - -} // namespace common - -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/transactionlog/common.h b/searchlib/src/vespa/searchlib/transactionlog/common.h index 43b20b57045..5d07d51cdf2 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/common.h +++ b/searchlib/src/vespa/searchlib/transactionlog/common.h @@ -16,7 +16,7 @@ class SerialNumRange { public: SerialNumRange() : _from(0), _to(0) { } - SerialNumRange(SerialNum f) : _from(f), _to(f ? f-1 : f) { } + explicit SerialNumRange(SerialNum f) : _from(f), _to(f ? f-1 : f) { } SerialNumRange(SerialNum f, SerialNum t) : _from(f), _to(t) { } bool operator == (const SerialNumRange & b) const { return cmp(b) == 0; } bool operator < (const SerialNumRange & b) const { return cmp(b) < 0; } @@ -63,7 +63,7 @@ public: vespalib::ConstBufferRef _data; }; public: - Packet(size_t reserved) : _count(0), _range(), _buf(reserved) { } + explicit Packet(size_t reserved) : _count(0), _range(), _buf(reserved) { } Packet(const void * buf, size_t sz); void add(const Entry & data); void clear() { _buf.clear(); _count = 0; _range.from(0); _range.to(0); } @@ -101,7 +101,7 @@ public: }; virtual ~Writer() = default; virtual void append(const Packet & packet, DoneCallback done) = 0; - virtual CommitResult startCommit(DoneCallback onDone) = 0; + [[nodiscard]] virtual CommitResult startCommit(DoneCallback onDone) = 0; }; class WriterFactory { diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index 7a7be3ab86b..ff2d963ccd9 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -25,7 +25,6 @@ using vespalib::makeClosure; using vespalib::makeLambdaTask; using vespalib::Monitor; using vespalib::MonitorGuard; -using search::common::FileHeaderContext; using std::runtime_error; using std::make_shared; @@ -35,7 +34,7 @@ Domain::Domain(const string &domainName, const string & baseDir, Executor & exec const DomainConfig & cfg, const FileHeaderContext &fileHeaderContext) : _config(cfg), _lastSerial(0), - _singleCommiter(std::make_unique(1, 128*1024)), + _singleCommitter(std::make_unique(1, 128 * 1024)), _executor(executor), _sessionId(1), _syncMonitor(), diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h index 86e3681f0a1..361ac8c1805 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h @@ -18,8 +18,9 @@ public: using SP = std::shared_ptr; using Executor = vespalib::SyncableThreadExecutor; using DomainPartSP = std::shared_ptr; + using FileHeaderContext = common::FileHeaderContext; Domain(const vespalib::string &name, const vespalib::string &baseDir, Executor & executor, - const DomainConfig & cfg, const common::FileHeaderContext &fileHeaderContext); + const DomainConfig & cfg, const FileHeaderContext &fileHeaderContext); ~Domain() override; @@ -28,14 +29,13 @@ public: bool erase(SerialNum to); void append(const Packet & packet, Writer::DoneCallback onDone) override; - CommitResult startCommit(DoneCallback onDone) override; + [[nodiscard]] CommitResult startCommit(DoneCallback onDone) override; int visit(const Domain::SP & self, SerialNum from, SerialNum to, std::unique_ptr dest); SerialNum begin() const; SerialNum end() const; SerialNum getSynced() const; void triggerSyncNow(); - bool commitIfStale(); bool getMarkedDeleted() const { return _markedDeleted; } void markDeleted() { _markedDeleted = true; } @@ -73,23 +73,23 @@ private: using DomainPartList = std::map; using DurationSeconds = std::chrono::duration; - DomainConfig _config; - SerialNum _lastSerial; - std::unique_ptr _singleCommiter; - Executor & _executor; - std::atomic _sessionId; - vespalib::Monitor _syncMonitor; - bool _pendingSync; - vespalib::string _name; - DomainPartList _parts; - vespalib::Lock _lock; - vespalib::Monitor _currentChunkMonitor; - vespalib::Lock _sessionLock; - SessionList _sessions; - DurationSeconds _maxSessionRunTime; - vespalib::string _baseDir; - const common::FileHeaderContext &_fileHeaderContext; - bool _markedDeleted; + DomainConfig _config; + SerialNum _lastSerial; + std::unique_ptr _singleCommitter; + Executor &_executor; + std::atomic _sessionId; + vespalib::Monitor _syncMonitor; + bool _pendingSync; + vespalib::string _name; + DomainPartList _parts; + vespalib::Lock _lock; + vespalib::Monitor _currentChunkMonitor; + vespalib::Lock _sessionLock; + SessionList _sessions; + DurationSeconds _maxSessionRunTime; + vespalib::string _baseDir; + const FileHeaderContext &_fileHeaderContext; + bool _markedDeleted; }; } diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index 9b7d328d9cb..7be3dd708a5 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -573,7 +573,7 @@ TransLogServer::domainCommit(FRT_RPCRequest *req) try { vespalib::Gate gate; domain->append(packet, make_shared(gate)); - domain->startCommit(make_shared()); + auto keep = domain->startCommit(make_shared()); gate.await(); ret.AddInt32(0); ret.AddString("ok"); -- cgit v1.2.3 From 9a66f21d375e4fa07a96069644615581e55129d5 Mon Sep 17 00:00:00 2001 From: Håvard Pettersen Date: Tue, 22 Sep 2020 08:28:02 +0000 Subject: handle onnx model config for inputs and outputs --- .../src/apps/verify_ranksetup/verify_ranksetup.cpp | 1 + .../index_environment/index_environment_test.cpp | 25 +++++++--- .../verify_ranksetup/verify_ranksetup_test.cpp | 37 +++++++++++++-- .../proton/matching/indexenvironment.cpp | 9 ++-- .../searchcore/proton/matching/indexenvironment.h | 2 +- .../searchcore/proton/matching/onnx_models.cpp | 35 +++++++------- .../vespa/searchcore/proton/matching/onnx_models.h | 15 ++---- .../proton/server/documentdbconfigmanager.cpp | 1 + .../features/onnx_feature/onnx_feature_test.cpp | 26 ++++++++-- .../src/vespa/searchlib/features/onnx_feature.cpp | 29 ++++++++---- searchlib/src/vespa/searchlib/fef/CMakeLists.txt | 3 +- .../src/vespa/searchlib/fef/iindexenvironment.h | 6 +-- searchlib/src/vespa/searchlib/fef/onnx_model.cpp | 55 ++++++++++++++++++++++ searchlib/src/vespa/searchlib/fef/onnx_model.h | 39 +++++++++++++++ .../vespa/searchlib/fef/test/indexenvironment.cpp | 12 ++--- .../vespa/searchlib/fef/test/indexenvironment.h | 7 +-- .../src/vespa/searchvisitor/indexenvironment.h | 4 +- 17 files changed, 230 insertions(+), 76 deletions(-) create mode 100644 searchlib/src/vespa/searchlib/fef/onnx_model.cpp create mode 100644 searchlib/src/vespa/searchlib/fef/onnx_model.h diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp index 2d028f47513..1d492cb558f 100644 --- a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp +++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp @@ -59,6 +59,7 @@ OnnxModels make_models(const OnnxModelsConfig &modelsCfg, const VerifyRanksetupC for (const auto &entry: modelsCfg.model) { if (auto file = get_file(entry.fileref, myCfg)) { model_list.emplace_back(entry.name, file.value()); + OnnxModels::configure(entry, model_list.back()); } else { LOG(warning, "could not find file for onnx model '%s' (ref:'%s')\n", entry.name.c_str(), entry.fileref.c_str()); diff --git a/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp b/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp index 508a60480d0..421ebffafa4 100644 --- a/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp +++ b/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp @@ -8,6 +8,7 @@ using namespace proton::matching; using search::fef::FieldInfo; using search::fef::FieldType; using search::fef::Properties; +using search::fef::OnnxModel; using search::index::Schema; using search::index::schema::CollectionType; using search::index::schema::DataType; @@ -16,8 +17,8 @@ using SIAF = Schema::ImportedAttributeField; OnnxModels make_models() { OnnxModels::Vector list; - list.emplace_back("model1", "path1"); - list.emplace_back("model2", "path2"); + list.emplace_back(OnnxModel("model1", "path1").input_feature("input1","feature1").output_name("output1", "out1")); + list.emplace_back(OnnxModel("model2", "path2")); return OnnxModels(list); } @@ -104,10 +105,22 @@ TEST_F("require that imported attribute fields are extracted in index environmen EXPECT_EQUAL("[documentmetastore]", f.env.getField(2)->name()); } -TEST_F("require that onnx model paths can be obtained", Fixture(buildEmptySchema())) { - EXPECT_EQUAL(f1.env.getOnnxModelFullPath("model1").value(), vespalib::string("path1")); - EXPECT_EQUAL(f1.env.getOnnxModelFullPath("model2").value(), vespalib::string("path2")); - EXPECT_FALSE(f1.env.getOnnxModelFullPath("model3").has_value()); +TEST_F("require that onnx model config can be obtained", Fixture(buildEmptySchema())) { + { + auto model = f1.env.getOnnxModel("model1"); + ASSERT_TRUE(model != nullptr); + EXPECT_EQUAL(model->file_path(), vespalib::string("path1")); + EXPECT_EQUAL(model->input_feature("input1").value(), vespalib::string("feature1")); + EXPECT_EQUAL(model->output_name("output1").value(), vespalib::string("out1")); + } + { + auto model = f1.env.getOnnxModel("model2"); + ASSERT_TRUE(model != nullptr); + EXPECT_EQUAL(model->file_path(), vespalib::string("path2")); + EXPECT_FALSE(model->input_feature("input1").has_value()); + EXPECT_FALSE(model->output_name("output1").has_value()); + } + EXPECT_TRUE(f1.env.getOnnxModel("model3") == nullptr); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp index 1cc8d0280f6..c46990732b7 100644 --- a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp +++ b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,7 @@ const char *invalid_feature = "invalid_feature_name and format"; using namespace search::fef::indexproperties; using namespace search::index; +using search::fef::OnnxModel; using search::index::schema::CollectionType; using search::index::schema::DataType; @@ -69,9 +71,12 @@ struct Setup { std::map properties; std::map constants; std::vector extra_profiles; - std::map onnx_models; + std::map onnx_models; Setup(); ~Setup(); + void add_onnx_model(const OnnxModel &model) { + onnx_models.insert_or_assign(model.name(), model); + } void index(const std::string &name, schema::DataType data_type, schema::CollectionType collection_type) { @@ -155,8 +160,20 @@ struct Setup { void write_onnx_models(const Writer &out) { size_t idx = 0; for (const auto &entry: onnx_models) { - out.fmt("model[%zu].name \"%s\"\n", idx, entry.first.c_str()); + out.fmt("model[%zu].name \"%s\"\n", idx, entry.second.name().c_str()); out.fmt("model[%zu].fileref \"onnx_ref_%zu\"\n", idx, idx); + size_t idx2 = 0; + for (const auto &input: entry.second.inspect_input_features()) { + out.fmt("model[%zu].input[%zu].name \"%s\"\n", idx, idx2, input.first.c_str()); + out.fmt("model[%zu].input[%zu].source \"%s\"\n", idx, idx2, input.second.c_str()); + ++idx2; + } + idx2 = 0; + for (const auto &output: entry.second.inspect_output_names()) { + out.fmt("model[%zu].output[%zu].name \"%s\"\n", idx, idx2, output.first.c_str()); + out.fmt("model[%zu].output[%zu].as \"%s\"\n", idx, idx2, output.second.c_str()); + ++idx2; + } ++idx; } } @@ -164,7 +181,7 @@ struct Setup { size_t idx = 0; for (const auto &entry: onnx_models) { out.fmt("file[%zu].ref \"onnx_ref_%zu\"\n", idx, idx); - out.fmt("file[%zu].path \"%s\"\n", idx, entry.second.c_str()); + out.fmt("file[%zu].path \"%s\"\n", idx, entry.second.file_path().c_str()); ++idx; } } @@ -225,7 +242,12 @@ struct SimpleSetup : Setup { struct OnnxSetup : Setup { OnnxSetup() : Setup() { - onnx_models["simple"] = TEST_PATH("../../../../../eval/src/tests/tensor/onnx_wrapper/simple.onnx"); + add_onnx_model(OnnxModel("simple", TEST_PATH("../../../../../eval/src/tests/tensor/onnx_wrapper/simple.onnx"))); + add_onnx_model(OnnxModel("mapped", TEST_PATH("../../../../../eval/src/tests/tensor/onnx_wrapper/simple.onnx")) + .input_feature("query_tensor", "rankingExpression(qt)") + .input_feature("attribute_tensor", "rankingExpression(at)") + .input_feature("bias_tensor", "rankingExpression(bt)") + .output_name("output", "result")); } }; @@ -350,6 +372,13 @@ TEST_F("require that input type mismatch makes onnx model fail verification", On f.verify_invalid({"onnxModel(simple)"}); } +TEST_F("require that onnx model can have inputs and outputs mapped", OnnxSetup()) { + f.rank_expr("qt", "tensor(a[1],b[4]):[[1,2,3,4]]"); + f.rank_expr("at", "tensor(a[4],b[1]):[[5],[6],[7],[8]]"); + f.rank_expr("bt", "tensor(a[1],b[1]):[[9]]"); + f.verify_valid({"onnxModel(mapped).result"}); +} + //----------------------------------------------------------------------------- TEST_F("cleanup files", Setup()) { diff --git a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp index 5743a3d44d6..013f359c4f9 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp @@ -131,13 +131,10 @@ IndexEnvironment::hintFieldAccess(uint32_t ) const { } void IndexEnvironment::hintAttributeAccess(const string &) const { } -std::optional -IndexEnvironment::getOnnxModelFullPath(const vespalib::string &name) const +const search::fef::OnnxModel * +IndexEnvironment::getOnnxModel(const vespalib::string &name) const { - if (const auto model = _onnxModels.getModel(name)) { - return model->filePath; - } - return std::nullopt; + return _onnxModels.getModel(name); } IndexEnvironment::~IndexEnvironment() = default; diff --git a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.h b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.h index d0e9a516cd0..ad51eb17b4d 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.h +++ b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.h @@ -69,7 +69,7 @@ public: return _constantValueRepo.getConstant(name); } - std::optional getOnnxModelFullPath(const vespalib::string &name) const override; + const search::fef::OnnxModel *getOnnxModel(const vespalib::string &name) const override; ~IndexEnvironment() override; }; diff --git a/searchcore/src/vespa/searchcore/proton/matching/onnx_models.cpp b/searchcore/src/vespa/searchcore/proton/matching/onnx_models.cpp index bdcf3e21d8e..ed80ca28bd6 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/onnx_models.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/onnx_models.cpp @@ -1,25 +1,10 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "onnx_models.h" +#include namespace proton::matching { -OnnxModels::Model::Model(const vespalib::string &name_in, - const vespalib::string &filePath_in) - : name(name_in), - filePath(filePath_in) -{ -} - -OnnxModels::Model::~Model() = default; - -bool -OnnxModels::Model::operator==(const Model &rhs) const -{ - return (name == rhs.name) && - (filePath == rhs.filePath); -} - OnnxModels::OnnxModels() : _models() { @@ -30,15 +15,15 @@ OnnxModels::~OnnxModels() = default; OnnxModels::OnnxModels(const Vector &models) : _models() { - for (const auto &model : models) { - _models.insert(std::make_pair(model.name, model)); + for (const auto &model: models) { + _models.emplace(model.name(), model); } } bool OnnxModels::operator==(const OnnxModels &rhs) const { - return _models == rhs._models; + return (_models == rhs._models); } const OnnxModels::Model * @@ -51,4 +36,16 @@ OnnxModels::getModel(const vespalib::string &name) const return nullptr; } +void +OnnxModels::configure(const ModelConfig &config, Model &model) +{ + assert(config.name == model.name()); + for (const auto &input: config.input) { + model.input_feature(input.name, input.source); + } + for (const auto &output: config.output) { + model.output_name(output.name, output.as); + } +} + } diff --git a/searchcore/src/vespa/searchcore/proton/matching/onnx_models.h b/searchcore/src/vespa/searchcore/proton/matching/onnx_models.h index fdaae657711..65ba524d8fc 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/onnx_models.h +++ b/searchcore/src/vespa/searchcore/proton/matching/onnx_models.h @@ -3,6 +3,8 @@ #pragma once #include +#include +#include #include #include @@ -14,16 +16,8 @@ namespace proton::matching { */ class OnnxModels { public: - struct Model { - vespalib::string name; - vespalib::string filePath; - - Model(const vespalib::string &name_in, - const vespalib::string &filePath_in); - ~Model(); - bool operator==(const Model &rhs) const; - }; - + using ModelConfig = vespa::config::search::core::OnnxModelsConfig::Model; + using Model = search::fef::OnnxModel; using Vector = std::vector; private: @@ -38,6 +32,7 @@ public: bool operator==(const OnnxModels &rhs) const; const Model *getModel(const vespalib::string &name) const; size_t size() const { return _models.size(); } + static void configure(const ModelConfig &config, Model &model); }; } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index a8996abc856..c8b701e82f8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -321,6 +321,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) LOG(info, "Got file path from file acquirer: '%s' (name='%s', ref='%s')", filePath.c_str(), rc.name.c_str(), rc.fileref.c_str()); models.emplace_back(rc.name, filePath); + OnnxModels::configure(rc, models.back()); } } newOnnxModels = std::make_shared(models); diff --git a/searchlib/src/tests/features/onnx_feature/onnx_feature_test.cpp b/searchlib/src/tests/features/onnx_feature/onnx_feature_test.cpp index b49d9c365de..6a1e4ef9fa1 100644 --- a/searchlib/src/tests/features/onnx_feature/onnx_feature_test.cpp +++ b/searchlib/src/tests/features/onnx_feature/onnx_feature_test.cpp @@ -58,8 +58,8 @@ struct OnnxFeatureTest : ::testing::Test { vespalib::string expr_name = feature_name + ".rankingScript"; indexEnv.getProperties().add(expr_name, expr); } - void add_onnx(const vespalib::string &name, const vespalib::string &file) { - indexEnv.addOnnxModel(name, file); + void add_onnx(const OnnxModel &model) { + indexEnv.addOnnxModel(model); } void compile(const vespalib::string &seed) { resolver->addSeed(seed); @@ -89,7 +89,7 @@ TEST_F(OnnxFeatureTest, simple_onnx_model_can_be_calculated) { add_expr("query_tensor", "tensor(a[1],b[4]):[[docid,2,3,4]]"); add_expr("attribute_tensor", "tensor(a[4],b[1]):[[5],[6],[7],[8]]"); add_expr("bias_tensor", "tensor(a[1],b[1]):[[9]]"); - add_onnx("simple", simple_model); + add_onnx(OnnxModel("simple", simple_model)); compile(onnx_feature("simple")); EXPECT_EQ(get(1), TensorSpec("tensor(d0[1],d1[1])").add({{"d0",0},{"d1",0}}, 79.0)); EXPECT_EQ(get("onnxModel(simple).output", 1), TensorSpec("tensor(d0[1],d1[1])").add({{"d0",0},{"d1",0}}, 79.0)); @@ -101,7 +101,7 @@ TEST_F(OnnxFeatureTest, dynamic_onnx_model_can_be_calculated) { add_expr("query_tensor", "tensor(a[1],b[4]):[[docid,2,3,4]]"); add_expr("attribute_tensor", "tensor(a[4],b[1]):[[5],[6],[7],[8]]"); add_expr("bias_tensor", "tensor(a[1],b[2]):[[4,5]]"); - add_onnx("dynamic", dynamic_model); + add_onnx(OnnxModel("dynamic", dynamic_model)); compile(onnx_feature("dynamic")); EXPECT_EQ(get(1), TensorSpec("tensor(d0[1],d1[1])").add({{"d0",0},{"d1",0}}, 79.0)); EXPECT_EQ(get("onnxModel(dynamic).output", 1), TensorSpec("tensor(d0[1],d1[1])").add({{"d0",0},{"d1",0}}, 79.0)); @@ -112,7 +112,7 @@ TEST_F(OnnxFeatureTest, dynamic_onnx_model_can_be_calculated) { TEST_F(OnnxFeatureTest, strange_input_and_output_names_are_normalized) { add_expr("input_0", "tensor(a[2]):[10,20]"); add_expr("input_1", "tensor(a[2]):[5,10]"); - add_onnx("strange_names", strange_names_model); + add_onnx(OnnxModel("strange_names", strange_names_model)); compile(onnx_feature("strange_names")); auto expect_add = TensorSpec("tensor(d0[2])").add({{"d0",0}},15).add({{"d0",1}},30); auto expect_sub = TensorSpec("tensor(d0[2])").add({{"d0",0}},5).add({{"d0",1}},10); @@ -121,4 +121,20 @@ TEST_F(OnnxFeatureTest, strange_input_and_output_names_are_normalized) { EXPECT_EQ(get("onnxModel(strange_names)._baz_0", 1), expect_sub); } +TEST_F(OnnxFeatureTest, input_features_and_output_names_can_be_specified) { + add_expr("my_first_input", "tensor(a[2]):[10,20]"); + add_expr("my_second_input", "tensor(a[2]):[5,10]"); + add_onnx(OnnxModel("custom_names", strange_names_model) + .input_feature("input:0", "rankingExpression(my_first_input)") + .input_feature("input/1", "rankingExpression(my_second_input)") + .output_name("foo/bar", "my_first_output") + .output_name("-baz:0", "my_second_output")); + compile(onnx_feature("custom_names")); + auto expect_add = TensorSpec("tensor(d0[2])").add({{"d0",0}},15).add({{"d0",1}},30); + auto expect_sub = TensorSpec("tensor(d0[2])").add({{"d0",0}},5).add({{"d0",1}},10); + EXPECT_EQ(get(1), expect_add); + EXPECT_EQ(get("onnxModel(custom_names).my_first_output", 1), expect_add); + EXPECT_EQ(get("onnxModel(custom_names).my_second_output", 1), expect_sub); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/features/onnx_feature.cpp b/searchlib/src/vespa/searchlib/features/onnx_feature.cpp index 698d2309e5a..fca8988ba36 100644 --- a/searchlib/src/vespa/searchlib/features/onnx_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/onnx_feature.cpp @@ -2,6 +2,7 @@ #include "onnx_feature.h" #include +#include #include #include #include @@ -85,37 +86,45 @@ OnnxBlueprint::setup(const IIndexEnvironment &env, auto optimize = (env.getFeatureMotivation() == env.FeatureMotivation::VERIFY_SETUP) ? Onnx::Optimize::DISABLE : Onnx::Optimize::ENABLE; - auto file_name = env.getOnnxModelFullPath(params[0].getValue()); - if (!file_name.has_value()) { + auto model_cfg = env.getOnnxModel(params[0].getValue()); + if (!model_cfg) { return fail("no model with name '%s' found", params[0].getValue().c_str()); } try { - _model = std::make_unique(file_name.value(), optimize); + _model = std::make_unique(model_cfg->file_path(), optimize); } catch (std::exception &ex) { return fail("model setup failed: %s", ex.what()); } Onnx::WirePlanner planner; for (size_t i = 0; i < _model->inputs().size(); ++i) { const auto &model_input = _model->inputs()[i]; - vespalib::string input_name = normalize_name(model_input.name, "input"); - if (auto maybe_input = defineInput(fmt("rankingExpression(\"%s\")", input_name.c_str()), AcceptInput::OBJECT)) { + auto input_feature = model_cfg->input_feature(model_input.name); + if (!input_feature.has_value()) { + input_feature = fmt("rankingExpression(\"%s\")", normalize_name(model_input.name, "input").c_str()); + } + if (auto maybe_input = defineInput(input_feature.value(), AcceptInput::OBJECT)) { const FeatureType &feature_input = maybe_input.value(); assert(feature_input.is_object()); if (!planner.bind_input_type(feature_input.type(), model_input)) { - return fail("incompatible type for input '%s': %s -> %s", input_name.c_str(), + return fail("incompatible type for input (%s -> %s): %s -> %s", + input_feature.value().c_str(), model_input.name.c_str(), feature_input.type().to_spec().c_str(), model_input.type_as_string().c_str()); } } } for (size_t i = 0; i < _model->outputs().size(); ++i) { const auto &model_output = _model->outputs()[i]; - vespalib::string output_name = normalize_name(model_output.name, "output"); + auto output_name = model_cfg->output_name(model_output.name); + if (!output_name.has_value()) { + output_name = normalize_name(model_output.name, "output"); + } ValueType output_type = planner.make_output_type(model_output); if (output_type.is_error()) { - return fail("unable to make compatible type for output '%s': %s -> error", - output_name.c_str(), model_output.type_as_string().c_str()); + return fail("unable to make compatible type for output (%s -> %s): %s -> error", + model_output.name.c_str(), output_name.value().c_str(), + model_output.type_as_string().c_str()); } - describeOutput(output_name, "output from onnx model", FeatureType::object(output_type)); + describeOutput(output_name.value(), "output from onnx model", FeatureType::object(output_type)); } _wire_info = planner.get_wire_info(*_model); return true; diff --git a/searchlib/src/vespa/searchlib/fef/CMakeLists.txt b/searchlib/src/vespa/searchlib/fef/CMakeLists.txt index 178de1b8b87..d6f8764cd63 100644 --- a/searchlib/src/vespa/searchlib/fef/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/fef/CMakeLists.txt @@ -4,12 +4,12 @@ vespa_add_library(searchlib_fef OBJECT blueprint.cpp blueprintfactory.cpp blueprintresolver.cpp + feature_resolver.cpp feature_type.cpp featureexecutor.cpp featurenamebuilder.cpp featurenameparser.cpp featureoverrider.cpp - feature_resolver.cpp fef.cpp fieldinfo.cpp fieldpositionsiterator.cpp @@ -19,6 +19,7 @@ vespa_add_library(searchlib_fef OBJECT matchdata.cpp matchdatalayout.cpp objectstore.cpp + onnx_model.cpp parameter.cpp parameterdescriptions.cpp parametervalidator.cpp diff --git a/searchlib/src/vespa/searchlib/fef/iindexenvironment.h b/searchlib/src/vespa/searchlib/fef/iindexenvironment.h index 26e88a98033..384b81643cc 100644 --- a/searchlib/src/vespa/searchlib/fef/iindexenvironment.h +++ b/searchlib/src/vespa/searchlib/fef/iindexenvironment.h @@ -3,7 +3,6 @@ #pragma once #include -#include namespace vespalib::eval { struct ConstantValue; } @@ -12,6 +11,7 @@ namespace search::fef { class Properties; class FieldInfo; class ITableManager; +class OnnxModel; /** * Abstract view of index related information available to the @@ -122,9 +122,9 @@ public: virtual std::unique_ptr getConstantValue(const vespalib::string &name) const = 0; /** - * Get the full path of the file containing the given onnx model + * Get configuration for the given onnx model. **/ - virtual std::optional getOnnxModelFullPath(const vespalib::string &name) const = 0; + virtual const OnnxModel *getOnnxModel(const vespalib::string &name) const = 0; virtual uint32_t getDistributionKey() const = 0; diff --git a/searchlib/src/vespa/searchlib/fef/onnx_model.cpp b/searchlib/src/vespa/searchlib/fef/onnx_model.cpp new file mode 100644 index 00000000000..ba5adaae857 --- /dev/null +++ b/searchlib/src/vespa/searchlib/fef/onnx_model.cpp @@ -0,0 +1,55 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "onnx_model.h" +#include + +namespace search::fef { + +OnnxModel::OnnxModel(const vespalib::string &name_in, + const vespalib::string &file_path_in) + : _name(name_in), + _file_path(file_path_in), + _input_features(), + _output_names() +{ +} + +OnnxModel & +OnnxModel::input_feature(const vespalib::string &model_input_name, const vespalib::string &input_feature) { + _input_features[model_input_name] = input_feature; + return *this; +} + +OnnxModel & +OnnxModel::output_name(const vespalib::string &model_output_name, const vespalib::string &output_name) { + _output_names[model_output_name] = output_name; + return *this; +} + +std::optional +OnnxModel::input_feature(const vespalib::string &model_input_name) const { + auto pos = _input_features.find(model_input_name); + if (pos != _input_features.end()) { + return pos->second; + } + return std::nullopt; +} + +std::optional +OnnxModel::output_name(const vespalib::string &model_output_name) const { + auto pos = _output_names.find(model_output_name); + if (pos != _output_names.end()) { + return pos->second; + } + return std::nullopt; +} + +bool +OnnxModel::operator==(const OnnxModel &rhs) const { + return (std::tie(_name, _file_path, _input_features, _output_names) == + std::tie(rhs._name, rhs._file_path, rhs._input_features, rhs._output_names)); +} + +OnnxModel::~OnnxModel() = default; + +} diff --git a/searchlib/src/vespa/searchlib/fef/onnx_model.h b/searchlib/src/vespa/searchlib/fef/onnx_model.h new file mode 100644 index 00000000000..2195a50600d --- /dev/null +++ b/searchlib/src/vespa/searchlib/fef/onnx_model.h @@ -0,0 +1,39 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include +#include +#include + +namespace search::fef { + +/** + * Class containing configuration for a single onnx model setup. This + * class is used both by the IIndexEnvironment api as well as the + * OnnxModels config adapter in the search core (matching component). + **/ +class OnnxModel { +private: + vespalib::string _name; + vespalib::string _file_path; + std::map _input_features; + std::map _output_names; + +public: + OnnxModel(const vespalib::string &name_in, + const vespalib::string &file_path_in); + ~OnnxModel(); + + const vespalib::string &name() const { return _name; } + const vespalib::string &file_path() const { return _file_path; } + OnnxModel &input_feature(const vespalib::string &model_input_name, const vespalib::string &input_feature); + OnnxModel &output_name(const vespalib::string &model_output_name, const vespalib::string &output_name); + std::optional input_feature(const vespalib::string &model_input_name) const; + std::optional output_name(const vespalib::string &model_output_name) const; + bool operator==(const OnnxModel &rhs) const; + const std::map &inspect_input_features() const { return _input_features; } + const std::map &inspect_output_names() const { return _output_names; } +}; + +} diff --git a/searchlib/src/vespa/searchlib/fef/test/indexenvironment.cpp b/searchlib/src/vespa/searchlib/fef/test/indexenvironment.cpp index 6e2e0b88fbb..d2d336dcdc8 100644 --- a/searchlib/src/vespa/searchlib/fef/test/indexenvironment.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/indexenvironment.cpp @@ -54,20 +54,20 @@ IndexEnvironment::addConstantValue(const vespalib::string &name, (void) insertRes; } -std::optional -IndexEnvironment::getOnnxModelFullPath(const vespalib::string &name) const +const OnnxModel * +IndexEnvironment::getOnnxModel(const vespalib::string &name) const { auto pos = _models.find(name); if (pos != _models.end()) { - return pos->second; + return &pos->second; } - return std::nullopt; + return nullptr; } void -IndexEnvironment::addOnnxModel(const vespalib::string &name, const vespalib::string &path) +IndexEnvironment::addOnnxModel(const OnnxModel &model) { - _models[name] = path; + _models.insert_or_assign(model.name(), model); } diff --git a/searchlib/src/vespa/searchlib/fef/test/indexenvironment.h b/searchlib/src/vespa/searchlib/fef/test/indexenvironment.h index 6602d9f8ee9..0d8d0091921 100644 --- a/searchlib/src/vespa/searchlib/fef/test/indexenvironment.h +++ b/searchlib/src/vespa/searchlib/fef/test/indexenvironment.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -47,7 +48,7 @@ public: }; using ConstantsMap = std::map; - using ModelMap = std::map; + using ModelMap = std::map; IndexEnvironment(); ~IndexEnvironment(); @@ -84,8 +85,8 @@ public: vespalib::eval::ValueType type, std::unique_ptr value); - std::optional getOnnxModelFullPath(const vespalib::string &name) const override; - void addOnnxModel(const vespalib::string &name, const vespalib::string &path); + const OnnxModel *getOnnxModel(const vespalib::string &name) const override; + void addOnnxModel(const OnnxModel &model); private: IndexEnvironment(const IndexEnvironment &); // hide diff --git a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h index 3bbfb0b23f9..dc7be36c290 100644 --- a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h +++ b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h @@ -73,8 +73,8 @@ public: return vespalib::eval::ConstantValue::UP(); } - std::optional getOnnxModelFullPath(const vespalib::string &) const override { - return std::nullopt; + const search::fef::OnnxModel *getOnnxModel(const vespalib::string &) const override { + return nullptr; } bool addField(const vespalib::string & name, bool isAttribute); -- cgit v1.2.3 From 0067c2a02f4c239d964a9ab925438bb4de791583 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Tue, 22 Sep 2020 10:29:57 +0000 Subject: log more details on shutdown, add timeout --- .../com/yahoo/vespa/config/proxy/ProxyServer.java | 30 +++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java index 34510805aca..4635ab4fd86 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java @@ -8,12 +8,19 @@ import com.yahoo.jrt.Transport; import java.util.logging.Level; import com.yahoo.log.LogSetup; import com.yahoo.log.event.Event; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.vespa.config.RawConfig; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import com.yahoo.vespa.config.proxy.filedistribution.FileDistributionAndUrlDownload; import com.yahoo.yolean.system.CatchSignals; import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Logger; @@ -28,6 +35,7 @@ import static com.yahoo.vespa.config.proxy.Mode.ModeName.DEFAULT; */ public class ProxyServer implements Runnable { + private static final DaemonThreadFactory threadFactory = new DaemonThreadFactory("ProxyServer"); private static final int DEFAULT_RPC_PORT = 19090; private static final int JRT_TRANSPORT_THREADS = 4; static final String DEFAULT_PROXY_CONFIG_SOURCES = "tcp/localhost:19070"; @@ -58,9 +66,8 @@ public class ProxyServer implements Runnable { @Override public void run() { if (rpcServer != null) { - Thread t = new Thread(rpcServer); + Thread t = threadFactory.newThread(rpcServer); t.setName("RpcServer"); - t.setDaemon(true); t.start(); } } @@ -125,7 +132,15 @@ public class ProxyServer implements Runnable { } } } - stop(); + ExecutorService executor = Executors.newSingleThreadExecutor(threadFactory); + Callable stopper = () -> { stop(); return "clean shutdown"; }; + Future future = executor.submit(stopper); + try { + String result = future.get(5, TimeUnit.SECONDS); + Event.stopping("configproxy", result); + } catch (Exception e) { + System.exit(1); + } System.exit(0); } @@ -146,9 +161,8 @@ public class ProxyServer implements Runnable { ProxyServer proxyServer = new ProxyServer(new Spec(null, port), configSources, new MemoryCache(), null); // catch termination and interrupt signal proxyServer.setupSignalHandler(); - Thread proxyserverThread = new Thread(proxyServer); + Thread proxyserverThread = threadFactory.newThread(proxyServer); proxyserverThread.setName("configproxy"); - proxyserverThread.setDaemon(true); proxyserverThread.start(); proxyServer.waitForShutdown(); } @@ -174,11 +188,15 @@ public class ProxyServer implements Runnable { } void stop() { - Event.stopping("configproxy", "shutdown"); + Event.stopping("configproxy", "shutdown rpcServer"); if (rpcServer != null) rpcServer.shutdown(); + Event.stopping("configproxy", "cancel configClient"); if (configClient != null) configClient.cancel(); + Event.stopping("configproxy", "flush"); flush(); + Event.stopping("configproxy", "close fileDistribution"); fileDistributionAndUrlDownload.close(); + Event.stopping("configproxy", "stop complete"); } MemoryCache getMemoryCache() { -- cgit v1.2.3 From b35edfe22fe24a44e29719a67cb252d6c000ee03 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 22 Sep 2020 12:48:52 +0200 Subject: Add injectable document access to ApplicationContainerCluster instead --- .../com/yahoo/vespa/model/container/ApplicationContainerCluster.java | 1 + .../src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 1427fa492dc..2f33684c64e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -94,6 +94,7 @@ public final class ApplicationContainerCluster extends ContainerCluster addSimpleComponent(com.yahoo.metrics.simple.jdisc.JdiscMetricsFactory.class.getName(), null, MetricProperties.BUNDLE_SYMBOLIC_NAME); addSimpleComponent("com.yahoo.container.jdisc.state.StateMonitor"); addSimpleComponent("com.yahoo.container.jdisc.ContainerThreadFactory"); - addSimpleComponent(com.yahoo.container.core.documentapi.MessageBusDocumentAccessProvider.class.getName()); addSimpleComponent("com.yahoo.container.handler.VipStatus"); addSimpleComponent(com.yahoo.container.handler.ClustersStatus.class.getName()); addJaxProviders(); -- cgit v1.2.3 From eec0777e3767e83d622070fd44a5565c64952a9e Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 22 Sep 2020 11:16:01 +0000 Subject: Add currentChunk and allign members --- .../src/vespa/searchlib/transactionlog/domain.cpp | 9 ++++++ .../src/vespa/searchlib/transactionlog/domain.h | 35 +++++++++++----------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index ff2d963ccd9..9e0f1a8a1aa 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -29,10 +29,19 @@ using std::runtime_error; using std::make_shared; namespace search::transactionlog { +namespace { + +std::unique_ptr +createCommitChunk(const DomainConfig &cfg) { + return std::make_unique(cfg.getChunkSizeLimit(), cfg.getChunkSizeLimit()/256); +} + +} Domain::Domain(const string &domainName, const string & baseDir, Executor & executor, const DomainConfig & cfg, const FileHeaderContext &fileHeaderContext) : _config(cfg), + _currentChunk(createCommitChunk(cfg)), _lastSerial(0), _singleCommitter(std::make_unique(1, 128 * 1024)), _executor(executor), diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h index 361ac8c1805..7e77e6ef0ef 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h @@ -73,23 +73,24 @@ private: using DomainPartList = std::map; using DurationSeconds = std::chrono::duration; - DomainConfig _config; - SerialNum _lastSerial; - std::unique_ptr _singleCommitter; - Executor &_executor; - std::atomic _sessionId; - vespalib::Monitor _syncMonitor; - bool _pendingSync; - vespalib::string _name; - DomainPartList _parts; - vespalib::Lock _lock; - vespalib::Monitor _currentChunkMonitor; - vespalib::Lock _sessionLock; - SessionList _sessions; - DurationSeconds _maxSessionRunTime; - vespalib::string _baseDir; - const FileHeaderContext &_fileHeaderContext; - bool _markedDeleted; + DomainConfig _config; + std::unique_ptr _currentChunk; + SerialNum _lastSerial; + std::unique_ptr _singleCommitter; + Executor &_executor; + std::atomic _sessionId; + vespalib::Monitor _syncMonitor; + bool _pendingSync; + vespalib::string _name; + DomainPartList _parts; + vespalib::Lock _lock; + vespalib::Monitor _currentChunkMonitor; + vespalib::Lock _sessionLock; + SessionList _sessions; + DurationSeconds _maxSessionRunTime; + vespalib::string _baseDir; + const FileHeaderContext &_fileHeaderContext; + bool _markedDeleted; }; } -- cgit v1.2.3 From 655690784b67f9d7e547d068d3386900df9283ac Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 18 Sep 2020 11:57:00 +0200 Subject: Add switch hostname to node --- .../com/yahoo/vespa/hosted/provision/Node.java | 62 +++++++++++++++------- .../persistence/CuratorDatabaseClient.java | 3 +- .../provision/persistence/NodeSerializer.java | 10 +++- .../DynamicProvisioningMaintainerTest.java | 3 +- .../provision/persistence/NodeSerializerTest.java | 23 +++++--- .../provisioning/AllocationSimulator.java | 3 +- .../provisioning/DynamicDockerAllocationTest.java | 2 +- .../provision/provisioning/NodeCandidateTest.java | 6 +-- 8 files changed, 77 insertions(+), 35 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index b08dc6bbaf2..9b86c57df55 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -43,6 +43,7 @@ public final class Node { private final Reports reports; private final Optional modelName; private final Optional reservedTo; + private final Optional switchHostname; /** Record of the last event of each type happening to this node */ private final History history; @@ -54,20 +55,21 @@ public final class Node { public static Node createDockerNode(Set ipAddresses, String hostname, String parentHostname, NodeResources resources, NodeType type) { return new Node("fake-" + hostname, new IP.Config(ipAddresses, Set.of()), hostname, Optional.of(parentHostname), new Flavor(resources), Status.initial(), State.reserved, - Optional.empty(), History.empty(), type, new Reports(), Optional.empty(), Optional.empty()); + Optional.empty(), History.empty(), type, new Reports(), Optional.empty(), Optional.empty(), + Optional.empty()); } /** Creates a node in the initial state (provisioned) */ public static Node create(String openStackId, IP.Config ipConfig, String hostname, Optional parentHostname, Optional modelName, Flavor flavor, Optional reservedTo, NodeType type) { return new Node(openStackId, ipConfig, hostname, parentHostname, flavor, Status.initial(), State.provisioned, - Optional.empty(), History.empty(), type, new Reports(), modelName, reservedTo); + Optional.empty(), History.empty(), type, new Reports(), modelName, reservedTo, Optional.empty()); } /** Creates a node. See also the {@code create} helper methods. */ public Node(String id, IP.Config ipConfig, String hostname, Optional parentHostname, Flavor flavor, Status status, State state, Optional allocation, History history, NodeType type, - Reports reports, Optional modelName, Optional reservedTo) { + Reports reports, Optional modelName, Optional reservedTo, Optional switchHostname) { this.id = Objects.requireNonNull(id, "A node must have an ID"); this.hostname = requireNonEmptyString(hostname, "A node must have a hostname"); this.ipConfig = Objects.requireNonNull(ipConfig, "A node must a have an IP config"); @@ -81,6 +83,7 @@ public final class Node { this.reports = Objects.requireNonNull(reports, "A null reports is not permitted"); this.modelName = Objects.requireNonNull(modelName, "A null modelName is not permitted"); this.reservedTo = Objects.requireNonNull(reservedTo, "reservedTo cannot be null"); + this.switchHostname = requireNonEmptyString(switchHostname, "switchHostname cannot be null"); if (state == State.active) requireNonEmpty(ipConfig.primary(), "Active node " + hostname + " must have at least one valid IP address"); @@ -88,6 +91,7 @@ public final class Node { if (parentHostname.isPresent()) { if (!ipConfig.pool().asSet().isEmpty()) throw new IllegalArgumentException("A child node cannot have an IP address pool"); if (modelName.isPresent()) throw new IllegalArgumentException("A child node cannot have model name set"); + if (switchHostname.isPresent()) throw new IllegalArgumentException("A child node cannot have switch hostname set"); } if (type != NodeType.host && reservedTo.isPresent()) @@ -151,7 +155,7 @@ public final class Node { /** Returns all the reports on this node. */ public Reports reports() { return reports; } - /** Returns the hardware model of this node */ + /** Returns the hardware model of this node, if any */ public Optional modelName() { return modelName; } /** @@ -160,6 +164,11 @@ public final class Node { */ public Optional reservedTo() { return reservedTo; } + /** Returns the hostname of the switch this node is connected to, if any */ + public Optional switchHostname() { + return switchHostname; + } + /** * Returns a copy of this node with wantToRetire and wantToDeprovision set to the given values and updated history. * @@ -217,42 +226,44 @@ public final class Node { /** Returns a node with the status assigned to the given value */ public Node with(Status status) { - return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, + reports, modelName, reservedTo, switchHostname); } /** Returns a node with the type assigned to the given value */ public Node with(NodeType type) { - return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, + reports, modelName, reservedTo, switchHostname); } /** Returns a node with the flavor assigned to the given value */ public Node with(Flavor flavor) { - return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, + reports, modelName, reservedTo, switchHostname); } /** Returns a copy of this with the reboot generation set to generation */ public Node withReboot(Generation generation) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status.withReboot(generation), state, - allocation, history, type, reports, modelName, reservedTo); + allocation, history, type, reports, modelName, reservedTo, switchHostname); } /** Returns a copy of this with the openStackId set */ public Node withOpenStackId(String openStackId) { return new Node(openStackId, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo); + allocation, history, type, reports, modelName, reservedTo, switchHostname); } /** Returns a copy of this with model name set to given value */ public Node withModelName(String modelName) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, Optional.of(modelName), reservedTo); + allocation, history, type, reports, Optional.of(modelName), reservedTo, switchHostname); } /** Returns a copy of this with model name cleared */ public Node withoutModelName() { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, Optional.empty(), reservedTo); + allocation, history, type, reports, Optional.empty(), reservedTo, switchHostname); } /** Returns a copy of this with a history record saying it was detected to be down at this instant */ @@ -278,39 +289,50 @@ public final class Node { */ public Node with(Allocation allocation) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - Optional.of(allocation), history, type, reports, modelName, reservedTo); + Optional.of(allocation), history, type, reports, modelName, reservedTo, switchHostname); } /** Returns a new Node without an allocation. */ public Node withoutAllocation() { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - Optional.empty(), history, type, reports, modelName, reservedTo); + Optional.empty(), history, type, reports, modelName, reservedTo, switchHostname); } /** Returns a copy of this node with IP config set to the given value. */ public Node with(IP.Config ipConfig) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo); + allocation, history, type, reports, modelName, reservedTo, switchHostname); } /** Returns a copy of this node with the parent hostname assigned to the given value. */ public Node withParentHostname(String parentHostname) { return new Node(id, ipConfig, hostname, Optional.of(parentHostname), flavor, status, state, - allocation, history, type, reports, modelName, reservedTo); + allocation, history, type, reports, modelName, reservedTo, switchHostname); } public Node withReservedTo(TenantName tenant) { if (type != NodeType.host) throw new IllegalArgumentException("Only host nodes can be reserved, " + hostname + " has type " + type); return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, Optional.of(tenant)); + allocation, history, type, reports, modelName, Optional.of(tenant), switchHostname); } /** Returns a copy of this node which is not reserved to a tenant */ public Node withoutReservedTo() { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, Optional.empty()); + allocation, history, type, reports, modelName, Optional.empty(), switchHostname); + } + + /** Returns a copy of this node with switch hostname set to given value */ + public Node withSwitchHostname(String switchHostname) { + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, + allocation, history, type, reports, modelName, reservedTo, Optional.ofNullable(switchHostname)); + } + + /** Returns a copy of this node with switch hostname unset */ + public Node withoutSwitchHostname() { + return withSwitchHostname(null); } /** Returns a copy of this node with the current reboot generation set to the given number at the given instant */ @@ -343,12 +365,12 @@ public final class Node { /** Returns a copy of this node with the given history. */ public Node with(History history) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo); + allocation, history, type, reports, modelName, reservedTo, switchHostname); } public Node with(Reports reports) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo); + allocation, history, type, reports, modelName, reservedTo, switchHostname); } private static Optional requireNonEmptyString(Optional value, String message) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index cc62ae67e84..504854108b6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -216,7 +216,8 @@ public class CuratorDatabaseClient { toState, toState.isAllocated() ? node.allocation() : Optional.empty(), node.history().recordStateTransition(node.state(), toState, agent, clock.instant()), - node.type(), node.reports(), node.modelName(), node.reservedTo()); + node.type(), node.reports(), node.modelName(), node.reservedTo(), + node.switchHostname()); writeNode(toState, curatorTransaction, node, newNode); writtenNodes.add(newNode); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 5a3584b6ff4..0ce81e49bda 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -79,6 +79,7 @@ public class NodeSerializer { private static final String reportsKey = "reports"; private static final String modelNameKey = "modelName"; private static final String reservedToKey = "reservedTo"; + private static final String switchHostnameKey = "switchHostname"; // Node resource fields private static final String flavorKey = "flavor"; @@ -143,6 +144,7 @@ public class NodeSerializer { node.status().osVersion().current().ifPresent(version -> object.setString(osVersionKey, version.toString())); node.status().osVersion().wanted().ifPresent(version -> object.setString(wantedOsVersionKey, version.toFullString())); node.status().firmwareVerifiedAt().ifPresent(instant -> object.setLong(firmwareCheckKey, instant.toEpochMilli())); + node.switchHostname().ifPresent(switchHostname -> object.setString(switchHostnameKey, switchHostname)); node.reports().toSlime(object, reportsKey); node.modelName().ifPresent(modelName -> object.setString(modelNameKey, modelName)); node.reservedTo().ifPresent(tenant -> object.setString(reservedToKey, tenant.value())); @@ -212,7 +214,8 @@ public class NodeSerializer { nodeTypeFromString(object.field(nodeTypeKey).asString()), Reports.fromSlime(object.field(reportsKey)), modelNameFromSlime(object), - reservedToFromSlime(object.field(reservedToKey))); + reservedToFromSlime(object.field(reservedToKey)), + switchHostnameFromSlime(object.field(switchHostnameKey))); } private Status statusFromSlime(Inspector object) { @@ -227,6 +230,11 @@ public class NodeSerializer { instantFromSlime(object.field(firmwareCheckKey))); } + private Optional switchHostnameFromSlime(Inspector field) { + if (!field.valid()) return Optional.empty(); + return Optional.of(field.asString()); + } + private Flavor flavorFromSlime(Inspector object) { Inspector resources = object.field(resourcesKey); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index 12cf114b2d2..9978f37e835 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -273,7 +273,8 @@ public class DynamicProvisioningMaintainerTest { false)); var ipConfig = new IP.Config(state == Node.State.active ? Set.of("::1") : Set.of(), Set.of()); return new Node("fake-id-" + hostname, ipConfig, hostname, parentHostname, flavor, Status.initial(), - state, allocation, History.empty(), nodeType, new Reports(), Optional.empty(), Optional.empty()); + state, allocation, History.empty(), nodeType, new Reports(), Optional.empty(), Optional.empty(), + Optional.empty()); } private long provisionedHostsMatching(NodeResources resources) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java index dbbad0b8982..e9910157592 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java @@ -57,7 +57,7 @@ public class NodeSerializerTest { private final ManualClock clock = new ManualClock(); @Test - public void testProvisionedNodeSerialization() { + public void provisioned_node_serialization() { Node node = createNode(); Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node)); @@ -69,7 +69,7 @@ public class NodeSerializerTest { } @Test - public void testReservedNodeSerialization() { + public void reserved_node_serialization() { Node node = createNode(); NodeResources requestedResources = new NodeResources(1.2, 3.4, 5.6, 7.8, NodeResources.DiskSpeed.any); @@ -110,7 +110,7 @@ public class NodeSerializerTest { } @Test - public void testRebootAndRestartAndTypeNoCurrentValuesSerialization() { + public void reboot_and_restart_and_type_no_current_values_serialization() { String nodeData = "{\n" + " \"type\" : \"tenant\",\n" + @@ -158,7 +158,7 @@ public class NodeSerializerTest { } @Test - public void testRetiredNodeSerialization() { + public void retired_node_serialization() { Node node = createNode(); clock.advance(Duration.ofMinutes(3)); @@ -185,7 +185,7 @@ public class NodeSerializerTest { } @Test - public void testAssimilatedDeserialization() { + public void assimilated_node_deserialization() { Node node = nodeSerializer.fromJson(Node.State.active, ("{\n" + " \"type\": \"tenant\",\n" + " \"hostname\": \"assimilate2.vespahosted.yahoo.tld\",\n" + @@ -211,7 +211,7 @@ public class NodeSerializerTest { } @Test - public void testSetFailCount() { + public void fail_count() { Node node = createNode(); node = node.allocate(ApplicationId.from(TenantName.from("myTenant"), ApplicationName.from("myApplication"), @@ -392,7 +392,7 @@ public class NodeSerializerTest { } @Test - public void testNodeWithNetworkPorts() { + public void network_ports_serialization() { Node node = createNode(); List list = new ArrayList<>(); list.add(new NetworkPorts.Allocation(8080, "container", "default/0", "http")); @@ -416,6 +416,15 @@ public class NodeSerializerTest { assertEquals(list, listCopy); } + @Test + public void switch_hostname_serialization() { + Node node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(createNode())); + assertFalse(node.switchHostname().isPresent()); + String switchHostname = "switch0.example.com"; + node = node.withSwitchHostname(switchHostname); + node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(node)); + assertEquals(switchHostname, node.switchHostname().get()); + } private byte[] createNodeJson(String hostname, String... ipAddress) { String ipAddressJsonPart = ""; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java index b113c281289..051420d694d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java @@ -80,7 +80,8 @@ public class AllocationSimulator { var ipConfig = new IP.Config(Set.of("127.0.0.1"), parent.isPresent() ? Set.of() : getAdditionalIP()); return new Node("fake", ipConfig, hostname, parent, flavor, Status.initial(), parent.isPresent() ? Node.State.ready : Node.State.active, allocation(tenant, flavor), History.empty(), - parent.isPresent() ? NodeType.tenant : NodeType.host, new Reports(), Optional.empty(), Optional.empty()); + parent.isPresent() ? NodeType.tenant : NodeType.host, new Reports(), Optional.empty(), Optional.empty(), + Optional.empty()); } private Set getAdditionalIP() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java index ab62972df30..7e416bfc397 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java @@ -425,7 +425,7 @@ public class DynamicDockerAllocationTest { private void addAndAssignNode(ApplicationId id, String hostname, String parentHostname, ClusterSpec clusterSpec, NodeResources flavor, int index, ProvisioningTester tester) { Node node1a = Node.create("open1", new IP.Config(Set.of("127.0.233." + index), Set.of()), hostname, - Optional.of(parentHostname), Optional.empty(), new Flavor(flavor), Optional.empty(), NodeType.tenant + Optional.of(parentHostname), Optional.empty(), new Flavor(flavor), Optional.empty(), NodeType.tenant, Optional.empty() ); ClusterMembership clusterMembership1 = ClusterMembership.from( clusterSpec.with(Optional.of(ClusterSpec.Group.from(0))), index); // Need to add group here so that group is serialized in node allocation diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java index 95b9f334bb4..ec001556f58 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java @@ -127,7 +127,7 @@ public class NodeCandidateTest { return new Node(hostname, new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.empty(), new Flavor(new NodeResources(2, 2, 2, 2)), Status.initial(), state, Optional.empty(), History.empty(), NodeType.tenant, new Reports(), - Optional.empty(), Optional.empty()); + Optional.empty(), Optional.empty(), Optional.empty()); } private static NodeCandidate node(String hostname, @@ -137,11 +137,11 @@ public class NodeCandidateTest { Node node = new Node(hostname, new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.of(hostname + "parent"), new Flavor(nodeResources), Status.initial(), Node.State.ready, Optional.empty(), History.empty(), NodeType.tenant, - new Reports(), Optional.empty(), Optional.empty()); + new Reports(), Optional.empty(), Optional.empty(), Optional.empty()); Node parent = new Node(hostname + "parent", new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.empty(), new Flavor(totalHostResources), Status.initial(), Node.State.ready, Optional.empty(), History.empty(), NodeType.host, - new Reports(), Optional.empty(), Optional.empty()); + new Reports(), Optional.empty(), Optional.empty(), Optional.empty()); return new NodeCandidate(node, totalHostResources.subtract(allocatedHostResources), Optional.of(parent), false, false, true, false); } -- cgit v1.2.3 From 48b32b454f9c72029dee804ce76aa4ff66ef09a4 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 18 Sep 2020 12:16:46 +0200 Subject: Add switch hostname to REST API --- .../IdentityDocumentGeneratorTest.java | 2 +- .../InstanceValidatorTest.java | 2 +- .../com/yahoo/vespa/hosted/provision/Node.java | 4 +-- .../vespa/hosted/provision/NodeRepository.java | 2 +- .../provision/provisioning/ProvisionedHost.java | 2 +- .../hosted/provision/restapi/NodePatcher.java | 2 ++ .../hosted/provision/restapi/NodesResponse.java | 1 + .../provision/restapi/NodesV2ApiHandler.java | 9 ++++-- .../provision/maintenance/MetricsReporterTest.java | 2 +- .../yahoo/vespa/hosted/provision/node/IPTest.java | 2 +- .../provision/persistence/NodeSerializerTest.java | 6 ++-- .../provision/provisioning/HostCapacityTest.java | 8 ++--- .../hosted/provision/restapi/NodesV2ApiTest.java | 37 ++++++++++++++++++---- 13 files changed, 56 insertions(+), 23 deletions(-) diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java index 8e5622e6c2f..f92f02f7908 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java @@ -63,7 +63,7 @@ public class IdentityDocumentGeneratorTest { Optional.empty(), new MockNodeFlavors().getFlavorOrThrow("default"), Optional.empty(), - NodeType.host); + NodeType.host, Optional.empty()); Node containerNode = Node.createDockerNode(Set.of("::1"), containerHostname, parentHostname, diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java index ab81cb8eda5..5e965761874 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java @@ -231,7 +231,7 @@ public class InstanceValidatorTest { Optional.empty(), flavors.getFlavorOrThrow("default"), Optional.empty(), - NodeType.tenant); + NodeType.tenant, Optional.empty()); nodeList.add(node); } return nodeList; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 9b86c57df55..3b3711d1081 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -61,9 +61,9 @@ public final class Node { /** Creates a node in the initial state (provisioned) */ public static Node create(String openStackId, IP.Config ipConfig, String hostname, Optional parentHostname, - Optional modelName, Flavor flavor, Optional reservedTo, NodeType type) { + Optional modelName, Flavor flavor, Optional reservedTo, NodeType type, Optional switchHostname) { return new Node(openStackId, ipConfig, hostname, parentHostname, flavor, Status.initial(), State.provisioned, - Optional.empty(), History.empty(), type, new Reports(), modelName, reservedTo, Optional.empty()); + Optional.empty(), History.empty(), type, new Reports(), modelName, reservedTo, switchHostname); } /** Creates a node. See also the {@code create} helper methods. */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 4ec7ddd04c4..8523db7a970 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -407,7 +407,7 @@ public class NodeRepository extends AbstractComponent { Flavor flavor, Optional reservedTo, NodeType type) { if (ipConfig.primary().isEmpty()) // TODO: Remove this. Only test code hits this path ipConfig = ipConfig.with(nameResolver.getAllByNameOrThrow(hostname)); - return Node.create(openStackId, ipConfig, hostname, parentHostname, Optional.empty(), flavor, reservedTo, type); + return Node.create(openStackId, ipConfig, hostname, parentHostname, Optional.empty(), flavor, reservedTo, type, Optional.empty()); } public Node createNode(String openStackId, String hostname, Optional parentHostname, Flavor flavor, NodeType type) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java index 151fcb4233f..298404a4cb4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java @@ -38,7 +38,7 @@ public class ProvisionedHost { /** Generate {@link Node} instance representing the provisioned physical host */ public Node generateHost() { - var node = Node.create(id, IP.Config.EMPTY, hostHostname, Optional.empty(), Optional.empty(), hostFlavor, Optional.empty(), NodeType.host); + var node = Node.create(id, IP.Config.EMPTY, hostHostname, Optional.empty(), Optional.empty(), hostFlavor, Optional.empty(), NodeType.host, Optional.empty()); return node.with(node.status().withOsVersion(OsVersion.EMPTY.withCurrent(Optional.of(osVersion)))); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index 19276a81ef8..834bed00bd0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -149,6 +149,8 @@ public class NodePatcher { return patchRequiredDiskSpeed(node, asString(value)); case "reservedTo": return value.type() == Type.NIX ? node.withoutReservedTo() : node.withReservedTo(TenantName.from(value.asString())); + case "switchHostname": + return value.type() == Type.NIX ? node.withoutSwitchHostname() : node.withSwitchHostname(value.asString()); default : throw new IllegalArgumentException("Could not apply field '" + name + "' on a node: No such modifiable field"); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index 8e3ae6358df..0137bee5fbd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -189,6 +189,7 @@ class NodesResponse extends HttpResponse { ipAddressesToSlime(node.ipConfig().pool().asSet(), object.setArray("additionalIpAddresses")); node.reports().toSlime(object, "reports"); node.modelName().ifPresent(modelName -> object.setString("modelName", modelName)); + node.switchHostname().ifPresent(switchHostname -> object.setString("switchHostname", switchHostname)); } private void toSlime(ApplicationId id, Cursor object) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index 5080dafe2a5..1891b9d2f82 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -261,8 +261,13 @@ public class NodesV2ApiHandler extends LoggingRequestHandler { modelName, flavorFromSlime(inspector), reservedToFromSlime(inspector.field("reservedTo")), - nodeTypeFromSlime(inspector.field("type")) - ); + nodeTypeFromSlime(inspector.field("type")), + switchHostnameFromSlime(inspector.field("switchHostname"))); + } + + private Optional switchHostnameFromSlime(Inspector field) { + if (!field.valid()) return Optional.empty(); + return Optional.of(field.asString()); } private Flavor flavorFromSlime(Inspector inspector) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 20c9d24d1b6..6897a293525 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -163,7 +163,7 @@ public class MetricsReporterTest { Set ipAddressPool = Set.of("::2", "::3", "::4", "::5"); Node dockerHost = Node.create("openStackId1", new IP.Config(Set.of("::1"), ipAddressPool), "dockerHost", - Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host); + Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host, Optional.empty()); nodeRepository.addNodes(List.of(dockerHost), Agent.system); nodeRepository.dirtyRecursively("dockerHost", Agent.system, getClass().getSimpleName()); nodeRepository.setReady("dockerHost", Agent.system, getClass().getSimpleName()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java index fcf2ab5a52d..4602a8f3560 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java @@ -179,7 +179,7 @@ public class IPTest { private static Node createNode(Set ipAddresses) { return Node.create("id1", new IP.Config(Set.of("127.0.0.1"), ipAddresses), "host1", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), - Optional.empty(), NodeType.host); + Optional.empty(), NodeType.host, Optional.empty()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java index e9910157592..97c9bef0dd0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java @@ -229,7 +229,7 @@ public class NodeSerializerTest { @Test public void serialize_parentHostname() { final String parentHostname = "parent.yahoo.com"; - Node node = Node.create("myId", new IP.Config(Set.of("127.0.0.1"), Set.of()), "myHostname", Optional.of(parentHostname), Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), Optional.empty(), NodeType.tenant); + Node node = Node.create("myId", new IP.Config(Set.of("127.0.0.1"), Set.of()), "myHostname", Optional.of(parentHostname), Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), Optional.empty(), NodeType.tenant, Optional.empty()); Node deserializedNode = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(node)); assertEquals(parentHostname, deserializedNode.parentHostname().get()); @@ -450,8 +450,8 @@ public class NodeSerializerTest { Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), - Optional.empty(), NodeType.tenant - ); + Optional.empty(), NodeType.tenant, + Optional.empty()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java index da78aff493e..f831f2d501b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java @@ -44,9 +44,9 @@ public class HostCapacityTest { NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("host", "docker", "docker2"); // Create three docker hosts - host1 = Node.create("host1", new IP.Config(Set.of("::1"), generateIPs(2, 4)), "host1", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host); - host2 = Node.create("host2", new IP.Config(Set.of("::11"), generateIPs(12, 3)), "host2", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host); - host3 = Node.create("host3", new IP.Config(Set.of("::21"), generateIPs(22, 1)), "host3", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host); + host1 = Node.create("host1", new IP.Config(Set.of("::1"), generateIPs(2, 4)), "host1", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host, Optional.empty()); + host2 = Node.create("host2", new IP.Config(Set.of("::11"), generateIPs(12, 3)), "host2", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host, Optional.empty()); + host3 = Node.create("host3", new IP.Config(Set.of("::21"), generateIPs(22, 1)), "host3", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host, Optional.empty()); // Add two containers to host1 var nodeA = Node.createDockerNode(Set.of("::2"), "nodeA", "host1", resources1, NodeType.tenant); @@ -111,7 +111,7 @@ public class HostCapacityTest { // Dev host can assign both configserver and tenant containers. var nodeFlavors = FlavorConfigBuilder.createDummies("devhost", "container"); - var devHost = Node.create("devhost", new IP.Config(Set.of("::1"), generateIPs(2, 10)), "devhost", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("devhost"), Optional.empty(), NodeType.devhost); + var devHost = Node.create("devhost", new IP.Config(Set.of("::1"), generateIPs(2, 10)), "devhost", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("devhost"), Optional.empty(), NodeType.devhost, Optional.empty()); var cfg = Node.createDockerNode(Set.of("::2"), "cfg", "devhost", resources1, NodeType.config); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index 2cd2fe1fc28..1a354686ae4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -333,7 +333,7 @@ public class NodesV2ApiTest { // Node types running a single container can share their IP address with child node tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asNodeJson("cfghost42.yahoo.com", NodeType.confighost, "default", Optional.empty(), "127.0.42.1") + "]", + "[" + asNodeJson("cfghost42.yahoo.com", NodeType.confighost, "default", Optional.empty(), Optional.empty(), "127.0.42.1") + "]", Request.Method.POST), 200, "{\"message\":\"Added 1 nodes to the provisioned state\"}"); tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", @@ -349,7 +349,7 @@ public class NodesV2ApiTest { // ... nor with child node on different host tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asNodeJson("cfghost43.yahoo.com", NodeType.confighost, "default", Optional.empty(), "127.0.43.1") + "]", + "[" + asNodeJson("cfghost43.yahoo.com", NodeType.confighost, "default", Optional.empty(), Optional.empty(), "127.0.43.1") + "]", Request.Method.POST), 200, "{\"message\":\"Added 1 nodes to the provisioned state\"}"); tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/cfg42.yahoo.com", @@ -939,6 +939,30 @@ public class NodesV2ApiTest { "\"resources\":{\"vcpu\":56.0,\"memoryGb\":34.0,\"diskGb\":12.0,\"bandwidthGbps\":78.0,\"diskSpeed\":\"fast\",\"storageType\":\"remote\"}"); } + @Test + public void test_node_switch_hostname() throws Exception { + String hostname = "host42.yahoo.com"; + // Add host with switch hostname + String json = asNodeJson(hostname, NodeType.host, "default", Optional.empty(), Optional.of("switch0"), "127.0.42.1", "::42:1"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[" + json + "]").getBytes(StandardCharsets.UTF_8), + Request.Method.POST), + "{\"message\":\"Added 1 nodes to the provisioned state\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + hostname), "\"switchHostname\":\"switch0\""); + + // Update switch hostname + json = "{\"switchHostname\":\"switch1\"}"; + assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + hostname, json.getBytes(StandardCharsets.UTF_8), Request.Method.PATCH), + "{\"message\":\"Updated host42.yahoo.com\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + hostname), "\"switchHostname\":\"switch1\""); + + // Clear switch hostname + json = "{\"switchHostname\":null}"; + assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + hostname, json.getBytes(StandardCharsets.UTF_8), Request.Method.PATCH), + "{\"message\":\"Updated host42.yahoo.com\"}"); + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/" + hostname), "switchHostname", false); + } + private static String asDockerNodeJson(String hostname, String parentHostname, String... ipAddress) { return asDockerNodeJson(hostname, NodeType.tenant, parentHostname, ipAddress); } @@ -958,14 +982,15 @@ public class NodesV2ApiTest { } private static String asHostJson(String hostname, String flavor, Optional reservedTo, String... ipAddress) { - return asNodeJson(hostname, NodeType.host, flavor, reservedTo, ipAddress); + return asNodeJson(hostname, NodeType.host, flavor, reservedTo, Optional.empty(), ipAddress); } - private static String asNodeJson(String hostname, NodeType nodeType, String flavor, Optional reservedTo, String... ipAddress) { + private static String asNodeJson(String hostname, NodeType nodeType, String flavor, Optional reservedTo, Optional switchHostname, String... ipAddress) { return "{\"hostname\":\"" + hostname + "\", \"openStackId\":\"" + hostname + "\"," + createIpAddresses(ipAddress) + "\"flavor\":\"" + flavor + "\"" + (reservedTo.isPresent() ? ", \"reservedTo\":\"" + reservedTo.get().value() + "\"" : "") + + (switchHostname.isPresent() ? ", \"switchHostname\":\"" + switchHostname.get() + "\"" : "") + ", \"type\":\"" + nodeType + "\"}"; } @@ -989,8 +1014,8 @@ public class NodesV2ApiTest { tester.assertFile(request, file); } - private void assertResponse(Request request, String file) throws IOException { - tester.assertResponse(request, file); + private void assertResponse(Request request, String response) throws IOException { + tester.assertResponse(request, response); } } -- cgit v1.2.3 From 14c1f947b2906e8cfa38d98ec48ea44bf5fe9e22 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 18 Sep 2020 13:18:18 +0200 Subject: Unify node patch methods in tests --- .../hosted/provision/autoscale/AutoscalingTester.java | 2 +- .../maintenance/InactiveAndFailedExpirerTest.java | 6 ++---- .../provision/maintenance/OsUpgradeActivatorTest.java | 8 +------- .../vespa/hosted/provision/os/OsVersionsTest.java | 17 ++++------------- .../provisioning/DynamicDockerProvisionTest.java | 3 +-- .../provision/provisioning/ProvisioningTest.java | 6 +++--- .../provision/provisioning/ProvisioningTester.java | 18 +++++++++++++++++- .../provisioning/VirtualNodeProvisioningTest.java | 14 +++++--------- 8 files changed, 34 insertions(+), 40 deletions(-) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index 19911076e69..cc755c01405 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -93,7 +93,7 @@ class AutoscalingTester { public void makeReady(String hostname) { Node node = nodeRepository().getNode(hostname).get(); - nodeRepository().write(node.with(new IP.Config(Set.of("::" + 0 + ":0"), Set.of())), nodeRepository().lock(node)); + provisioningTester.patchNode(node, (n) -> n.with(new IP.Config(Set.of("::" + 0 + ":0"), Set.of()))); Node host = nodeRepository().getNode(node.parentHostname().get()).get(); host = host.with(new IP.Config(Set.of("::" + 0 + ":0"), Set.of("::" + 0 + ":2"))); if (host.state() == Node.State.provisioned) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java index 3d17cbf0217..056fe041377 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java @@ -133,7 +133,7 @@ public class InactiveAndFailedExpirerTest { // Flag one node for retirement and redeploy { Node toRetire = tester.getNodes(applicationId, Node.State.active).asList().get(0); - tester.patchNode(toRetire.withWantToRetire(true, Agent.operator, tester.clock().instant())); + tester.patchNode(toRetire, (node) -> node.withWantToRetire(true, Agent.operator, tester.clock().instant())); List hostSpecs = tester.prepare(applicationId, cluster, Capacity.from(new ClusterResources(2, 1, nodeResources))); tester.activate(applicationId, new HashSet<>(hostSpecs)); } @@ -203,9 +203,7 @@ public class InactiveAndFailedExpirerTest { assertEquals(2, inactiveNodes.size()); // Nodes marked for deprovisioning are moved to parked - tester.nodeRepository().write(inactiveNodes.stream() - .map(node -> node.withWantToRetire(true, true, Agent.system, tester.clock().instant())) - .collect(Collectors.toList()), () -> {}); + tester.patchNodes(inactiveNodes, (node) -> node.withWantToRetire(true, true, Agent.system, tester.clock().instant())); tester.advanceTime(Duration.ofMinutes(11)); new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10), new TestMetric()).run(); assertEquals(2, tester.nodeRepository().getNodes(Node.State.parked).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivatorTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivatorTest.java index 218812f9a3d..f795dbaaa1c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivatorTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivatorTest.java @@ -97,13 +97,7 @@ public class OsUpgradeActivatorTest { } private void completeUpgradeOf(List nodes) { - for (var node : nodes) { - try (var lock = tester.nodeRepository().lock(node)) { - node = tester.nodeRepository().getNode(node.hostname()).get(); - node = node.with(node.status().withVespaVersion(node.allocation().get().membership().cluster().vespaVersion())); - tester.nodeRepository().write(node, lock); - } - } + tester.patchNodes(nodes, (node) -> node.with(node.status().withVespaVersion(node.allocation().get().membership().cluster().vespaVersion()))); } private Stream streamUpdatedNodes(List nodes) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java index 6a41e766ace..715ecdb5949 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java @@ -21,7 +21,6 @@ import java.util.List; import java.util.Optional; import java.util.function.Function; import java.util.function.Supplier; -import java.util.function.UnaryOperator; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; @@ -273,20 +272,11 @@ public class OsVersionsTest { } private void setWantedVersion(List nodes, Version wantedVersion) { - writeNode(nodes, node -> node.with(node.status().withOsVersion(node.status().osVersion().withWanted(Optional.of(wantedVersion))))); + tester.patchNodes(nodes, node -> node.with(node.status().withOsVersion(node.status().osVersion().withWanted(Optional.of(wantedVersion))))); } private void setCurrentVersion(List nodes, Version currentVersion) { - writeNode(nodes, node -> node.with(node.status().withOsVersion(node.status().osVersion().withCurrent(Optional.of(currentVersion))))); - } - - private void writeNode(List nodes, UnaryOperator updateFunc) { - for (var node : nodes) { - try (var lock = tester.nodeRepository().lock(node)) { - node = tester.nodeRepository().getNode(node.hostname()).get(); - tester.nodeRepository().write(updateFunc.apply(node), lock); - } - } + tester.patchNodes(nodes, node -> node.with(node.status().withOsVersion(node.status().osVersion().withCurrent(Optional.of(currentVersion))))); } private void completeUpgradeOf(List nodes) { @@ -294,7 +284,8 @@ public class OsVersionsTest { } private void completeUpgradeOf(List nodes, NodeType nodeType) { - writeNode(nodes, (node) -> { + // Complete upgrade by deprovisioning stale hosts and provisioning new ones + tester.patchNodes(nodes, (node) -> { Optional wantedOsVersion = node.status().osVersion().wanted(); if (node.status().wantToDeprovision()) { // Complete upgrade by deprovisioning stale hosts and provisioning new ones diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java index 411283abf33..57c7c46c2d9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java @@ -128,8 +128,7 @@ public class DynamicDockerProvisionTest { NodeResources resources = new NodeResources(10, 10, 10, 10); ApplicationId app = tester.makeApplicationId(); - Function retireNode = node -> - tester.nodeRepository().write(node.withWantToRetire(true, Agent.system, Instant.now()), () -> {}); + Function retireNode = node -> tester.patchNode(node, (n) -> n.withWantToRetire(true, Agent.system, Instant.now())); Function getNodeInGroup = group -> tester.nodeRepository().getNodes(app).stream() .filter(node -> node.allocation().get().membership().cluster().group().get().index() == group) .findAny().orElseThrow(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index d53a0732c89..6cf5a2c8342 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -673,7 +673,7 @@ public class ProvisioningTest { ApplicationId application = tester.makeApplicationId(); // Flag all nodes for retirement List readyNodes = tester.makeReadyNodes(5, defaultResources); - readyNodes.forEach(node -> tester.patchNode(node.withWantToRetire(true, Agent.system, tester.clock().instant()))); + tester.patchNodes(readyNodes, (node) -> node.withWantToRetire(true, Agent.system, tester.clock().instant())); try { prepare(application, 2, 0, 2, 0, defaultResources, tester); @@ -701,7 +701,7 @@ public class ProvisioningTest { assertEquals(0, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).retired().size()); // Mark the nodes as want to retire - tester.nodeRepository().getNodes(application, Node.State.active).forEach(node -> tester.patchNode(node.withWantToRetire(true, Agent.system, tester.clock().instant()))); + tester.nodeRepository().getNodes(application, Node.State.active).forEach(node -> tester.patchNode(node, (n) -> n.withWantToRetire(true, Agent.system, tester.clock().instant()))); // redeploy without allow failing tester.activate(application, tester.prepare(application, cluster, capacityFORCED)); @@ -786,7 +786,7 @@ public class ProvisioningTest { // Retire some nodes and redeploy { List nodesToRetire = tester.getNodes(application, Node.State.active).asList().subList(0, 2); - nodesToRetire.forEach(node -> tester.patchNode(node.withWantToRetire(true, Agent.system, tester.clock().instant()))); + tester.patchNodes(nodesToRetire, (node) -> node.withWantToRetire(true, Agent.system, tester.clock().instant())); SystemState state = prepare(application, 2, 0, 2, 0, defaultResources, tester); tester.activate(application, state.allHosts); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 0a3c85d3702..4c8d5caad43 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -53,6 +53,7 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.function.Function; +import java.util.function.UnaryOperator; import java.util.logging.Level; import java.util.stream.Collectors; @@ -139,7 +140,22 @@ public class ProvisioningTester { public CapacityPolicies capacityPolicies() { return capacityPolicies; } public NodeList getNodes(ApplicationId id, Node.State ... inState) { return NodeList.copyOf(nodeRepository.getNodes(id, inState)); } - public void patchNode(Node node) { nodeRepository.write(node, () -> {}); } + public Node patchNode(Node node, UnaryOperator patcher) { + return patchNodes(List.of(node), patcher).get(0); + } + + public List patchNodes(List nodes, UnaryOperator patcher) { + List updated = new ArrayList<>(); + for (var node : nodes) { + try (var lock = nodeRepository.lock(node)) { + node = nodeRepository.getNode(node.hostname()).get(); + node = patcher.apply(node); + nodeRepository.write(node, lock); + updated.add(node); + } + } + return updated; + } public List prepare(ApplicationId application, ClusterSpec cluster, int nodeCount, int groups, NodeResources resources) { return prepare(application, cluster, nodeCount, groups, false, resources); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java index 51261c29a71..0ef7071b095 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java @@ -129,9 +129,7 @@ public class VirtualNodeProvisioningTest { assertDistinctParentHosts(nodes, ClusterSpec.Type.container, containerNodeCount); assertDistinctParentHosts(nodes, ClusterSpec.Type.content, contentNodeCount); - for (Node n : nodes) { - tester.patchNode(n.withParentHostname("clashing")); - } + tester.patchNodes(nodes, (n) -> n.withParentHostname("clashing")); containerHosts = prepare(containerClusterSpec, containerNodeCount, groups); contentHosts = prepare(contentClusterSpec, contentNodeCount, groups); activate(containerHosts, contentHosts); @@ -160,9 +158,7 @@ public class VirtualNodeProvisioningTest { assertDistinctParentHosts(nodes, ClusterSpec.Type.container, containerNodeCount); assertDistinctParentHosts(nodes, ClusterSpec.Type.content, contentNodeCount); - for (Node n : nodes) { - tester.patchNode(n.withParentHostname("clashing")); - } + tester.patchNodes(nodes, (n) -> n.withParentHostname("clashing")); OutOfCapacityException expected = null; try { containerHosts = prepare(containerClusterSpec, containerNodeCount, groups); @@ -216,9 +212,9 @@ public class VirtualNodeProvisioningTest { assertEquals(3, nodes.size()); // Set indistinct parents - tester.patchNode(nodes.get(0).withParentHostname("parentHost1")); - tester.patchNode(nodes.get(1).withParentHostname("parentHost1")); - tester.patchNode(nodes.get(2).withParentHostname("parentHost2")); + tester.patchNode(nodes.get(0), (n) -> n.withParentHostname("parentHost1")); + tester.patchNode(nodes.get(1), (n) -> n.withParentHostname("parentHost1")); + tester.patchNode(nodes.get(2), (n) -> n.withParentHostname("parentHost2")); nodes = getNodes(applicationId); assertEquals(3, nodes.stream().filter(n -> n.parentHostname().isPresent()).count()); -- cgit v1.2.3 From 2fc48b76ba4187b88c303f5fb7372c4df2a93f1e Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Wed, 9 Sep 2020 22:07:26 +0200 Subject: - Group commits to the TLS and sync to disk before acking. - Add zstd as compression. - No immediate commit. Delay ack until fully committed instead. --- .../vespa/searchcore/proton/server/feedhandler.cpp | 32 +++++++++- .../vespa/searchcore/proton/server/feedhandler.h | 5 ++ .../searchcore/proton/server/i_operation_storer.h | 4 ++ .../src/vespa/searchlib/config/translogserver.def | 6 +- .../src/vespa/searchlib/transactionlog/domain.cpp | 71 ++++++++++++++++++++-- .../src/vespa/searchlib/transactionlog/domain.h | 5 ++ .../vespa/searchlib/transactionlog/domainpart.cpp | 12 +++- .../searchlib/transactionlog/translogserver.cpp | 7 ++- 8 files changed, 128 insertions(+), 14 deletions(-) diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 37dfddf0c2c..209a35ce4a2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -402,6 +402,8 @@ FeedHandler::FeedHandler(IThreadingService &writeService, _tlsReplayProgress(), _serialNum(0), _prunedSerialNum(0), + _numPendingCommit(0), + _numCommitsCompleted(0), _delayedPrune(false), _feedLock(), _feedState(make_shared(getDocTypeName())), @@ -494,12 +496,40 @@ FeedHandler::getTransactionLogReplayDone() const { return _tlsMgr.getReplayDone(); } +void +FeedHandler::onCommitDone(uint64_t numPendingAtStart) { + assert(numPendingAtStart <= _numPendingCommit); + _numPendingCommit -= numPendingAtStart; + if (_numPendingCommit > 0) { + enqueCommitTask(); + } +} + +void FeedHandler::enqueCommitTask() { + _writeService.master().execute(makeLambdaTask([this]() { initiateCommit(); })); +} + +void +FeedHandler::initiateCommit() { + auto commitResult = _tlsWriter->startCommit(std::make_shared( + _writeService.master(), + makeLambdaTask([this, numPendingAtStart=_numPendingCommit]() { + onCommitDone(numPendingAtStart); + }))); + if (_activeFeedView) { + _activeFeedView->forceCommit(_serialNum, std::make_shared>(std::move(commitResult))); + } +} + void FeedHandler::appendOperation(const FeedOperation &op, TlsWriter::DoneCallback onDone) { if (!op.getSerialNum()) { const_cast(op).setSerialNum(incSerialNum()); } _tlsWriter->appendOperation(op, std::move(onDone)); + if (++_numPendingCommit == 1) { + enqueCommitTask(); + } } FeedHandler::CommitResult @@ -510,7 +540,7 @@ FeedHandler::startCommit(DoneCallback onDone) { void FeedHandler::storeOperationSync(const FeedOperation &op) { vespalib::Gate gate; - appendOperation(op, make_shared(gate)); + appendAndCommitOperation(op, make_shared(gate)); gate.await(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index 4807c596130..97629bfc018 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -76,6 +76,8 @@ private: // the serial num of the last message in the transaction log SerialNum _serialNum; SerialNum _prunedSerialNum; + size_t _numPendingCommit; + size_t _numCommitsCompleted; bool _delayedPrune; mutable std::shared_mutex _feedLock; FeedStateSP _feedState; @@ -125,6 +127,9 @@ private: FeedStateSP getFeedState() const; void changeFeedState(FeedStateSP newState); void doChangeFeedState(FeedStateSP newState); + void onCommitDone(uint64_t numPendingAtStart); + void initiateCommit(); + void enqueCommitTask(); public: FeedHandler(const FeedHandler &) = delete; FeedHandler & operator = (const FeedHandler &) = delete; diff --git a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h index b276779c2ee..c3b76a9db75 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h @@ -22,6 +22,10 @@ struct IOperationStorer */ virtual void appendOperation(const FeedOperation &op, DoneCallback onDone) = 0; [[nodiscard]] virtual CommitResult startCommit(DoneCallback onDone) = 0; + void appendAndCommitOperation(const FeedOperation &op, DoneCallback onDone) { + appendOperation(op, onDone); + (void) startCommit(std::move(onDone)); + } }; } // namespace proton diff --git a/searchlib/src/vespa/searchlib/config/translogserver.def b/searchlib/src/vespa/searchlib/config/translogserver.def index 38741745773..defce8c3421 100644 --- a/searchlib/src/vespa/searchlib/config/translogserver.def +++ b/searchlib/src/vespa/searchlib/config/translogserver.def @@ -15,7 +15,7 @@ basedir string default="tmp" restart ## Use fsync after each commit. ## If not the below interval is used. -usefsync bool default=false restart +usefsync bool default=false ##Number of threads available for visiting/subscription. maxthreads int default=4 restart @@ -24,12 +24,12 @@ maxthreads int default=4 restart crcmethod enum {ccitt_crc32, xxh64} default=xxh64 ## Control compression type. -compression.type enum {NONE, NONE_MULTI, LZ4, ZSTD} default=LZ4 +compression.type enum {NONE, NONE_MULTI, LZ4, ZSTD} default=NONE ## Control compression level ## LZ4 has normal range 1..9 while ZSTD has range 1..19 ## 9 is a reasonable default for both -compression.level int default=9 +compression.level int default=3 ## How large a chunk can grow in memory before beeing flushed chunk.sizelimit int default = 256000 # 256k diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index 9e0f1a8a1aa..6d251a2a30e 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -113,7 +113,12 @@ Domain::addPart(SerialNum partId, bool isLastPart) { } } -Domain::~Domain() { } +Domain::~Domain() { + MonitorGuard guard(_currentChunkMonitor); + guard.broadcast(); + commitChunk(grabCurrentChunk(guard), guard); + _singleCommitter->shutdown().sync(); +} DomainInfo Domain::getDomainInfo() const @@ -318,22 +323,78 @@ Domain::optionallyRotateFile(SerialNum serialNum) { return dp; } +void +Domain::append(const Packet & packet, Writer::DoneCallback onDone) { + vespalib::MonitorGuard guard(_currentChunkMonitor); + if (_lastSerial >= packet.range().from()) { + throw runtime_error(fmt("Incoming serial number(%" PRIu64 ") must be bigger than the last one (%" PRIu64 ").", + packet.range().from(), _lastSerial)); + } else { + _lastSerial = packet.range().to(); + } + _currentChunk->add(packet, std::move(onDone)); + commitIfFull(guard); +} + Domain::CommitResult Domain::startCommit(DoneCallback onDone) { - (void) onDone; + vespalib::MonitorGuard guard(_currentChunkMonitor); + if ( !_currentChunk->empty() ) { + auto completed = grabCurrentChunk(guard); + assert(completed); + completed->setCommitDoneCallback(std::move(onDone)); + CommitResult result(completed->createCommitResult()); + commitChunk(std::move(completed), guard); + return result; + } return CommitResult(); } void -Domain::append(const Packet & packet, Writer::DoneCallback onDone) -{ - (void) onDone; +Domain::commitIfFull(const vespalib::MonitorGuard &guard) { + if (_currentChunk->sizeBytes() > _config.getChunkSizeLimit()) { + auto completed = std::move(_currentChunk); + _currentChunk = std::make_unique(_config.getChunkSizeLimit(), completed->stealCallbacks()); + if (completed) { + commitChunk(std::move(completed), guard); + } + } +} + +std::unique_ptr +Domain::grabCurrentChunk(const vespalib::MonitorGuard & guard) { + assert(guard.monitors(_currentChunkMonitor)); + auto chunk = std::move(_currentChunk); + _currentChunk = createCommitChunk(_config); + return chunk; +} + +bool +Domain::commitChunk(std::unique_ptr chunk, const vespalib::MonitorGuard & chunkOrderGuard) { + assert(chunkOrderGuard.monitors(_currentChunkMonitor)); + if ( ! chunk->getPacket().empty()) { + _singleCommitter->execute( makeLambdaTask([this, chunk = std::move(chunk)]() mutable { + doCommit(std::move(chunk)); + })); + return true; + } + return false; +} + +void +Domain::doCommit(std::unique_ptr chunk) { + const Packet & packet = chunk->getPacket(); vespalib::nbostream_longlivedbuf is(packet.getHandle().data(), packet.getHandle().size()); Packet::Entry entry; entry.deserialize(is); DomainPart::SP dp = optionallyRotateFile(entry.serial()); dp->commit(entry.serial(), packet); + if (_config.getFSyncOnCommit()) { + dp->sync(); + } cleanSessions(); + LOG(debug, "Releasing %zu acks and %zu entries and %zu bytes.", + chunk->getNumCallBacks(), chunk->getPacket().size(), chunk->sizeBytes()); } bool diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h index 7e77e6ef0ef..5bd11ea2bdf 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h @@ -56,6 +56,11 @@ public: uint64_t size() const; Domain & setConfig(const DomainConfig & cfg); private: + void commitIfFull(const vespalib::MonitorGuard & guard); + + std::unique_ptr grabCurrentChunk(const vespalib::MonitorGuard & guard); + bool commitChunk(std::unique_ptr chunk, const vespalib::MonitorGuard & chunkOrderGuard); + void doCommit(std::unique_ptr chunk); SerialNum begin(const vespalib::LockGuard & guard) const; SerialNum end(const vespalib::LockGuard & guard) const; size_t byteSize(const vespalib::LockGuard & guard) const; diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp index 8855183226d..b7e02894e6b 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp @@ -252,7 +252,7 @@ DomainPart::buildPacketMapping(bool allowTruncate) DomainPart::DomainPart(const string & name, const string & baseDir, SerialNum s, Encoding encoding, uint8_t compressionLevel, const FileHeaderContext &fileHeaderContext, bool allowTruncate) - : _encoding(encoding.getCrc(), Encoding::Compression::none), //TODO We do not yet support compression + : _encoding(encoding), _compressionLevel(compressionLevel), _lock(), _fileLock(), @@ -396,16 +396,19 @@ DomainPart::commit(SerialNum firstSerial, const Packet &packet) if (_range.from() == 0) { _range.from(firstSerial); } + IChunk::UP chunk = IChunk::create(_encoding, _compressionLevel); for (size_t i(0); h.size() > 0; i++) { //LOG(spam, //"Pos(%d) Len(%d), Lim(%d), Remaining(%d)", //h.getPos(), h.getLength(), h.getLimit(), h.getRemaining()); - IChunk::UP chunk = IChunk::create(_encoding, _compressionLevel); Packet::Entry entry; entry.deserialize(h); if (_range.to() < entry.serial()) { chunk->add(entry); - write(*_transLog, *chunk); + if (_encoding.getCompression() == Encoding::Compression::none) { + write(*_transLog, *chunk); + chunk = IChunk::create(_encoding, _compressionLevel); + } _sz++; _range.to(entry.serial()); } else { @@ -413,6 +416,9 @@ DomainPart::commit(SerialNum firstSerial, const Packet &packet) entry.serial(), _range.to())); } } + if ( ! chunk->getEntries().empty()) { + write(*_transLog, *chunk); + } bool merged(false); LockGuard guard(_lock); diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index 7be3dd708a5..0c0c9186e12 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -572,8 +572,11 @@ TransLogServer::domainCommit(FRT_RPCRequest *req) Packet packet(params[1]._data._buf, params[1]._data._len); try { vespalib::Gate gate; - domain->append(packet, make_shared(gate)); - auto keep = domain->startCommit(make_shared()); + { + // Need to scope in order to drain out all the callbacks. + domain->append(packet, make_shared(gate)); + auto keep = domain->startCommit(make_shared()); + } gate.await(); ret.AddInt32(0); ret.AddString("ok"); -- cgit v1.2.3 From 5afff095ad3ba0b0811b5788ebaeb3c84832d7f6 Mon Sep 17 00:00:00 2001 From: Tor Brede Vekterli Date: Tue, 22 Sep 2020 12:24:12 +0000 Subject: Avoid address stringification in common lookup path Precompute internal address hash over tuple. No other fields are included in the hash, as this is only used for storage API lookups. Remove automatic resending field from address, as we never use MBus resending functionality in the backend communication protocols. --- .../storage/storageserver/communicationmanager.cpp | 4 +-- .../rpc/caching_rpc_target_resolver.cpp | 28 ++++++++-------- .../rpc/caching_rpc_target_resolver.h | 37 ++++++++++++---------- storageapi/src/tests/messageapi/CMakeLists.txt | 5 ++- .../messageapi/storage_message_address_test.cpp | 36 +++++++++++++++++++++ .../vespa/storageapi/messageapi/storagemessage.cpp | 28 ++++++++++++---- .../vespa/storageapi/messageapi/storagemessage.h | 10 ++++-- 7 files changed, 106 insertions(+), 42 deletions(-) create mode 100644 storageapi/src/tests/messageapi/storage_message_address_test.cpp diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index 5471d66a864..99fdb97e435 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -581,7 +581,7 @@ CommunicationManager::sendCommand( auto cmd = std::make_unique(msg); cmd->setContext(mbus::Context(msg->getMsgId())); - cmd->setRetryEnabled(address.retryEnabled()); + cmd->setRetryEnabled(false); cmd->setTimeRemaining(msg->getTimeout()); cmd->setTrace(msg->getTrace()); sendMessageBusMessage(msg, std::move(cmd), address.getRoute()); @@ -597,7 +597,7 @@ CommunicationManager::sendCommand( if (mbusMsg) { MBUS_TRACE(msg->getTrace(), 7, "Communication manager: Converted OK"); mbusMsg->setTrace(msg->getTrace()); - mbusMsg->setRetryEnabled(address.retryEnabled()); + mbusMsg->setRetryEnabled(false); { vespalib::LockGuard lock(_messageBusSentLock); diff --git a/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.cpp b/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.cpp index 5241ec6f769..6bcb154aed5 100644 --- a/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.cpp @@ -33,9 +33,9 @@ CachingRpcTargetResolver::address_to_slobrok_id(const api::StorageMessageAddress } std::shared_ptr -CachingRpcTargetResolver::lookup_target(const vespalib::string& slobrok_id, uint32_t curr_slobrok_gen) { +CachingRpcTargetResolver::lookup_target(const api::StorageMessageAddress& address, uint32_t curr_slobrok_gen) { std::shared_lock lock(_targets_rwmutex); - auto itr = _targets.find(slobrok_id); + auto itr = _targets.find(address); if ((itr != _targets.end()) && itr->second->_target->is_valid() && (itr->second->_slobrok_gen == curr_slobrok_gen)) { @@ -45,18 +45,19 @@ CachingRpcTargetResolver::lookup_target(const vespalib::string& slobrok_id, uint } std::shared_ptr -CachingRpcTargetResolver::consider_update_target(const vespalib::string& slobrok_id, +CachingRpcTargetResolver::consider_update_target(const api::StorageMessageAddress& address, const vespalib::string& connection_spec, uint32_t curr_slobrok_gen, [[maybe_unused]] const UniqueLock& targets_lock) { // If address has the same spec as the existing target, just reuse it. - auto itr = _targets.find(slobrok_id); + auto itr = _targets.find(address); if ((itr != _targets.end()) && (itr->second->_target->is_valid()) && (itr->second->_spec == connection_spec)) { LOG(debug, "Updating existing mapping '%s' -> '%s' (gen %u) to gen %u", - slobrok_id.c_str(), connection_spec.c_str(), itr->second->_slobrok_gen, curr_slobrok_gen); + address.toString().c_str(), connection_spec.c_str(), + itr->second->_slobrok_gen, curr_slobrok_gen); itr->second->_slobrok_gen = curr_slobrok_gen; return itr->second; } @@ -64,26 +65,27 @@ CachingRpcTargetResolver::consider_update_target(const vespalib::string& slobrok } std::shared_ptr -CachingRpcTargetResolver::insert_new_target_mapping(const vespalib::string& slobrok_id, +CachingRpcTargetResolver::insert_new_target_mapping(const api::StorageMessageAddress& address, const vespalib::string& connection_spec, uint32_t curr_slobrok_gen, [[maybe_unused]] const UniqueLock& targets_lock) { auto target = _target_factory.make_target(connection_spec, curr_slobrok_gen); // TODO expensive inside lock? assert(target); std::shared_ptr rpc_target(std::move(target)); - _targets[slobrok_id] = rpc_target; - LOG(debug, "Added mapping '%s' -> '%s' at gen %u", slobrok_id.c_str(), connection_spec.c_str(), curr_slobrok_gen); + // TODO emplacement (with replace) semantics to avoid need for default constructed K/V + _targets[address] = rpc_target; + LOG(debug, "Added mapping '%s' -> '%s' at gen %u", address.toString().c_str(), + connection_spec.c_str(), curr_slobrok_gen); return rpc_target; } std::shared_ptr CachingRpcTargetResolver::resolve_rpc_target(const api::StorageMessageAddress& address) { - // TODO or map directly from address to target instead of going via stringification? Needs hashing, if so. - auto slobrok_id = address_to_slobrok_id(address); const uint32_t curr_slobrok_gen = _slobrok_mirror.updates(); - if (auto result = lookup_target(slobrok_id, curr_slobrok_gen)) { + if (auto result = lookup_target(address, curr_slobrok_gen)) { return result; } + auto slobrok_id = address_to_slobrok_id(address); auto specs = _slobrok_mirror.lookup(slobrok_id); // FIXME string type mismatch; implicit conv! if (specs.empty()) { LOG(debug, "Found no mapping for '%s'", slobrok_id.c_str()); @@ -95,10 +97,10 @@ CachingRpcTargetResolver::resolve_rpc_target(const api::StorageMessageAddress& a assert(specs.size() == 1); const auto& connection_spec = specs[0].second; std::unique_lock lock(_targets_rwmutex); - if (auto result = consider_update_target(slobrok_id, connection_spec, curr_slobrok_gen, lock)) { + if (auto result = consider_update_target(address, connection_spec, curr_slobrok_gen, lock)) { return result; } - return insert_new_target_mapping(slobrok_id, connection_spec, curr_slobrok_gen, lock); + return insert_new_target_mapping(address, connection_spec, curr_slobrok_gen, lock); } } diff --git a/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.h b/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.h index cf94f7545bc..52b505d5476 100644 --- a/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.h +++ b/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.h @@ -3,38 +3,44 @@ #include "rpc_target.h" #include "rpc_target_factory.h" +#include #include #include #include namespace slobrok::api { class IMirrorAPI; } -namespace storage { - -namespace api { class StorageMessageAddress; } - -namespace rpc { +namespace storage::rpc { /** * Class that resolves and caches rpc targets based on StorageMessageAddress that is mapped to slobrok id, * with lookup in a slobrok mirror. */ class CachingRpcTargetResolver { -private: - const slobrok::api::IMirrorAPI& _slobrok_mirror; - const RpcTargetFactory& _target_factory; + + struct AddressInternalHasher { + size_t operator()(const api::StorageMessageAddress& addr) const noexcept { + return addr.internal_storage_hash(); + } + }; + using TargetHashMap = vespalib::hash_map, + AddressInternalHasher>; using UniqueLock = std::unique_lock; - mutable std::shared_mutex _targets_rwmutex; - // TODO LRU? Size cap? - vespalib::hash_map> _targets; - std::shared_ptr lookup_target(const vespalib::string& slobrok_id, uint32_t curr_slobrok_gen); - std::shared_ptr consider_update_target(const vespalib::string& slobrok_id, + const slobrok::api::IMirrorAPI& _slobrok_mirror; + const RpcTargetFactory& _target_factory; + mutable std::shared_mutex _targets_rwmutex; + TargetHashMap _targets; // TODO LRU? Size cap? + + std::shared_ptr lookup_target(const api::StorageMessageAddress& address, + uint32_t curr_slobrok_gen); + std::shared_ptr consider_update_target(const api::StorageMessageAddress& address, const vespalib::string& connection_spec, uint32_t curr_slobrok_gen, const UniqueLock& targets_lock); - std::shared_ptr insert_new_target_mapping(const vespalib::string& slobrok_id, + std::shared_ptr insert_new_target_mapping(const api::StorageMessageAddress& address, const vespalib::string& connection_spec, uint32_t curr_slobrok_gen, const UniqueLock& targets_lock); @@ -49,5 +55,4 @@ public: std::shared_ptr resolve_rpc_target(const api::StorageMessageAddress& address); }; -} // rpc -} // storage +} // storage::rpc diff --git a/storageapi/src/tests/messageapi/CMakeLists.txt b/storageapi/src/tests/messageapi/CMakeLists.txt index 4833dc45acf..50f0b306191 100644 --- a/storageapi/src/tests/messageapi/CMakeLists.txt +++ b/storageapi/src/tests/messageapi/CMakeLists.txt @@ -1,5 +1,8 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storageapi_testmessageapi INTERFACE +vespa_add_library(storageapi_testmessageapi SOURCES + storage_message_address_test.cpp DEPENDS + storageapi + GTest::GTest ) diff --git a/storageapi/src/tests/messageapi/storage_message_address_test.cpp b/storageapi/src/tests/messageapi/storage_message_address_test.cpp new file mode 100644 index 00000000000..c340cba4b28 --- /dev/null +++ b/storageapi/src/tests/messageapi/storage_message_address_test.cpp @@ -0,0 +1,36 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include +#include + +using namespace ::testing; + +namespace storage::api { + +namespace { + +size_t hash_of(vespalib::stringref cluster, const lib::NodeType& type, uint16_t index) { + return StorageMessageAddress(cluster, type, index).internal_storage_hash(); +} + +} + +TEST(StorageMessageAddressTest, storage_hash_covers_all_expected_fields) { + EXPECT_EQ(hash_of("foo", lib::NodeType::STORAGE, 0), + hash_of("foo", lib::NodeType::STORAGE, 0)); + EXPECT_EQ(hash_of("foo", lib::NodeType::DISTRIBUTOR, 0), + hash_of("foo", lib::NodeType::DISTRIBUTOR, 0)); + EXPECT_EQ(hash_of("foo", lib::NodeType::STORAGE, 123), + hash_of("foo", lib::NodeType::STORAGE, 123)); + + // These tests are all true with extremely high probability, though they do + // depend on a hash function that may inherently cause collisions. + EXPECT_NE(hash_of("foo", lib::NodeType::STORAGE, 0), + hash_of("bar", lib::NodeType::STORAGE, 0)); + EXPECT_NE(hash_of("foo", lib::NodeType::STORAGE, 0), + hash_of("foo", lib::NodeType::DISTRIBUTOR, 0)); + EXPECT_NE(hash_of("foo", lib::NodeType::STORAGE, 0), + hash_of("foo", lib::NodeType::STORAGE, 1)); +} + +} // storage::api diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp index d1bd24f5087..8276587834a 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp +++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -141,8 +142,8 @@ MessageType::print(std::ostream& out, bool verbose, const std::string& indent) c StorageMessageAddress::StorageMessageAddress(const mbus::Route& route) : _route(route), - _retryEnabled(false), _protocol(DOCUMENT), + _precomputed_storage_hash(0), _cluster(""), _type(nullptr), _index(0xFFFF) @@ -160,17 +161,34 @@ createAddress(vespalib::stringref cluster, const lib::NodeType& type, uint16_t i return os.str(); } +// TODO we ideally want this removed. Currently just in place to support usage as map key when emplacement not available +StorageMessageAddress::StorageMessageAddress() + : _route(), + _protocol(Protocol::STORAGE), + _precomputed_storage_hash(0), + _cluster(), + _type(nullptr), + _index(0) +{} + + StorageMessageAddress::StorageMessageAddress(vespalib::stringref cluster, const lib::NodeType& type, uint16_t index, Protocol protocol) : _route(), - _retryEnabled(false), _protocol(protocol), + _precomputed_storage_hash(0), _cluster(cluster), _type(&type), _index(index) { std::vector directives; - directives.emplace_back(std::make_shared(createAddress(cluster, type, index))); + auto address_as_str = createAddress(cluster, type, index); + // We reuse the string representation and pass it to vespalib's hashValue instead of + // explicitly combining a running hash over the individual fields. This is because + // hashValue internally uses xxhash, which offers great dispersion of bits even for + // minimal changes in the input (such as single bit differences in the index). + _precomputed_storage_hash = vespalib::hashValue(address_as_str.data(), address_as_str.size()); + directives.emplace_back(std::make_shared(std::move(address_as_str))); _route.addHop(mbus::Hop(std::move(directives), false)); } @@ -207,7 +225,6 @@ bool StorageMessageAddress::operator==(const StorageMessageAddress& other) const { if (_protocol != other._protocol) return false; - if (_retryEnabled != other._retryEnabled) return false; if (_type != other._type) return false; if (_type) { if (_cluster != other._cluster) return false; @@ -234,9 +251,6 @@ StorageMessageAddress::print(vespalib::asciistream & out) const } else { out << "Document protocol"; } - if (_retryEnabled) { - out << ", retry enabled"; - } if (!_type) { out << ", " << _route.toString() << ")"; } else { diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h index 415bd7717f2..85d4e072171 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h +++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h @@ -269,14 +269,15 @@ public: private: mbus::Route _route; - bool _retryEnabled; Protocol _protocol; // Used for internal VDS addresses only + size_t _precomputed_storage_hash; vespalib::string _cluster; const lib::NodeType* _type; uint16_t _index; public: + StorageMessageAddress(); // Only to be used when transient default ctor semantics are needed by containers StorageMessageAddress(const mbus::Route& route); StorageMessageAddress(vespalib::stringref clusterName, const lib::NodeType& type, uint16_t index, @@ -284,15 +285,18 @@ public: ~StorageMessageAddress(); void setProtocol(Protocol p) { _protocol = p; } - void enableRetry(bool enable = true) { _retryEnabled = enable; } const mbus::Route& getRoute() const { return _route; } - bool retryEnabled() const { return _retryEnabled; } Protocol getProtocol() const { return _protocol; } uint16_t getIndex() const; const lib::NodeType& getNodeType() const; const vespalib::string& getCluster() const; + // Returns precomputed hash over tuple. Other fields not included. + [[nodiscard]] size_t internal_storage_hash() const noexcept { + return _precomputed_storage_hash; + } + bool operator==(const StorageMessageAddress& other) const; vespalib::string toString() const; friend std::ostream & operator << (std::ostream & os, const StorageMessageAddress & addr); -- cgit v1.2.3 From a4621d00adcc2fd6ece64b0ecacff05d3d20e67b Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Thu, 17 Sep 2020 15:48:31 +0200 Subject: Set name of metric reporter thread --- .../yahoo/container/handler/threadpool/ExecutorServiceWrapper.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java index 9a73c98597a..3b2b5697e5c 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java @@ -30,7 +30,7 @@ class ExecutorServiceWrapper extends ForwardingExecutorService { ExecutorServiceWrapper( WorkerCompletionTimingThreadPoolExecutor wrapped, ThreadPoolMetric metric, ProcessTerminator processTerminator, - long maxThreadExecutionTimeMillis) { + long maxThreadExecutionTimeMillis, String name) { this.wrapped = wrapped; this.metric = metric; this.processTerminator = processTerminator; @@ -39,13 +39,14 @@ class ExecutorServiceWrapper extends ForwardingExecutorService { metric.reportThreadPoolSize(wrapped.getPoolSize()); metric.reportActiveThreads(wrapped.getActiveCount()); metricReporter = new Thread(this::reportMetrics); + metricReporter.setName(name + "-threadpool-metric-reporter"); metricReporter.setDaemon(true); metricReporter.start(); } int queuedTasks() { return wrapped.getQueue().size(); } - private final void reportMetrics() { + private void reportMetrics() { try { while (!closed.get()) { metric.reportThreadPoolSize(wrapped.getPoolSize()); -- cgit v1.2.3 From e7f7be2e596453f16d80896eb11f68895d275eb1 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 22 Sep 2020 12:43:10 +0000 Subject: Remove unnecessary guard and move check for emptiness inside doCommit to ensure ordering also of empty chunks. --- .../src/vespa/searchlib/transactionlog/domain.cpp | 19 +++++++------------ searchlib/src/vespa/searchlib/transactionlog/domain.h | 2 +- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index 6d251a2a30e..fdee9dc1a24 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -341,7 +341,6 @@ Domain::startCommit(DoneCallback onDone) { vespalib::MonitorGuard guard(_currentChunkMonitor); if ( !_currentChunk->empty() ) { auto completed = grabCurrentChunk(guard); - assert(completed); completed->setCommitDoneCallback(std::move(onDone)); CommitResult result(completed->createCommitResult()); commitChunk(std::move(completed), guard); @@ -355,9 +354,7 @@ Domain::commitIfFull(const vespalib::MonitorGuard &guard) { if (_currentChunk->sizeBytes() > _config.getChunkSizeLimit()) { auto completed = std::move(_currentChunk); _currentChunk = std::make_unique(_config.getChunkSizeLimit(), completed->stealCallbacks()); - if (completed) { - commitChunk(std::move(completed), guard); - } + commitChunk(std::move(completed), guard); } } @@ -369,20 +366,18 @@ Domain::grabCurrentChunk(const vespalib::MonitorGuard & guard) { return chunk; } -bool +void Domain::commitChunk(std::unique_ptr chunk, const vespalib::MonitorGuard & chunkOrderGuard) { assert(chunkOrderGuard.monitors(_currentChunkMonitor)); - if ( ! chunk->getPacket().empty()) { - _singleCommitter->execute( makeLambdaTask([this, chunk = std::move(chunk)]() mutable { - doCommit(std::move(chunk)); - })); - return true; - } - return false; + _singleCommitter->execute( makeLambdaTask([this, chunk = std::move(chunk)]() mutable { + doCommit(std::move(chunk)); + })); } void Domain::doCommit(std::unique_ptr chunk) { + if (chunk->empty()) return; + const Packet & packet = chunk->getPacket(); vespalib::nbostream_longlivedbuf is(packet.getHandle().data(), packet.getHandle().size()); Packet::Entry entry; diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h index 5bd11ea2bdf..041ec27cf23 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h @@ -59,7 +59,7 @@ private: void commitIfFull(const vespalib::MonitorGuard & guard); std::unique_ptr grabCurrentChunk(const vespalib::MonitorGuard & guard); - bool commitChunk(std::unique_ptr chunk, const vespalib::MonitorGuard & chunkOrderGuard); + void commitChunk(std::unique_ptr chunk, const vespalib::MonitorGuard & chunkOrderGuard); void doCommit(std::unique_ptr chunk); SerialNum begin(const vespalib::LockGuard & guard) const; SerialNum end(const vespalib::LockGuard & guard) const; -- cgit v1.2.3 From 4e45bca990ca47af41f3ba20535c17c22ac38d5e Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Tue, 22 Sep 2020 14:49:18 +0200 Subject: Use feature flag use-config-server-vip --- .../vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java index 7c44ae6d1a5..f5e6354da05 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java @@ -11,6 +11,8 @@ import com.yahoo.restapi.Path; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; @@ -33,12 +35,14 @@ public class ZoneApiHandler extends AuditLoggingRequestHandler { private final ZoneRegistry zoneRegistry; private final ConfigServerRestExecutor proxy; + private final BooleanFlag useConfigServerVip; public ZoneApiHandler(LoggingRequestHandler.Context parentCtx, ServiceRegistry serviceRegistry, ConfigServerRestExecutor proxy, Controller controller) { super(parentCtx, controller.auditLogger()); this.zoneRegistry = serviceRegistry.zoneRegistry(); this.proxy = proxy; + this.useConfigServerVip = Flags.USE_CONFIG_SERVER_VIP.bindTo(controller.flagSource()); } @Override @@ -108,8 +112,8 @@ public class ZoneApiHandler extends AuditLoggingRequestHandler { } private ProxyRequest proxyRequest(ZoneId zoneId, String path, HttpRequest request) { - // TODO: Use config server VIP for all zones that have one - if (zoneId.region().value().startsWith("aws-") || zoneId.region().value().contains("-aws-")) { + // TODO: Still need to hardcode AWS since flag cannot be set until flag has been rolled out + if (zoneId.region().value().startsWith("aws-") || useConfigServerVip.value()) { return ProxyRequest.tryOne(zoneRegistry.getConfigServerVipUri(zoneId), path, request); } return ProxyRequest.tryAll(zoneRegistry.getConfigServerUris(zoneId), path, request); -- cgit v1.2.3 From 2b7f9bf28844837b7a7dfd21857806406578ec0e Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 22 Sep 2020 12:52:11 +0000 Subject: Check for no data, as there might be callbacks that have been moved over. --- searchlib/src/vespa/searchlib/transactionlog/domain.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index fdee9dc1a24..bd7feec0598 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -376,9 +376,9 @@ Domain::commitChunk(std::unique_ptr chunk, const vespalib::MonitorG void Domain::doCommit(std::unique_ptr chunk) { - if (chunk->empty()) return; - const Packet & packet = chunk->getPacket(); + if (packet.empty()) return; + vespalib::nbostream_longlivedbuf is(packet.getHandle().data(), packet.getHandle().size()); Packet::Entry entry; entry.deserialize(is); -- cgit v1.2.3 From 449dbcad01d0b18531b0a2707e933cbbe60733d8 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Thu, 17 Sep 2020 16:06:30 +0200 Subject: Add metrics for the threadpool's work queue --- .../yahoo/vespa/model/admin/monitoring/VespaMetricSet.java | 14 +++++++++++++- .../handler/threadpool/DefaultContainerThreadpool.java | 5 +++-- .../handler/threadpool/ExecutorServiceWrapper.java | 8 +++++--- .../container/handler/threadpool/ThreadPoolMetric.java | 2 ++ 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 120e323e652..78e97719af0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.admin.monitoring; import java.util.Collections; import java.util.LinkedHashSet; +import java.util.List; import java.util.Set; import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; @@ -129,7 +130,12 @@ public class VespaMetricSet { metrics.add(new Metric("serverActiveThreads.count")); metrics.add(new Metric("serverActiveThreads.last")); - metrics.add(new Metric("jdisc.thread_pool.unhandled_exceptions.rate")); + { + List suffices = List.of("sum", "count", "last", "min", "max"); + addMetric(metrics, "jdisc.thread_pool.unhandled_exceptions", suffices); + addMetric(metrics, "jdisc.thread_pool.work_queue.capacity", suffices); + addMetric(metrics, "jdisc.thread_pool.work_queue.size", suffices); + } metrics.add(new Metric("httpapi_latency.max")); metrics.add(new Metric("httpapi_latency.sum")); @@ -687,4 +693,10 @@ public class VespaMetricSet { return metrics; } + private static void addMetric(Set metrics, String metricName, List aggregateSuffices) { + for (String suffix : aggregateSuffices) { + metrics.add(new Metric(metricName + "." + suffix)); + } + } + } diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/DefaultContainerThreadpool.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/DefaultContainerThreadpool.java index 4d8c245a25a..46b3a86798b 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/DefaultContainerThreadpool.java +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/DefaultContainerThreadpool.java @@ -44,8 +44,9 @@ public class DefaultContainerThreadpool extends AbstractComponent implements Aut // get the dreaded thread locals initialized even if they will never run. // That counters what we we want to achieve with the Q that will prefer thread locality. executor.prestartAllCoreThreads(); - threadpool = new ExecutorServiceWrapper(executor, threadPoolMetric, processTerminator, - config.maxThreadExecutionTimeSeconds() * 1000L); + threadpool = new ExecutorServiceWrapper( + executor, threadPoolMetric, processTerminator, config.maxThreadExecutionTimeSeconds() * 1000L, + config.name(), config.queueSize()); } @Override public Executor executor() { return threadpool; } diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java index 3b2b5697e5c..771c1da82b6 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java @@ -24,17 +24,19 @@ class ExecutorServiceWrapper extends ForwardingExecutorService { private final ThreadPoolMetric metric; private final ProcessTerminator processTerminator; private final long maxThreadExecutionTimeMillis; + private final int queueCapacity; private final Thread metricReporter; private final AtomicBoolean closed = new AtomicBoolean(false); ExecutorServiceWrapper( WorkerCompletionTimingThreadPoolExecutor wrapped, ThreadPoolMetric metric, ProcessTerminator processTerminator, - long maxThreadExecutionTimeMillis, String name) { + long maxThreadExecutionTimeMillis, String name, int queueCapacity) { this.wrapped = wrapped; this.metric = metric; this.processTerminator = processTerminator; this.maxThreadExecutionTimeMillis = maxThreadExecutionTimeMillis; + this.queueCapacity = queueCapacity; metric.reportThreadPoolSize(wrapped.getPoolSize()); metric.reportActiveThreads(wrapped.getActiveCount()); @@ -44,13 +46,13 @@ class ExecutorServiceWrapper extends ForwardingExecutorService { metricReporter.start(); } - int queuedTasks() { return wrapped.getQueue().size(); } - private void reportMetrics() { try { while (!closed.get()) { metric.reportThreadPoolSize(wrapped.getPoolSize()); metric.reportActiveThreads(wrapped.getActiveCount()); + metric.reportWorkQueueSize(wrapped.getQueue().size()); + metric.reportWorkQueueCapacity(queueCapacity); Thread.sleep(100); } } catch (InterruptedException e) { } diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java index d9ab020bcb8..18ccf3ba8e5 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java @@ -25,6 +25,8 @@ class ThreadPoolMetric { void reportRejectRequest() { metric.add("serverRejectedRequests", 1L, defaultContext); } void reportThreadPoolSize(long size) { metric.set("serverThreadPoolSize", size, defaultContext); } void reportActiveThreads(long threads) { metric.set("serverActiveThreads", threads, defaultContext); } + void reportWorkQueueCapacity(long capacity) { metric.set("jdisc.thread_pool.work_queue.capacity", capacity, defaultContext); } + void reportWorkQueueSize(long size) { metric.set("jdisc.thread_pool.work_queue.size", size, defaultContext); } void reportUnhandledException(Throwable t) { Metric.Context ctx = metric.createContext(Map.of( THREAD_POOL_NAME_DIMENSION, threadPoolName, -- cgit v1.2.3 From 28f7c0ced6e771fe9fd44d1716d71f3788bf1d5c Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 22 Sep 2020 13:53:41 +0000 Subject: Control visibility-delay with feature flag. --- .../com/yahoo/config/model/api/ModelContext.java | 1 + .../yahoo/config/model/deploy/TestProperties.java | 7 +++++ .../vespa/model/content/ContentSearchCluster.java | 1 + .../model/builder/xml/dom/ContentBuilderTest.java | 35 ++++++++++++++++++---- .../config/server/deploy/ModelContextImpl.java | 4 +++ .../src/main/java/com/yahoo/vespa/flags/Flags.java | 6 ++++ 6 files changed, 48 insertions(+), 6 deletions(-) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 756961933db..e4c203fda6b 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -84,6 +84,7 @@ public interface ModelContext { boolean skipCommunicationManagerThread(); boolean skipMbusRequestThread(); boolean skipMbusReplyThread(); + double visibilityDelay(); boolean useContentNodeBtreeDb(); diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 22bdf31350a..d9cebd2b1d2 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -41,6 +41,7 @@ public class TestProperties implements ModelContext.Properties { private double defaultTermwiseLimit = 1.0; private double threadPoolSizeFactor = 0.0; private double queueSizeFactor = 0.0; + private double visibilityDelay = 0.0; private String jvmGCOptions = null; private String sequencerType = "LATENCY"; private String responseSequencerType = "ADAPTIVE"; @@ -83,6 +84,7 @@ public class TestProperties implements ModelContext.Properties { @Override public boolean skipMbusRequestThread() { return false; } @Override public boolean skipMbusReplyThread() { return false; } @Override public Quota quota() { return quota; } + @Override public double visibilityDelay() { return visibilityDelay; } public TestProperties setJvmGCOptions(String gcOptions) { jvmGCOptions = gcOptions; @@ -139,6 +141,11 @@ public class TestProperties implements ModelContext.Properties { return this; } + public TestProperties setVisibilityDelay(double visibilityDelay) { + this.visibilityDelay = visibilityDelay; + return this; + } + public TestProperties setMultitenant(boolean multitenant) { this.multitenant = multitenant; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index f329f62ee7b..50a6054d1b9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -200,6 +200,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot this.flushOnShutdown = flushOnShutdown; this.combined = combined; feedSequencerType = convertFeedSequencerType(featureFlags.feedSequencerType()); + visibilityDelay = featureFlags.visibilityDelay(); } public void setVisibilityDelay(double delay) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 99f8b8cbb5e..443bd2433c1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -760,15 +760,12 @@ public class ContentBuilderTest extends DomBuilderTest { } } - private void verifyFeedSequencer(String input, String expected) { - verifyFeedSequencer(input, expected, 0); - } - private void verifyFeedSequencer(String input, String expected, double visibilityDelay) { - String hostedXml = "" + + private String xmlWithVisibilityDelay(Double visibilityDelay) { + return "" + "" + " 1" + " " + - " " + visibilityDelay + "" + + ((visibilityDelay != null) ? " " + visibilityDelay + "" : "") + " " + " " + " " + @@ -776,6 +773,13 @@ public class ContentBuilderTest extends DomBuilderTest { " " + "" + ""; + } + + private void verifyFeedSequencer(String input, String expected) { + verifyFeedSequencer(input, expected, 0); + } + private void verifyFeedSequencer(String input, String expected, double visibilityDelay) { + String hostedXml = xmlWithVisibilityDelay(visibilityDelay); DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().setFeedSequencerType(input)); VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() @@ -800,6 +804,25 @@ public class ContentBuilderTest extends DomBuilderTest { } + private void verifyThatFeatureFlagControlsVisibilityDelayDefault(double defaultVisibiliDelay, Double xmlOverride, double expected) { + String hostedXml = xmlWithVisibilityDelay(xmlOverride); + DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().setVisibilityDelay(defaultVisibiliDelay)); + VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() + .withServices(hostedXml) + .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .build()) + .create(deployStateBuilder); + ProtonConfig config = getProtonConfig(model.getContentClusters().values().iterator().next()); + assertEquals(expected, config.documentdb(0).visibilitydelay(), 0.0); + } + @Test + public void verifyThatFeatureFlagControlsVisibilityDelayDefault() { + verifyThatFeatureFlagControlsVisibilityDelayDefault(0.0, null, 0.0); + verifyThatFeatureFlagControlsVisibilityDelayDefault(0.3, null, 0.3); + verifyThatFeatureFlagControlsVisibilityDelayDefault(0.0, 0.5, 0.5); + verifyThatFeatureFlagControlsVisibilityDelayDefault(0.3, 0.6, 0.6); + } + @Test public void failWhenNoDocumentsElementSpecified() { expectedException.expect(IllegalArgumentException.class); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 87b0ed965d3..4ac20f6220f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -164,6 +164,7 @@ public class ModelContextImpl implements ModelContext { private final Optional athenzDomain; private final Optional applicationRoles; private final double feedCoreThreadPoolSizeFactor; + private final double visibilityDelay; private final Quota quota; public Properties(ApplicationId applicationId, @@ -204,6 +205,8 @@ public class ModelContextImpl implements ModelContext { .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); threadPoolSizeFactor = Flags.DEFAULT_THREADPOOL_SIZE_FACTOR.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + visibilityDelay = Flags.VISIBILITY_DELAY.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); queueSizefactor = Flags.DEFAULT_QUEUE_SIZE_FACTOR.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); jvmGCOPtions = Flags.JVM_GC_OPTIONS.bindTo(flagSource) @@ -313,6 +316,7 @@ public class ModelContextImpl implements ModelContext { @Override public boolean skipMbusRequestThread() { return skipMbusRequestThread; } @Override public boolean skipMbusReplyThread() { return skipMbusReplyThread; } @Override public double feedCoreThreadPoolSizeFactor() { return feedCoreThreadPoolSizeFactor; } + @Override public double visibilityDelay() { return visibilityDelay; } @Override public Quota quota() { return quota; } } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 35b2f05f30b..a6748667b7e 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -196,6 +196,12 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundDoubleFlag VISIBILITY_DELAY = defineDoubleFlag( + "visibility-delay", 0.0, + "Default visibility-delay", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag USE_DIRECT_STORAGE_API_RPC = defineFeatureFlag( "use-direct-storage-api-rpc", false, "Whether to use direct RPC for Storage API communication between content cluster nodes.", -- cgit v1.2.3 From 9a15b63e3e9e95739f938a064fb26ddfd1f9ae84 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Tue, 22 Sep 2020 11:26:09 +0200 Subject: Add service layer in benchmark. --- .../src/apps/vespa-spi-feed-bm/CMakeLists.txt | 3 + .../apps/vespa-spi-feed-bm/vespa_spi_feed_bm.cpp | 416 ++++++++++++++++++++- 2 files changed, 410 insertions(+), 9 deletions(-) diff --git a/searchcore/src/apps/vespa-spi-feed-bm/CMakeLists.txt b/searchcore/src/apps/vespa-spi-feed-bm/CMakeLists.txt index e188bc16ec0..5b562cd1d28 100644 --- a/searchcore/src/apps/vespa-spi-feed-bm/CMakeLists.txt +++ b/searchcore/src/apps/vespa-spi-feed-bm/CMakeLists.txt @@ -20,5 +20,8 @@ vespa_add_executable(searchcore_vespa_spi_feed_bm_app searchcore_grouping searchcore_proton_metrics searchcore_fconfig + storageserver_storageapp + messagebus_messagebus-test + messagebus searchlib_searchlib_uca ) diff --git a/searchcore/src/apps/vespa-spi-feed-bm/vespa_spi_feed_bm.cpp b/searchcore/src/apps/vespa-spi-feed-bm/vespa_spi_feed_bm.cpp index ecfa2a07cef..1bebcee8a1d 100644 --- a/searchcore/src/apps/vespa-spi-feed-bm/vespa_spi_feed_bm.cpp +++ b/searchcore/src/apps/vespa-spi-feed-bm/vespa_spi_feed_bm.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -37,6 +38,40 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include @@ -51,7 +86,28 @@ using namespace vespa::config::search::summary; using namespace vespa::config::search; using namespace std::chrono_literals; using vespa::config::content::core::BucketspacesConfig; +using vespa::config::content::core::BucketspacesConfigBuilder; +using vespa::config::content::StorDistributionConfigBuilder; +using vespa::config::content::StorFilestorConfigBuilder; +using vespa::config::content::PersistenceConfigBuilder; +using vespa::config::content::core::StorBouncerConfigBuilder; +using vespa::config::content::core::StorCommunicationmanagerConfigBuilder; +using vespa::config::content::core::StorBucketInitConfigBuilder; +using vespa::config::content::core::StorOpsloggerConfigBuilder; +using vespa::config::content::core::StorPrioritymappingConfigBuilder; +using vespa::config::content::LoadTypeConfigBuilder; +using vespa::config::content::UpgradingConfigBuilder; +using vespa::config::content::core::StorServerConfigBuilder; +using vespa::config::content::core::StorStatusConfigBuilder; +using vespa::config::content::core::StorVisitorConfigBuilder; +using metrics::MetricsmanagerConfigBuilder; +using cloud::config::SlobroksConfigBuilder; +using messagebus::MessagebusConfigBuilder; +using config::ConfigContext; +using config::ConfigUri; +using config::ConfigSet; +using config::IConfigContext; using document::AssignValueUpdate; using document::BucketId; using document::BucketSpace; @@ -59,17 +115,23 @@ using document::Document; using document::DocumentId; using document::DocumentType; using document::DocumentTypeRepo; +using document::DocumentTypeRepoFactory; using document::DocumenttypesConfig; +using document::DocumenttypesConfigBuilder; using document::DocumentUpdate; using document::Field; using document::FieldUpdate; using document::IntFieldValue; using document::test::makeBucketSpace; +using documentapi::LoadTypeSet; using search::TuneFileDocumentDB; using search::index::DummyFileHeaderContext; using search::index::Schema; using search::index::SchemaBuilder; using search::transactionlog::TransLogServer; +using storage::rpc::MessageCodecProvider; +using storage::rpc::SharedRpcResources; +using storage::rpc::StorageApiRpcService; using storage::spi::Bucket; using storage::spi::PartitionId; using storage::spi::PersistenceProvider; @@ -242,6 +304,7 @@ class BMParams { uint32_t _put_passes; uint32_t _update_passes; uint32_t _remove_passes; + bool _enable_service_layer; uint32_t get_start(uint32_t thread_id) const { return (_documents / _threads) * thread_id + std::min(thread_id, _documents % _threads); } @@ -251,7 +314,8 @@ public: _threads(32), _put_passes(2), _update_passes(1), - _remove_passes(2) + _remove_passes(2), + _enable_service_layer(false) { } BMRange get_range(uint32_t thread_id) const { @@ -262,11 +326,13 @@ public: uint32_t get_put_passes() const { return _put_passes; } uint32_t get_update_passes() const { return _update_passes; } uint32_t get_remove_passes() const { return _remove_passes; } + bool get_enable_service_layer() const { return _enable_service_layer; } void set_documents(uint32_t documents_in) { _documents = documents_in; } void set_threads(uint32_t threads_in) { _threads = threads_in; } void set_put_passes(uint32_t put_passes_in) { _put_passes = put_passes_in; } void set_update_passes(uint32_t update_passes_in) { _update_passes = update_passes_in; } void set_remove_passes(uint32_t remove_passes_in) { _remove_passes = remove_passes_in; } + void set_enable_service_layer(bool enable_service_layer_in) { _enable_service_layer = enable_service_layer_in; } bool check() const; }; @@ -292,8 +358,230 @@ BMParams::check() const return true; } + +class MyServiceLayerProcess : public storage::ServiceLayerProcess { + PersistenceProvider& _provider; + +public: + MyServiceLayerProcess(const config::ConfigUri & configUri, + PersistenceProvider &provider); + ~MyServiceLayerProcess() override { shutdown(); } + + void shutdown() override; + void setupProvider() override; + PersistenceProvider& getProvider() override; +}; + +MyServiceLayerProcess::MyServiceLayerProcess(const config::ConfigUri & configUri, + PersistenceProvider &provider) + : ServiceLayerProcess(configUri), + _provider(provider) +{ +} + +void +MyServiceLayerProcess::shutdown() +{ + ServiceLayerProcess::shutdown(); +} + +void +MyServiceLayerProcess::setupProvider() +{ +} + +PersistenceProvider& +MyServiceLayerProcess::getProvider() +{ + return _provider; +} + +struct MyStorageConfig +{ + vespalib::string config_id; + DocumenttypesConfigBuilder documenttypes; + PersistenceConfigBuilder persistence; + StorDistributionConfigBuilder stor_distribution; + StorFilestorConfigBuilder stor_filestor; + StorBouncerConfigBuilder stor_bouncer; + StorCommunicationmanagerConfigBuilder stor_communicationmanager; + StorBucketInitConfigBuilder stor_bucket_init; + StorOpsloggerConfigBuilder stor_opslogger; + StorPrioritymappingConfigBuilder stor_prioritymapping; + UpgradingConfigBuilder upgrading; + StorServerConfigBuilder stor_server; + StorStatusConfigBuilder stor_status; + StorVisitorConfigBuilder stor_visitor; + BucketspacesConfigBuilder bucketspaces; + LoadTypeConfigBuilder load_type; + MetricsmanagerConfigBuilder metricsmanager; + SlobroksConfigBuilder slobroks; + MessagebusConfigBuilder messagebus; + + MyStorageConfig(const vespalib::string& config_id_in, const DocumenttypesConfig& documenttypes_in, int slobrok_port, int status_port) + : config_id(config_id_in), + documenttypes(documenttypes_in), + persistence(), + stor_distribution(), + stor_filestor(), + stor_bouncer(), + stor_communicationmanager(), + stor_bucket_init(), + stor_opslogger(), + stor_prioritymapping(), + upgrading(), + stor_server(), + stor_status(), + stor_visitor(), + bucketspaces(), + load_type(), + metricsmanager(), + slobroks(), + messagebus() + { + { + auto &dc = stor_distribution; + { + StorDistributionConfigBuilder::Group group; + { + StorDistributionConfigBuilder::Group::Nodes node; + node.index = 0; + group.nodes.push_back(std::move(node)); + } + group.index = "invalid"; + group.name = "invalid"; + group.capacity = 1.0; + group.partitions = ""; + dc.group.push_back(std::move(group)); + } + dc.redundancy = 1; + dc.readyCopies = 1; + } + stor_server.rootFolder = "storage"; + { + SlobroksConfigBuilder::Slobrok slobrok; + slobrok.connectionspec = vespalib::make_string("tcp/localhost:%d", slobrok_port); + slobroks.slobrok.push_back(std::move(slobrok)); + } + stor_communicationmanager.useDirectStorageapiRpc = true; + stor_status.httpport = status_port; + } + + ~MyStorageConfig(); + + void add_builders(ConfigSet &set) { + set.addBuilder(config_id, &documenttypes); + set.addBuilder(config_id, &persistence); + set.addBuilder(config_id, &stor_distribution); + set.addBuilder(config_id, &stor_filestor); + set.addBuilder(config_id, &stor_bouncer); + set.addBuilder(config_id, &stor_communicationmanager); + set.addBuilder(config_id, &stor_bucket_init); + set.addBuilder(config_id, &stor_opslogger); + set.addBuilder(config_id, &stor_prioritymapping); + set.addBuilder(config_id, &upgrading); + set.addBuilder(config_id, &stor_server); + set.addBuilder(config_id, &stor_status); + set.addBuilder(config_id, &stor_visitor); + set.addBuilder(config_id, &bucketspaces); + set.addBuilder(config_id, &load_type); + set.addBuilder(config_id, &metricsmanager); + set.addBuilder(config_id, &slobroks); + set.addBuilder(config_id, &messagebus); + } +}; + +MyStorageConfig::~MyStorageConfig() = default; + +struct MyRpcClientConfig { + vespalib::string config_id; + SlobroksConfigBuilder slobroks; + + MyRpcClientConfig(const vespalib::string &config_id_in, int slobrok_port) + : config_id(config_id_in), + slobroks() + { + { + SlobroksConfigBuilder::Slobrok slobrok; + slobrok.connectionspec = vespalib::make_string("tcp/localhost:%d", slobrok_port); + slobroks.slobrok.push_back(std::move(slobrok)); + } + } + ~MyRpcClientConfig(); + + void add_builders(ConfigSet &set) { + set.addBuilder(config_id, &slobroks); + } +}; + +MyRpcClientConfig::~MyRpcClientConfig() = default; + +class MyMessageDispatcher : public storage::MessageDispatcher +{ + std::mutex _mutex; + vespalib::hash_map _pending; +public: + MyMessageDispatcher() + : storage::MessageDispatcher(), + _mutex(), + _pending() + { + } + ~MyMessageDispatcher() override; + void dispatch_sync(std::shared_ptr msg) override { + release(msg->getMsgId()); + } + void dispatch_async(std::shared_ptr msg) override { + release(msg->getMsgId()); + } + void retain(uint64_t msg_id, MyPendingTracker &tracker) { + tracker.retain(); + std::lock_guard lock(_mutex); + _pending.insert(std::make_pair(msg_id, &tracker)); + } + void release(uint64_t msg_id) { + MyPendingTracker *tracker = nullptr; + { + std::lock_guard lock(_mutex); + auto itr = _pending.find(msg_id); + assert(itr != _pending.end()); + tracker = itr->second; + _pending.erase(itr); + } + tracker->release(); + } +}; + +MyMessageDispatcher::~MyMessageDispatcher() +{ + std::lock_guard lock(_mutex); + assert(_pending.empty()); } +FRT_RPCRequest *make_set_cluster_state_request() { + storage::lib::ClusterStateBundle bundle(storage::lib::ClusterState("version:2 distributor:1 storage:1")); + storage::rpc::SlimeClusterStateBundleCodec codec; + auto encoded_bundle = codec.encode(bundle); + auto *req = new FRT_RPCRequest(); + auto* params = req->GetParams(); + params->AddInt8(static_cast(encoded_bundle._compression_type)); + params->AddInt32(encoded_bundle._uncompressed_length); + const auto buf_len = encoded_bundle._buffer->getDataLen(); + params->AddData(encoded_bundle._buffer->stealBuffer(), buf_len); + req->SetMethodName("setdistributionstates"); + return req; +} + +void set_cluster_up(SharedRpcResources &shared_rpc_resources, storage::api::StorageMessageAddress &storage_address) { + auto req = make_set_cluster_state_request(); + auto target_resolver = std::make_unique(shared_rpc_resources.slobrok_mirror(), shared_rpc_resources.target_factory()); + auto target = target_resolver->resolve_rpc_target(storage_address); + target->_target->get()->InvokeSync(req, 10.0); // 10 seconds timeout + assert(!req->IsError()); + req->SubRef(); +} + +} struct PersistenceProviderFixture { std::shared_ptr _document_types; @@ -305,6 +593,9 @@ struct PersistenceProviderFixture { vespalib::string _base_dir; DummyFileHeaderContext _file_header_context; int _tls_listen_port; + int _slobrok_port; + int _status_port; + int _rpc_client_port; TransLogServer _tls; vespalib::string _tls_spec; matching::QueryLimiter _query_limiter; @@ -320,6 +611,17 @@ struct PersistenceProviderFixture { std::shared_ptr _persistence_engine; storage::spi::Context _context; uint32_t _bucket_bits; + MyStorageConfig _service_layer_config; + MyRpcClientConfig _rpc_client_config; + ConfigSet _config_set; + std::shared_ptr _config_context; + storage::api::StorageMessageAddress _storage_address; + std::unique_ptr _slobrok; + std::unique_ptr _service_layer; + std::unique_ptr _message_codec_provider; + std::unique_ptr _rpc_client_shared_rpc_resources; + std::unique_ptr _rpc_message_dispatcher; + std::unique_ptr _rpc_client; PersistenceProviderFixture(); ~PersistenceProviderFixture(); @@ -331,11 +633,14 @@ struct PersistenceProviderFixture { std::unique_ptr make_document(uint32_t i) const; std::unique_ptr make_document_update(uint32_t i) const; void create_buckets(); + void start_service_layer(); + void shutdown_service_layer(); + void send_rpc(std::shared_ptr cmd, MyPendingTracker& pending_tracker); }; PersistenceProviderFixture::PersistenceProviderFixture() : _document_types(make_document_type()), - _repo(std::make_shared(*_document_types)), + _repo(DocumentTypeRepoFactory::make(*_document_types)), _doc_type_name("test"), _document_type(_repo->getDocumentType(_doc_type_name.getName())), _field(_document_type->getField("int")), @@ -343,6 +648,9 @@ PersistenceProviderFixture::PersistenceProviderFixture() _base_dir(base_dir), _file_header_context(), _tls_listen_port(9017), + _slobrok_port(9018), + _status_port(9019), + _rpc_client_port(9020), _tls("tls", _tls_listen_port, _base_dir, _file_header_context), _tls_spec(vespalib::make_string("tcp/localhost:%d", _tls_listen_port)), _query_limiter(), @@ -357,12 +665,25 @@ PersistenceProviderFixture::PersistenceProviderFixture() _write_filter(), _persistence_engine(), _context(default_load_type, Priority(0), Trace::TraceLevel(0)), - _bucket_bits(16) + _bucket_bits(16), + _service_layer_config("bm-servicelayer", *_document_types, _slobrok_port, _status_port), + _rpc_client_config("bm-rpc-client", _slobrok_port), + _config_set(), + _config_context(std::make_shared(_config_set)), + _storage_address("storage", storage::lib::NodeType::STORAGE, 0), + _slobrok(), + _service_layer(), + _message_codec_provider(), + _rpc_client_shared_rpc_resources(), + _rpc_message_dispatcher(), + _rpc_client() { create_document_db(); _persistence_engine = std::make_unique(_persistence_owner, _write_filter, -1, false); auto proxy = std::make_shared(_document_db); _persistence_engine->putHandler(_persistence_engine->getWLock(), _bucket_space, _doc_type_name, proxy); + _service_layer_config.add_builders(_config_set); + _rpc_client_config.add_builders(_config_set); } PersistenceProviderFixture::~PersistenceProviderFixture() @@ -454,6 +775,58 @@ PersistenceProviderFixture::create_buckets() } } +void +PersistenceProviderFixture::start_service_layer() +{ + LOG(info, "start slobrok"); + _slobrok = std::make_unique(_slobrok_port); + LOG(info, "start service layer"); + config::ConfigUri config_uri("bm-servicelayer", _config_context); + _service_layer = std::make_unique(config_uri, + *_persistence_engine); + _service_layer->setupConfig(100ms); + _service_layer->createNode(); + _service_layer->getNode().waitUntilInitialized(); + _message_codec_provider = std::make_unique(_repo, std::make_shared()); + LOG(info, "start rpc client shared resources"); + config::ConfigUri client_config_uri("bm-rpc-client", _config_context); + _rpc_client_shared_rpc_resources = std::make_unique(client_config_uri, _rpc_client_port, 100); + _rpc_client_shared_rpc_resources->start_server_and_register_slobrok("bm-rpc-client"); + _rpc_message_dispatcher = std::make_unique(); + _rpc_client = std::make_unique(*_rpc_message_dispatcher, *_rpc_client_shared_rpc_resources, *_message_codec_provider, StorageApiRpcService::Params()); + set_cluster_up(*_rpc_client_shared_rpc_resources, _storage_address); +} + +void +PersistenceProviderFixture::shutdown_service_layer() +{ + _rpc_client.reset(); + _rpc_message_dispatcher.reset(); + if (_rpc_client_shared_rpc_resources) { + LOG(info, "stop rpc client shared resources"); + _rpc_client_shared_rpc_resources->shutdown(); + _rpc_client_shared_rpc_resources.reset(); + } + if (_service_layer) { + LOG(info, "stop service layer"); + _service_layer->getNode().requestShutdown("controlled shutdown"); + _service_layer->shutdown(); + } + if (_slobrok) { + LOG(info, "stop slobrok"); + _slobrok.reset(); + } +} + +void +PersistenceProviderFixture::send_rpc(std::shared_ptr cmd, MyPendingTracker& pending_tracker) +{ + cmd->setSourceIndex(0); + cmd->setAddress(_storage_address); + _rpc_message_dispatcher->retain(cmd->getMsgId(), pending_tracker); + _rpc_client->send_rpc_v1_request(std::move(cmd)); +} + vespalib::nbostream make_put_feed(PersistenceProviderFixture &f, BMRange range) { @@ -501,7 +874,12 @@ put_async_task(PersistenceProviderFixture &f, BMRange range, const vespalib::nbo is >> bucket_id; Bucket bucket(document::Bucket(bucket_space, bucket_id), PartitionId(0)); auto document = std::make_unique(repo, is); - provider.putAsync(bucket, Timestamp(time_bias + i), std::move(document), context, std::make_unique(pending_tracker)); + if (f._rpc_client) { + auto cmd = std::make_unique(bucket.getBucket(), std::move(document), time_bias + i); + f.send_rpc(std::move(cmd), pending_tracker); + } else { + provider.putAsync(bucket, Timestamp(time_bias + i), std::move(document), context, std::make_unique(pending_tracker)); + } } assert(is.empty()); pending_tracker.drain(); @@ -552,7 +930,12 @@ update_async_task(PersistenceProviderFixture &f, BMRange range, const vespalib:: is >> bucket_id; Bucket bucket(document::Bucket(bucket_space, bucket_id), PartitionId(0)); auto document_update = DocumentUpdate::createHEAD(repo, is); - provider.updateAsync(bucket, Timestamp(time_bias + i), std::move(document_update), context, std::make_unique(pending_tracker)); + if (f._rpc_client) { + auto cmd = std::make_unique(bucket.getBucket(), std::move(document_update), time_bias + i); + f.send_rpc(std::move(cmd), pending_tracker); + } else { + provider.updateAsync(bucket, Timestamp(time_bias + i), std::move(document_update), context, std::make_unique(pending_tracker)); + } } assert(is.empty()); pending_tracker.drain(); @@ -603,7 +986,12 @@ remove_async_task(PersistenceProviderFixture &f, BMRange range, const vespalib:: is >> bucket_id; Bucket bucket(document::Bucket(bucket_space, bucket_id), PartitionId(0)); DocumentId document_id(is); - provider.removeAsync(bucket, Timestamp(time_bias + i), document_id, context, std::make_unique(pending_tracker)); + if (f._rpc_client) { + auto cmd = std::make_unique(bucket.getBucket(), document_id, time_bias + i); + f.send_rpc(std::move(cmd), pending_tracker); + } else { + provider.removeAsync(bucket, Timestamp(time_bias + i), document_id, context, std::make_unique(pending_tracker)); + } } assert(is.empty()); pending_tracker.drain(); @@ -635,6 +1023,9 @@ void benchmark_async_spi(const BMParams &bm_params) provider.initialize(); LOG(info, "create %u buckets", f.num_buckets()); f.create_buckets(); + if (bm_params.get_enable_service_layer()) { + f.start_service_layer(); + } vespalib::ThreadStackExecutor executor(bm_params.get_threads(), 128 * 1024); auto put_feed = make_feed(executor, bm_params, [&f](BMRange range) { return make_put_feed(f, range); }, "put"); auto update_feed = make_feed(executor, bm_params, [&f](BMRange range) { return make_update_feed(f, range); }, "update"); @@ -649,6 +1040,7 @@ void benchmark_async_spi(const BMParams &bm_params) for (uint32_t pass = 0; pass < bm_params.get_remove_passes(); ++pass) { run_remove_async_tasks(f, executor, pass, time_bias, remove_feed, bm_params); } + f.shutdown_service_layer(); } class App : public FastOS_Application @@ -682,7 +1074,8 @@ App::usage() "[--documents documents]\n" "[--put-passes put-passes]\n" "[--update-passes update-passes]\n" - "[--remove-passes remove-passes]" << std::endl; + "[--remove-passes remove-passes]\n" + "[--enable-service-layer]" << std::endl; } bool @@ -696,14 +1089,16 @@ App::get_options() { "documents", 1, nullptr, 0 }, { "put-passes", 1, nullptr, 0 }, { "update-passes", 1, nullptr, 0 }, - { "remove-passes", 1, nullptr, 0 } + { "remove-passes", 1, nullptr, 0 }, + { "enable-service-layer", 0, nullptr, 0 } }; enum longopts_enum { LONGOPT_THREADS, LONGOPT_DOCUMENTS, LONGOPT_PUT_PASSES, LONGOPT_UPDATE_PASSES, - LONGOPT_REMOVE_PASSES + LONGOPT_REMOVE_PASSES, + LONGOPT_ENABLE_SERVICE_LAYER }; int opt_index = 1; resetOptIndex(opt_index); @@ -726,6 +1121,9 @@ App::get_options() case LONGOPT_REMOVE_PASSES: _bm_params.set_remove_passes(atoi(opt_argument)); break; + case LONGOPT_ENABLE_SERVICE_LAYER: + _bm_params.set_enable_service_layer(true); + break; default: return false; } -- cgit v1.2.3 From 4a42df76dff9b518def12fe7234c8c13fb99703f Mon Sep 17 00:00:00 2001 From: Håvard Pettersen Date: Tue, 22 Sep 2020 14:59:13 +0000 Subject: new value api on top-level - use common TrivialIndex for dense values - added aborts for non-implementing subclasses: SimpleTensor, WrappedSimpleTensor, SparseTensor - renamed conflicting 'cells' functions to 'my_cells' --- .../tests/eval/simple_value/simple_value_test.cpp | 12 ++-- .../direct_sparse_tensor_builder_test.cpp | 2 +- .../packed_mappings/packed_mappings_test.cpp | 2 +- .../tensor/packed_mappings/packed_mixed_test.cpp | 6 +- eval/src/vespa/eval/eval/simple_tensor.cpp | 4 +- eval/src/vespa/eval/eval/simple_tensor.h | 4 +- eval/src/vespa/eval/eval/simple_tensor_engine.cpp | 2 +- eval/src/vespa/eval/eval/simple_value.cpp | 24 +++---- eval/src/vespa/eval/eval/simple_value.h | 74 +++------------------- eval/src/vespa/eval/eval/value.cpp | 34 ++++++++++ eval/src/vespa/eval/eval/value.h | 69 ++++++++++++++++++-- .../vespa/eval/tensor/dense/dense_tensor_view.h | 2 + eval/src/vespa/eval/tensor/join_tensors.h | 8 +-- .../eval/tensor/mixed/packed_mixed_builder.cpp | 2 +- .../vespa/eval/tensor/mixed/packed_mixed_builder.h | 2 +- .../eval/tensor/mixed/packed_mixed_tensor.cpp | 8 +-- .../vespa/eval/tensor/mixed/packed_mixed_tensor.h | 10 +-- .../src/vespa/eval/tensor/sparse/sparse_tensor.cpp | 14 ++-- eval/src/vespa/eval/tensor/sparse/sparse_tensor.h | 4 +- .../eval/tensor/sparse/sparse_tensor_apply.hpp | 8 +-- .../eval/tensor/sparse/sparse_tensor_match.cpp | 10 +-- .../eval/tensor/sparse/sparse_tensor_reduce.hpp | 8 +-- eval/src/vespa/eval/tensor/tensor_apply.cpp | 2 +- .../vespa/eval/tensor/wrapped_simple_tensor.cpp | 4 +- eval/src/vespa/eval/tensor/wrapped_simple_tensor.h | 2 + 25 files changed, 181 insertions(+), 136 deletions(-) diff --git a/eval/src/tests/eval/simple_value/simple_value_test.cpp b/eval/src/tests/eval/simple_value/simple_value_test.cpp index 32a099afce3..8ac553680f7 100644 --- a/eval/src/tests/eval/simple_value/simple_value_test.cpp +++ b/eval/src/tests/eval/simple_value/simple_value_test.cpp @@ -62,17 +62,17 @@ TensorSpec simple_tensor_join(const TensorSpec &a, const TensorSpec &b, join_fun } TensorSpec simple_value_new_join(const TensorSpec &a, const TensorSpec &b, join_fun_t function) { - auto lhs = new_value_from_spec(a, SimpleValueBuilderFactory()); - auto rhs = new_value_from_spec(b, SimpleValueBuilderFactory()); + auto lhs = value_from_spec(a, SimpleValueBuilderFactory()); + auto rhs = value_from_spec(b, SimpleValueBuilderFactory()); auto result = new_join(*lhs, *rhs, function, SimpleValueBuilderFactory()); - return spec_from_new_value(*result); + return spec_from_value(*result); } TEST(SimpleValueTest, simple_values_can_be_converted_from_and_to_tensor_spec) { for (const auto &layout: layouts) { TensorSpec expect = spec(layout, N()); - std::unique_ptr value = new_value_from_spec(expect, SimpleValueBuilderFactory()); - TensorSpec actual = spec_from_new_value(*value); + std::unique_ptr value = value_from_spec(expect, SimpleValueBuilderFactory()); + TensorSpec actual = spec_from_value(*value); EXPECT_EQ(actual, expect); } } @@ -92,7 +92,7 @@ TEST(SimpleValueTest, simple_value_can_be_built_and_inspected) { } seq += 100.0; } - std::unique_ptr value = builder->build(std::move(builder)); + std::unique_ptr value = builder->build(std::move(builder)); EXPECT_EQ(value->index().size(), 6); auto view = value->index().create_view({0}); vespalib::stringref query = "b"; diff --git a/eval/src/tests/tensor/direct_sparse_tensor_builder/direct_sparse_tensor_builder_test.cpp b/eval/src/tests/tensor/direct_sparse_tensor_builder/direct_sparse_tensor_builder_test.cpp index 651451d81f1..e4640cf2c6a 100644 --- a/eval/src/tests/tensor/direct_sparse_tensor_builder/direct_sparse_tensor_builder_test.cpp +++ b/eval/src/tests/tensor/direct_sparse_tensor_builder/direct_sparse_tensor_builder_test.cpp @@ -54,7 +54,7 @@ TEST("require that tensor can be constructed") Tensor::UP tensor = buildTensor(); const SparseTensor &sparseTensor = dynamic_cast(*tensor); const ValueType &type = sparseTensor.type(); - const SparseTensor::Cells &cells = sparseTensor.cells(); + const SparseTensor::Cells &cells = sparseTensor.my_cells(); EXPECT_EQUAL(2u, cells.size()); assertCellValue(10, TensorAddress({{"a","1"},{"b","2"}}), type, cells); assertCellValue(20, TensorAddress({{"c","3"},{"d","4"}}), type, cells); diff --git a/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp b/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp index b8072f92b6c..70f98904809 100644 --- a/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp +++ b/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp @@ -138,7 +138,7 @@ TEST_F(MappingsBuilderTest, some_random) class MixedBuilderTest : public ::testing::Test { public: std::unique_ptr> builder; - std::unique_ptr built; + std::unique_ptr built; MixedBuilderTest() = default; diff --git a/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp b/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp index fa3597e9858..48c72c3c591 100644 --- a/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp +++ b/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp @@ -26,8 +26,8 @@ std::vector layouts = { TEST(PackedMixedTest, packed_mixed_tensors_can_be_converted_from_and_to_tensor_spec) { for (const auto &layout: layouts) { TensorSpec expect = spec(layout, N()); - std::unique_ptr value = new_value_from_spec(expect, PackedMixedFactory()); - TensorSpec actual = spec_from_new_value(*value); + std::unique_ptr value = value_from_spec(expect, PackedMixedFactory()); + TensorSpec actual = spec_from_value(*value); EXPECT_EQ(actual, expect); } } @@ -47,7 +47,7 @@ TEST(PackedMixedTest, packed_mixed_tensors_can_be_built_and_inspected) { } seq += 100.0; } - std::unique_ptr value = builder->build(std::move(builder)); + std::unique_ptr value = builder->build(std::move(builder)); EXPECT_EQ(value->index().size(), 6); auto view = value->index().create_view({0}); vespalib::stringref query = "b"; diff --git a/eval/src/vespa/eval/eval/simple_tensor.cpp b/eval/src/vespa/eval/eval/simple_tensor.cpp index 6ab7df34b2f..64b2b6f8865 100644 --- a/eval/src/vespa/eval/eval/simple_tensor.cpp +++ b/eval/src/vespa/eval/eval/simple_tensor.cpp @@ -336,7 +336,7 @@ public: View(const SimpleTensor &tensor, const IndexList &selector) : _less(selector), _refs() { - for (const auto &cell: tensor.cells()) { + for (const auto &cell: tensor.my_cells()) { _refs.emplace_back(cell); } std::sort(_refs.begin(), _refs.end(), _less); @@ -738,7 +738,7 @@ SimpleTensor::encode(const SimpleTensor &tensor, nbostream &output) Format format(meta); output.putInt1_4Bytes(format.tag); encode_type(output, format, tensor.type(), meta); - maybe_encode_num_blocks(output, meta, tensor.cells().size() / meta.block_size); + maybe_encode_num_blocks(output, meta, tensor.my_cells().size() / meta.block_size); View view(tensor, meta.mapped); for (auto block = view.first_range(); !block.empty(); block = view.next_range(block)) { encode_mapped_labels(output, meta, block.begin()->get().address); diff --git a/eval/src/vespa/eval/eval/simple_tensor.h b/eval/src/vespa/eval/eval/simple_tensor.h index d717838176b..28e11a4ffd3 100644 --- a/eval/src/vespa/eval/eval/simple_tensor.h +++ b/eval/src/vespa/eval/eval/simple_tensor.h @@ -79,11 +79,13 @@ public: using join_fun_t = double (*)(double, double); SimpleTensor(); + TypedCells cells() const override { abort(); } + const Index &index() const override { abort(); } explicit SimpleTensor(double value); SimpleTensor(const ValueType &type_in, Cells cells_in); double as_double() const final override; const ValueType &type() const override { return _type; } - const Cells &cells() const { return _cells; } + const Cells &my_cells() const { return _cells; } std::unique_ptr map(map_fun_t function) const; std::unique_ptr reduce(Aggregator &aggr, const std::vector &dimensions) const; std::unique_ptr rename(const std::vector &from, const std::vector &to) const; diff --git a/eval/src/vespa/eval/eval/simple_tensor_engine.cpp b/eval/src/vespa/eval/eval/simple_tensor_engine.cpp index 6c2a0fcd53d..491a310dc0b 100644 --- a/eval/src/vespa/eval/eval/simple_tensor_engine.cpp +++ b/eval/src/vespa/eval/eval/simple_tensor_engine.cpp @@ -63,7 +63,7 @@ SimpleTensorEngine::to_spec(const Value &value) const const auto &dimensions = value.type().dimensions(); with_simple(value, [&spec,&dimensions](const SimpleTensor &simple_tensor) { - for (const auto &cell: simple_tensor.cells()) { + for (const auto &cell: simple_tensor.my_cells()) { TensorSpec::Address addr; assert(cell.address.size() == dimensions.size()); for (size_t i = 0; i < cell.address.size(); ++i) { diff --git a/eval/src/vespa/eval/eval/simple_value.cpp b/eval/src/vespa/eval/eval/simple_value.cpp index e8ab26078e6..4daa78375e5 100644 --- a/eval/src/vespa/eval/eval/simple_value.cpp +++ b/eval/src/vespa/eval/eval/simple_value.cpp @@ -26,7 +26,7 @@ struct CreateSimpleValueBuilderBase { }; struct CreateValueFromTensorSpec { - template static std::unique_ptr invoke(const ValueType &type, const TensorSpec &spec, const ValueBuilderFactory &factory) { + template static std::unique_ptr invoke(const ValueType &type, const TensorSpec &spec, const ValueBuilderFactory &factory) { using SparseKey = std::vector; using DenseMap = std::map; std::map map; @@ -57,7 +57,7 @@ struct CreateValueFromTensorSpec { }; struct CreateTensorSpecFromValue { - template static TensorSpec invoke(const NewValue &value) { + template static TensorSpec invoke(const Value &value) { auto cells = value.cells().typify(); TensorSpec spec(value.type().to_spec()); size_t subspace_id = 0; @@ -95,7 +95,7 @@ struct CreateTensorSpecFromValue { } }; -class SimpleValueView : public NewValue::Index::View { +class SimpleValueView : public Value::Index::View { private: using Addr = std::vector; using Map = std::map; @@ -174,8 +174,8 @@ public: // index in the largest index. struct SparseJoinState { bool swapped; - const NewValue::Index &first_index; - const NewValue::Index &second_index; + const Value::Index &first_index; + const Value::Index &second_index; const std::vector &second_view_dims; std::vector full_address; std::vector first_address; @@ -186,7 +186,7 @@ struct SparseJoinState { size_t &first_subspace; size_t &second_subspace; - SparseJoinState(const SparseJoinPlan &plan, const NewValue::Index &lhs, const NewValue::Index &rhs) + SparseJoinState(const SparseJoinPlan &plan, const Value::Index &lhs, const Value::Index &rhs) : swapped(rhs.size() < lhs.size()), first_index(swapped ? rhs : lhs), second_index(swapped ? lhs : rhs), second_view_dims(swapped ? plan.lhs_overlap : plan.rhs_overlap), @@ -216,8 +216,8 @@ SparseJoinState::~SparseJoinState() = default; // as input cell types since output cell type cannot always be // directly inferred. struct GenericJoin { - template static std::unique_ptr - invoke(const NewValue &lhs, const NewValue &rhs, join_fun_t function, + template static std::unique_ptr + invoke(const Value &lhs, const Value &rhs, join_fun_t function, const SparseJoinPlan &sparse_plan, const DenseJoinPlan &dense_plan, const ValueType &res_type, const ValueBuilderFactory &factory) { @@ -269,7 +269,7 @@ SimpleValue::SimpleValue(const ValueType &type, size_t num_mapped_dims_in, size_ SimpleValue::~SimpleValue() = default; -std::unique_ptr +std::unique_ptr SimpleValue::create_view(const std::vector &dims) const { return std::make_unique(_index, dims, _num_mapped_dims); @@ -385,7 +385,7 @@ SparseJoinPlan::~SparseJoinPlan() = default; using JoinTypify = TypifyValue; -std::unique_ptr new_join(const NewValue &a, const NewValue &b, join_fun_t function, const ValueBuilderFactory &factory) { +std::unique_ptr new_join(const Value &a, const Value &b, join_fun_t function, const ValueBuilderFactory &factory) { auto res_type = ValueType::join(a.type(), b.type()); assert(!res_type.is_error()); SparseJoinPlan sparse_plan(a.type(), b.type()); @@ -396,7 +396,7 @@ std::unique_ptr new_join(const NewValue &a, const NewValue &b, join_fu //----------------------------------------------------------------------------- -std::unique_ptr new_value_from_spec(const TensorSpec &spec, const ValueBuilderFactory &factory) { +std::unique_ptr value_from_spec(const TensorSpec &spec, const ValueBuilderFactory &factory) { ValueType type = ValueType::from_spec(spec.type()); assert(!type.is_error()); return typify_invoke<1,TypifyCellType,CreateValueFromTensorSpec>(type.cell_type(), type, spec, factory); @@ -404,7 +404,7 @@ std::unique_ptr new_value_from_spec(const TensorSpec &spec, const Valu //----------------------------------------------------------------------------- -TensorSpec spec_from_new_value(const NewValue &value) { +TensorSpec spec_from_value(const Value &value) { return typify_invoke<1,TypifyCellType,CreateTensorSpecFromValue>(value.type().cell_type(), value); } diff --git a/eval/src/vespa/eval/eval/simple_value.h b/eval/src/vespa/eval/eval/simple_value.h index 892dd6f1da6..4c7bccfa01c 100644 --- a/eval/src/vespa/eval/eval/simple_value.h +++ b/eval/src/vespa/eval/eval/simple_value.h @@ -3,8 +3,6 @@ #pragma once #include "value.h" -#include "value_type.h" -#include #include #include #include @@ -17,62 +15,6 @@ class TensorSpec; using TypedCells = ::vespalib::tensor::TypedCells; -/** - * Experimental interface layer that will be moved into Value when all - * existing implementations are able to implement it. This interface - * will try to unify scalars, dense tensors, sparse tensors and mixed - * tensors while also enabling operations to be implemented - * efficiently using this interface without having knowledge about the - * actual implementation. Baseline operations will treat all values as - * mixed tensors. Simplified and optimized variants may replace them - * as done today based on type knowledge. - * - * All values are expected to be separated into a continuous area - * storing cells as concatenated dense subspaces, and an index - * structure used to look up label combinations; mapping them into a - * set of dense subspaces. - **/ -struct NewValue : Value { - - // Root lookup structure for mapping labels to dense subspace indexes - struct Index { - - // A view able to look up dense subspace indexes from labels - // specifying a partial address for the dimensions given to - // create_view. A view is re-usable. Lookups are performed by - // calling the lookup function and lookup results are - // extracted using the next_result function. - struct View { - - // look up dense subspace indexes from labels specifying a - // partial address for the dimensions given to - // create_view. Results from the lookup is extracted using - // the next_result function. - virtual void lookup(const std::vector &addr) = 0; - - // Extract the next result (if any) from the previous - // lookup into the given partial address and index. Only - // the labels for the dimensions NOT specified in - // create_view will be extracted here. - virtual bool next_result(const std::vector &addr_out, size_t &idx_out) = 0; - - virtual ~View() {} - }; - - // total number of mappings (equal to the number of dense subspaces) - virtual size_t size() const = 0; - - // create a view able to look up dense subspaces based on - // labels from a subset of the mapped dimensions. - virtual std::unique_ptr create_view(const std::vector &dims) const = 0; - - virtual ~Index() {} - }; - virtual TypedCells cells() const = 0; - virtual const Index &index() const = 0; - virtual ~NewValue() {} -}; - /** * Tagging interface used as return type from factories before * downcasting to actual builder with specialized cell type. @@ -98,7 +40,7 @@ struct ValueBuilder : ValueBuilderBase { // Given the ownership of the builder itself, produce the newly // created value. This means that builders can only be used once, // it also means values can build themselves. - virtual std::unique_ptr build(std::unique_ptr self) = 0; + virtual std::unique_ptr build(std::unique_ptr self) = 0; }; /** @@ -140,7 +82,7 @@ protected: * test the correctness of tensor operations as they are moved away * from the implementation of individual tensor classes. **/ -class SimpleValue : public NewValue, public NewValue::Index +class SimpleValue : public Value, public Value::Index { private: using Addr = std::vector; @@ -155,7 +97,7 @@ public: SimpleValue(const ValueType &type, size_t num_mapped_dims_in, size_t subspace_size_in); ~SimpleValue() override; const ValueType &type() const override { return _type; } - const NewValue::Index &index() const override { return *this; } + const Value::Index &index() const override { return *this; } size_t size() const override { return _index.size(); } std::unique_ptr create_view(const std::vector &dims) const override; }; @@ -173,11 +115,11 @@ public: ~SimpleValueT() override; TypedCells cells() const override { return TypedCells(ConstArrayRef(_cells)); } ArrayRef add_subspace(const std::vector &addr) override; - std::unique_ptr build(std::unique_ptr> self) override { + std::unique_ptr build(std::unique_ptr> self) override { ValueBuilder* me = this; assert(me == self.get()); self.release(); - return std::unique_ptr(this); + return std::unique_ptr(this); } }; @@ -259,17 +201,17 @@ struct SparseJoinPlan { * dimensional overlap and result type. **/ using join_fun_t = double (*)(double, double); -std::unique_ptr new_join(const NewValue &a, const NewValue &b, join_fun_t function, const ValueBuilderFactory &factory); +std::unique_ptr new_join(const Value &a, const Value &b, join_fun_t function, const ValueBuilderFactory &factory); /** * Make a value from a tensor spec using a value builder factory * interface, making it work with any value implementation. **/ -std::unique_ptr new_value_from_spec(const TensorSpec &spec, const ValueBuilderFactory &factory); +std::unique_ptr value_from_spec(const TensorSpec &spec, const ValueBuilderFactory &factory); /** * Convert a generic value to a tensor spec. **/ -TensorSpec spec_from_new_value(const NewValue &value); +TensorSpec spec_from_value(const Value &value); } diff --git a/eval/src/vespa/eval/eval/value.cpp b/eval/src/vespa/eval/eval/value.cpp index 3629a5ad698..283950a6a67 100644 --- a/eval/src/vespa/eval/eval/value.cpp +++ b/eval/src/vespa/eval/eval/value.cpp @@ -6,6 +6,40 @@ namespace vespalib { namespace eval { +namespace { + +struct TrivialView : Value::Index::View { + bool first = false; + void lookup(const std::vector &) override { first = true; } + bool next_result(const std::vector &, size_t &idx_out) override { + if (first) { + idx_out = 0; + first = false; + return true; + } else { + return false; + } + } +}; + +} // + + +TrivialIndex::TrivialIndex() = default; +TrivialIndex TrivialIndex::_index; + +size_t +TrivialIndex::size() const +{ + return 1; +} + +std::unique_ptr +TrivialIndex::create_view(const std::vector &) const +{ + return std::make_unique(); +} + ValueType DoubleValue::_type = ValueType::double_type(); } // namespace vespalib::eval diff --git a/eval/src/vespa/eval/eval/value.h b/eval/src/vespa/eval/eval/value.h index cad76c93c5c..86c62d58eca 100644 --- a/eval/src/vespa/eval/eval/value.h +++ b/eval/src/vespa/eval/eval/value.h @@ -3,7 +3,10 @@ #pragma once #include "value_type.h" +#include +#include #include +#include #include namespace vespalib::eval { @@ -14,15 +17,71 @@ class Tensor; * An abstract Value. **/ struct Value { - typedef std::unique_ptr UP; - typedef std::reference_wrapper CREF; + using UP = std::unique_ptr; + using CREF = std::reference_wrapper; + using TypedCells = tensor::TypedCells; + virtual const ValueType &type() const = 0; + virtual ~Value() {} + +// ---- new interface enabling separation of values and operations + // Root lookup structure for mapping labels to dense subspace indexes + struct Index { + + // A view able to look up dense subspace indexes from labels + // specifying a partial address for the dimensions given to + // create_view. A view is re-usable. Lookups are performed by + // calling the lookup function and lookup results are + // extracted using the next_result function. + struct View { + + // look up dense subspace indexes from labels specifying a + // partial address for the dimensions given to + // create_view. Results from the lookup is extracted using + // the next_result function. + virtual void lookup(const std::vector &addr) = 0; + + // Extract the next result (if any) from the previous + // lookup into the given partial address and index. Only + // the labels for the dimensions NOT specified in + // create_view will be extracted here. + virtual bool next_result(const std::vector &addr_out, size_t &idx_out) = 0; + + virtual ~View() {} + }; + + // total number of mappings (equal to the number of dense subspaces) + virtual size_t size() const = 0; + + // create a view able to look up dense subspaces based on + // labels from a subset of the mapped dimensions. + virtual std::unique_ptr create_view(const std::vector &dims) const = 0; + + virtual ~Index() {} + }; + virtual TypedCells cells() const = 0; + virtual const Index &index() const = 0; +// --- end of new interface + +// --- old interface that may be (partially) removed in the future virtual bool is_double() const { return false; } virtual bool is_tensor() const { return false; } virtual double as_double() const { return 0.0; } bool as_bool() const { return (as_double() != 0.0); } virtual const Tensor *as_tensor() const { return nullptr; } - virtual const ValueType &type() const = 0; - virtual ~Value() {} +// --- end of old interface +}; + +/** + * Common index for values without any mapped dimensions. + **/ +class TrivialIndex : public Value::Index { +private: + TrivialIndex(); + static TrivialIndex _index; + size_t size() const override; + std::unique_ptr create_view(const std::vector &dims) const override; +public: + static const TrivialIndex &get() { return _index; } }; class DoubleValue : public Value @@ -32,6 +91,8 @@ private: static ValueType _type; public: DoubleValue(double value) : _value(value) {} + TypedCells cells() const override { return TypedCells(ConstArrayRef(&_value, 1)); } + const Index &index() const override { return TrivialIndex::get(); } bool is_double() const override { return true; } double as_double() const override { return _value; } const ValueType &type() const override { return _type; } diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h index cf3e2864a30..f69c068192a 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h @@ -28,6 +28,8 @@ public: const eval::ValueType &fast_type() const { return _typeRef; } const TypedCells &cellsRef() const { return _cellsRef; } + TypedCells cells() const override { return _cellsRef; } + const Index &index() const override { return eval::TrivialIndex::get(); } bool operator==(const DenseTensorView &rhs) const; CellsIterator cellsIterator() const { return CellsIterator(_typeRef, _cellsRef); } diff --git a/eval/src/vespa/eval/tensor/join_tensors.h b/eval/src/vespa/eval/tensor/join_tensors.h index aa493c23656..d66d0c1bf8e 100644 --- a/eval/src/vespa/eval/tensor/join_tensors.h +++ b/eval/src/vespa/eval/tensor/join_tensors.h @@ -18,8 +18,8 @@ joinTensors(const TensorImplType &lhs, Function &&func) { DirectSparseTensorBuilder - builder(lhs.combineDimensionsWith(rhs), lhs.cells()); - for (const auto &rhsCell : rhs.cells()) { + builder(lhs.combineDimensionsWith(rhs), lhs.my_cells()); + for (const auto &rhsCell : rhs.my_cells()) { builder.insertCell(rhsCell.first, rhsCell.second, func); } return builder.build(); @@ -36,8 +36,8 @@ joinTensorsNegated(const TensorImplType &lhs, Function &&func) { DirectSparseTensorBuilder - builder(lhs.combineDimensionsWith(rhs), lhs.cells()); - for (const auto &rhsCell : rhs.cells()) { + builder(lhs.combineDimensionsWith(rhs), lhs.my_cells()); + for (const auto &rhsCell : rhs.my_cells()) { builder.insertCell(rhsCell.first, -rhsCell.second, func); } return builder.build(); diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.cpp index e9390c3bd21..1816e06bf25 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.cpp @@ -19,7 +19,7 @@ PackedMixedBuilder::add_subspace(const std::vector &addr template -std::unique_ptr +std::unique_ptr PackedMixedBuilder::build(std::unique_ptr>) { size_t self_size = sizeof(PackedMixedTensor); diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h index 099f531ae38..c851b839756 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h @@ -34,7 +34,7 @@ public: ~PackedMixedBuilder() override = default; ArrayRef add_subspace(const std::vector &addr) override; - std::unique_ptr build(std::unique_ptr> self) override; + std::unique_ptr build(std::unique_ptr> self) override; }; } // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp index 2587878f16f..43f0c27ffbd 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp @@ -6,7 +6,7 @@ namespace vespalib::eval::packed_mixed_tensor { /*********************************************************************************/ -class PackedMixedTensorIndexView : public NewValue::Index::View +class PackedMixedTensorIndexView : public Value::Index::View { private: const PackedMappings& _mappings; @@ -92,7 +92,7 @@ PackedMixedTensorIndexView::next_result(const std::vector /*********************************************************************************/ -class PackedMixedTensorLookup : public NewValue::Index::View +class PackedMixedTensorLookup : public Value::Index::View { private: const PackedMappings& _mappings; @@ -148,7 +148,7 @@ PackedMixedTensorLookup::next_result(const std::vector &ad /*********************************************************************************/ -class PackedMixedTensorAllMappings : public NewValue::Index::View +class PackedMixedTensorAllMappings : public Value::Index::View { private: const PackedMappings& _mappings; @@ -194,7 +194,7 @@ PackedMixedTensorAllMappings::next_result(const std::vector +std::unique_ptr PackedMixedTensor::create_view(const std::vector &dims) const { if (dims.size() == 0) { diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h index ffef379b37b..60916b23565 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h @@ -12,13 +12,13 @@ namespace vespalib::eval::packed_mixed_tensor { /** - * An implementation of NewValue modeling a mixed tensor, + * An implementation of Value modeling a mixed tensor, * where all the data (cells and sparse address mappings) * can reside in a self-contained, contigous block of memory. * Currently must be built by a PackedMixedBuilder. * Immutable (all data always const). **/ -class PackedMixedTensor : public NewValue, public NewValue::Index +class PackedMixedTensor : public Value, public Value::Index { private: const ValueType _type; @@ -38,12 +38,12 @@ private: public: ~PackedMixedTensor() override; - // NewValue API: + // Value API: const ValueType &type() const override { return _type; } - const NewValue::Index &index() const override { return *this; } + const Value::Index &index() const override { return *this; } TypedCells cells() const override { return _cells; } - // NewValue::Index API: + // Value::Index API: size_t size() const override { return _mappings.size(); } std::unique_ptr create_view(const std::vector &dims) const override; }; diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp index 1538ecdd12f..98a20cd9630 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp @@ -204,18 +204,18 @@ SparseTensor::merge(join_fun_t function, const Tensor &arg) const const SparseTensor *rhs = dynamic_cast(&arg); assert(rhs && (fast_type().dimensions() == rhs->fast_type().dimensions())); DirectSparseTensorBuilder builder(eval::ValueType::merge(fast_type(), rhs->fast_type())); - builder.reserve(cells().size() + rhs->cells().size()); - for (const auto &cell: cells()) { - auto pos = rhs->cells().find(cell.first); - if (pos == rhs->cells().end()) { + builder.reserve(my_cells().size() + rhs->my_cells().size()); + for (const auto &cell: my_cells()) { + auto pos = rhs->my_cells().find(cell.first); + if (pos == rhs->my_cells().end()) { builder.insertCell(cell.first, cell.second); } else { builder.insertCell(cell.first, function(cell.second, pos->second)); } } - for (const auto &cell: rhs->cells()) { - auto pos = cells().find(cell.first); - if (pos == cells().end()) { + for (const auto &cell: rhs->my_cells()) { + auto pos = my_cells().find(cell.first); + if (pos == my_cells().end()) { builder.insertCell(cell.first, cell.second); } } diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h index 34a6622bdd6..002e0dac0ef 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h @@ -34,8 +34,10 @@ private: public: explicit SparseTensor(const eval::ValueType &type_in, const Cells &cells_in); SparseTensor(eval::ValueType &&type_in, Cells &&cells_in, Stash &&stash_in); + TypedCells cells() const override { abort(); } + const Index &index() const override { abort(); } ~SparseTensor() override; - const Cells &cells() const { return _cells; } + const Cells &my_cells() const { return _cells; } const eval::ValueType &fast_type() const { return _type; } bool operator==(const SparseTensor &rhs) const; eval::ValueType combineDimensionsWith(const SparseTensor &rhs) const; diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_apply.hpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_apply.hpp index 9d619f0f41a..8d46e88ca72 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_apply.hpp +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_apply.hpp @@ -14,13 +14,13 @@ apply(const SparseTensor &lhs, const SparseTensor &rhs, Function &&func) { DirectSparseTensorBuilder builder(lhs.combineDimensionsWith(rhs)); TensorAddressCombiner addressCombiner(lhs.fast_type(), rhs.fast_type()); - size_t estimatedCells = (lhs.cells().size() * rhs.cells().size()); + size_t estimatedCells = (lhs.my_cells().size() * rhs.my_cells().size()); if (addressCombiner.numOverlappingDimensions() != 0) { - estimatedCells = std::min(lhs.cells().size(), rhs.cells().size()); + estimatedCells = std::min(lhs.my_cells().size(), rhs.my_cells().size()); } builder.reserve(estimatedCells*2); - for (const auto &lhsCell : lhs.cells()) { - for (const auto &rhsCell : rhs.cells()) { + for (const auto &lhsCell : lhs.my_cells()) { + for (const auto &rhsCell : rhs.my_cells()) { bool combineSuccess = addressCombiner.combine(lhsCell.first, rhsCell.first); if (combineSuccess) { builder.insertCell(addressCombiner.getAddressRef(), diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_match.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_match.cpp index 75eff05cb37..9dc47b0176c 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_match.cpp +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_match.cpp @@ -12,10 +12,10 @@ namespace vespalib::tensor { void SparseTensorMatch::fastMatch(const TensorImplType &lhs, const TensorImplType &rhs) { - _builder.reserve(lhs.cells().size()); - for (const auto &lhsCell : lhs.cells()) { - auto rhsItr = rhs.cells().find(lhsCell.first); - if (rhsItr != rhs.cells().end()) { + _builder.reserve(lhs.my_cells().size()); + for (const auto &lhsCell : lhs.my_cells()) { + auto rhsItr = rhs.my_cells().find(lhsCell.first); + if (rhsItr != rhs.my_cells().end()) { _builder.insertCell(lhsCell.first, lhsCell.second * rhsItr->second); } } @@ -28,7 +28,7 @@ SparseTensorMatch::SparseTensorMatch(const TensorImplType &lhs, const TensorImpl assert (lhs.fast_type().dimensions().size() == _builder.fast_type().dimensions().size()); // Ensure that first tensor to fastMatch has fewest cells. - if (lhs.cells().size() <= rhs.cells().size()) { + if (lhs.my_cells().size() <= rhs.my_cells().size()) { fastMatch(lhs, rhs); } else { fastMatch(rhs, lhs); diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_reduce.hpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_reduce.hpp index 2016dc2207a..f55fec85155 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_reduce.hpp +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_reduce.hpp @@ -12,8 +12,8 @@ std::unique_ptr reduceAll(const SparseTensor &tensor, DirectSparseTensorBuilder &builder, Function &&func) { - auto itr = tensor.cells().begin(); - auto itrEnd = tensor.cells().end(); + auto itr = tensor.my_cells().begin(); + auto itrEnd = tensor.my_cells().end(); double result = 0.0; if (itr != itrEnd) { result = itr->second; @@ -47,8 +47,8 @@ reduce(const SparseTensor &tensor, return reduceAll(tensor, builder, func); } TensorAddressReducer addressReducer(tensor.fast_type(), dimensions); - builder.reserve(tensor.cells().size()*2); - for (const auto &cell : tensor.cells()) { + builder.reserve(tensor.my_cells().size()*2); + for (const auto &cell : tensor.my_cells()) { addressReducer.reduce(cell.first); builder.insertCell(addressReducer.getAddressRef(), cell.second, func); } diff --git a/eval/src/vespa/eval/tensor/tensor_apply.cpp b/eval/src/vespa/eval/tensor/tensor_apply.cpp index 8f0610fed65..98450797f0c 100644 --- a/eval/src/vespa/eval/tensor/tensor_apply.cpp +++ b/eval/src/vespa/eval/tensor/tensor_apply.cpp @@ -10,7 +10,7 @@ TensorApply::TensorApply(const TensorImplType &tensor, const CellFunction &func) : Parent(tensor.fast_type()) { - for (const auto &cell : tensor.cells()) { + for (const auto &cell : tensor.my_cells()) { _builder.insertCell(cell.first, func.apply(cell.second)); } } diff --git a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp index 08fbb945830..241b8026b59 100644 --- a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp +++ b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp @@ -41,7 +41,7 @@ WrappedSimpleTensor::accept(TensorVisitor &visitor) const { TensorAddressBuilder addr; const auto &dimensions = _tensor.type().dimensions(); - for (const auto &cell: _tensor.cells()) { + for (const auto &cell: _tensor.my_cells()) { addr.clear(); for (size_t i = 0; i < dimensions.size(); ++i) { if (dimensions[i].is_indexed()) { @@ -70,7 +70,7 @@ WrappedSimpleTensor::get_memory_usage() const Tensor::UP WrappedSimpleTensor::clone() const { - auto tensor = std::make_unique(_tensor.type(), _tensor.cells()); + auto tensor = std::make_unique(_tensor.type(), _tensor.my_cells()); return std::make_unique(std::move(tensor)); } diff --git a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.h b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.h index cf9103a4895..9c4031ac171 100644 --- a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.h +++ b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.h @@ -26,6 +26,8 @@ public: : _space(), _tensor(tensor) {} explicit WrappedSimpleTensor(std::unique_ptr tensor) : _space(std::move(tensor)), _tensor(*_space) {} + TypedCells cells() const override { abort(); } + const Index &index() const override { abort(); } ~WrappedSimpleTensor() {} const eval::SimpleTensor &get() const { return _tensor; } const eval::ValueType &type() const override { return _tensor.type(); } -- cgit v1.2.3 From 564242a32e28a7a7be44bde93bba62ebf8f56ef5 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Tue, 22 Sep 2020 17:24:49 +0200 Subject: Remove tests Covered by tests in SessionContentHandlerTest --- .../config/server/session/LocalSessionTest.java | 143 --------------------- 1 file changed, 143 deletions(-) delete mode 100644 configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java deleted file mode 100644 index fb268492dd7..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.component.Version; -import com.yahoo.config.application.api.ApplicationFile; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.TenantName; -import com.yahoo.path.Path; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; -import com.yahoo.vespa.config.server.deploy.ZooKeeperClient; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.mock.MockCurator; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.io.IOException; -import java.time.Instant; -import java.util.Collections; -import java.util.Optional; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -/** - * @author Ulf Lilleengen - */ -public class LocalSessionTest { - - private static final File testApp = new File("src/test/apps/app"); - private static final TenantName tenantName = TenantName.from("test_tenant"); - private static final Path tenantPath = Path.createRoot(); - - private TenantRepository tenantRepository; - private Curator curator; - private ConfigCurator configCurator; - - @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); - - @Before - public void setupTest() throws IOException { - curator = new MockCurator(); - TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() - .curator(curator) - .configServerConfig(new ConfigserverConfig.Builder() - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) - .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) - .build()) - .build(); - tenantRepository = new TenantRepository(componentRegistry); - tenantRepository.addTenant(tenantName); - configCurator = ConfigCurator.create(curator); - } - - @Test - public void require_that_session_is_initialized() throws Exception { - LocalSession session = createSession(applicationId(), 2); - assertThat(session.getSessionId(), is(2L)); - session = createSession(applicationId(), Long.MAX_VALUE); - assertThat(session.getSessionId(), is(Long.MAX_VALUE)); - assertThat(session.getActiveSessionAtCreate(), is(0L)); - } - - @Test - public void require_that_marking_session_modified_changes_status_to_new() throws Exception { - LocalSession session = createSession(applicationId(), 3); - doPrepare(session, applicationId()); - assertThat(session.getStatus(), is(Session.Status.PREPARE)); - session.getApplicationFile(Path.createRoot(), Session.Mode.READ); - assertThat(session.getStatus(), is(Session.Status.PREPARE)); - session.getApplicationFile(Path.createRoot(), Session.Mode.WRITE); - assertThat(session.getStatus(), is(Session.Status.NEW)); - } - - @Test - public void require_that_application_file_can_be_fetched() throws Exception { - LocalSession session = createSession(applicationId(), 3); - ApplicationFile f1 = session.getApplicationFile(Path.fromString("services.xml"), Session.Mode.READ); - ApplicationFile f2 = session.getApplicationFile(Path.fromString("services2.xml"), Session.Mode.READ); - assertTrue(f1.exists()); - assertFalse(f2.exists()); - } - - @Test(expected = IllegalStateException.class) - public void require_that_no_provision_info_throws_exception() throws Exception { - createSession(applicationId(), 3).getAllocatedHosts(); - } - - private LocalSession createSession(ApplicationId applicationId, long sessionId) throws Exception { - return createSession(applicationId, sessionId, Optional.empty()); - } - - private LocalSession createSession(ApplicationId applicationId, long sessionId, - Optional allocatedHosts) throws Exception { - TenantName tenantName = applicationId.tenant(); - SessionZooKeeperClient zkc = new MockSessionZKClient(curator, tenantName, sessionId, allocatedHosts); - zkc.createWriteStatusTransaction(Session.Status.NEW).commit(); - ZooKeeperClient zkClient = new ZooKeeperClient(configCurator, new BaseDeployLogger(), - TenantRepository.getSessionsPath(tenantName).append(String.valueOf(sessionId))); - if (allocatedHosts.isPresent()) { - zkClient.write(allocatedHosts.get()); - } - zkClient.write(Collections.singletonMap(new Version(0, 0, 0), new MockFileRegistry())); - TenantApplications applications = tenantRepository.getTenant(tenantName).getApplicationRepo(); - applications.createApplication(applicationId); - LocalSession session = new LocalSession(tenantName, sessionId, FilesApplicationPackage.fromFile(testApp), zkc); - session.setApplicationId(applicationId); - return session; - } - - private void doPrepare(LocalSession session, ApplicationId applicationId) { - doPrepare(session, new PrepareParams.Builder().applicationId(applicationId).build()); - } - - private void doPrepare(LocalSession session, PrepareParams params) { - SessionRepository sessionRepository = tenantRepository.getTenant(params.getApplicationId().tenant()).getSessionRepository(); - sessionRepository.prepareLocalSession(session, getLogger(), params, Optional.empty(), tenantPath, Instant.now()); - } - - private DeployHandlerLogger getLogger() { - return new DeployHandlerLogger(new Slime().get(), false, applicationId()); - } - - private ApplicationId applicationId() { - return new ApplicationId.Builder().tenant(tenantName).applicationName("testapp").build(); - } - -} -- cgit v1.2.3 From 297842af8553e3360ff3cab4ec67f7f54af42095 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 22 Sep 2020 18:36:09 +0200 Subject: Revert "- Group commits to the TLS and sync to disk before acking." --- .../vespa/searchcore/proton/server/feedhandler.cpp | 32 +---------- .../vespa/searchcore/proton/server/feedhandler.h | 5 -- .../searchcore/proton/server/i_operation_storer.h | 4 -- .../src/vespa/searchlib/config/translogserver.def | 6 +- .../src/vespa/searchlib/transactionlog/domain.cpp | 66 ++-------------------- .../src/vespa/searchlib/transactionlog/domain.h | 5 -- .../vespa/searchlib/transactionlog/domainpart.cpp | 12 +--- .../searchlib/transactionlog/translogserver.cpp | 7 +-- 8 files changed, 14 insertions(+), 123 deletions(-) diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 209a35ce4a2..37dfddf0c2c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -402,8 +402,6 @@ FeedHandler::FeedHandler(IThreadingService &writeService, _tlsReplayProgress(), _serialNum(0), _prunedSerialNum(0), - _numPendingCommit(0), - _numCommitsCompleted(0), _delayedPrune(false), _feedLock(), _feedState(make_shared(getDocTypeName())), @@ -496,40 +494,12 @@ FeedHandler::getTransactionLogReplayDone() const { return _tlsMgr.getReplayDone(); } -void -FeedHandler::onCommitDone(uint64_t numPendingAtStart) { - assert(numPendingAtStart <= _numPendingCommit); - _numPendingCommit -= numPendingAtStart; - if (_numPendingCommit > 0) { - enqueCommitTask(); - } -} - -void FeedHandler::enqueCommitTask() { - _writeService.master().execute(makeLambdaTask([this]() { initiateCommit(); })); -} - -void -FeedHandler::initiateCommit() { - auto commitResult = _tlsWriter->startCommit(std::make_shared( - _writeService.master(), - makeLambdaTask([this, numPendingAtStart=_numPendingCommit]() { - onCommitDone(numPendingAtStart); - }))); - if (_activeFeedView) { - _activeFeedView->forceCommit(_serialNum, std::make_shared>(std::move(commitResult))); - } -} - void FeedHandler::appendOperation(const FeedOperation &op, TlsWriter::DoneCallback onDone) { if (!op.getSerialNum()) { const_cast(op).setSerialNum(incSerialNum()); } _tlsWriter->appendOperation(op, std::move(onDone)); - if (++_numPendingCommit == 1) { - enqueCommitTask(); - } } FeedHandler::CommitResult @@ -540,7 +510,7 @@ FeedHandler::startCommit(DoneCallback onDone) { void FeedHandler::storeOperationSync(const FeedOperation &op) { vespalib::Gate gate; - appendAndCommitOperation(op, make_shared(gate)); + appendOperation(op, make_shared(gate)); gate.await(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index 97629bfc018..4807c596130 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -76,8 +76,6 @@ private: // the serial num of the last message in the transaction log SerialNum _serialNum; SerialNum _prunedSerialNum; - size_t _numPendingCommit; - size_t _numCommitsCompleted; bool _delayedPrune; mutable std::shared_mutex _feedLock; FeedStateSP _feedState; @@ -127,9 +125,6 @@ private: FeedStateSP getFeedState() const; void changeFeedState(FeedStateSP newState); void doChangeFeedState(FeedStateSP newState); - void onCommitDone(uint64_t numPendingAtStart); - void initiateCommit(); - void enqueCommitTask(); public: FeedHandler(const FeedHandler &) = delete; FeedHandler & operator = (const FeedHandler &) = delete; diff --git a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h index c3b76a9db75..b276779c2ee 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h @@ -22,10 +22,6 @@ struct IOperationStorer */ virtual void appendOperation(const FeedOperation &op, DoneCallback onDone) = 0; [[nodiscard]] virtual CommitResult startCommit(DoneCallback onDone) = 0; - void appendAndCommitOperation(const FeedOperation &op, DoneCallback onDone) { - appendOperation(op, onDone); - (void) startCommit(std::move(onDone)); - } }; } // namespace proton diff --git a/searchlib/src/vespa/searchlib/config/translogserver.def b/searchlib/src/vespa/searchlib/config/translogserver.def index defce8c3421..38741745773 100644 --- a/searchlib/src/vespa/searchlib/config/translogserver.def +++ b/searchlib/src/vespa/searchlib/config/translogserver.def @@ -15,7 +15,7 @@ basedir string default="tmp" restart ## Use fsync after each commit. ## If not the below interval is used. -usefsync bool default=false +usefsync bool default=false restart ##Number of threads available for visiting/subscription. maxthreads int default=4 restart @@ -24,12 +24,12 @@ maxthreads int default=4 restart crcmethod enum {ccitt_crc32, xxh64} default=xxh64 ## Control compression type. -compression.type enum {NONE, NONE_MULTI, LZ4, ZSTD} default=NONE +compression.type enum {NONE, NONE_MULTI, LZ4, ZSTD} default=LZ4 ## Control compression level ## LZ4 has normal range 1..9 while ZSTD has range 1..19 ## 9 is a reasonable default for both -compression.level int default=3 +compression.level int default=9 ## How large a chunk can grow in memory before beeing flushed chunk.sizelimit int default = 256000 # 256k diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index bd7feec0598..9e0f1a8a1aa 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -113,12 +113,7 @@ Domain::addPart(SerialNum partId, bool isLastPart) { } } -Domain::~Domain() { - MonitorGuard guard(_currentChunkMonitor); - guard.broadcast(); - commitChunk(grabCurrentChunk(guard), guard); - _singleCommitter->shutdown().sync(); -} +Domain::~Domain() { } DomainInfo Domain::getDomainInfo() const @@ -323,73 +318,22 @@ Domain::optionallyRotateFile(SerialNum serialNum) { return dp; } -void -Domain::append(const Packet & packet, Writer::DoneCallback onDone) { - vespalib::MonitorGuard guard(_currentChunkMonitor); - if (_lastSerial >= packet.range().from()) { - throw runtime_error(fmt("Incoming serial number(%" PRIu64 ") must be bigger than the last one (%" PRIu64 ").", - packet.range().from(), _lastSerial)); - } else { - _lastSerial = packet.range().to(); - } - _currentChunk->add(packet, std::move(onDone)); - commitIfFull(guard); -} - Domain::CommitResult Domain::startCommit(DoneCallback onDone) { - vespalib::MonitorGuard guard(_currentChunkMonitor); - if ( !_currentChunk->empty() ) { - auto completed = grabCurrentChunk(guard); - completed->setCommitDoneCallback(std::move(onDone)); - CommitResult result(completed->createCommitResult()); - commitChunk(std::move(completed), guard); - return result; - } + (void) onDone; return CommitResult(); } void -Domain::commitIfFull(const vespalib::MonitorGuard &guard) { - if (_currentChunk->sizeBytes() > _config.getChunkSizeLimit()) { - auto completed = std::move(_currentChunk); - _currentChunk = std::make_unique(_config.getChunkSizeLimit(), completed->stealCallbacks()); - commitChunk(std::move(completed), guard); - } -} - -std::unique_ptr -Domain::grabCurrentChunk(const vespalib::MonitorGuard & guard) { - assert(guard.monitors(_currentChunkMonitor)); - auto chunk = std::move(_currentChunk); - _currentChunk = createCommitChunk(_config); - return chunk; -} - -void -Domain::commitChunk(std::unique_ptr chunk, const vespalib::MonitorGuard & chunkOrderGuard) { - assert(chunkOrderGuard.monitors(_currentChunkMonitor)); - _singleCommitter->execute( makeLambdaTask([this, chunk = std::move(chunk)]() mutable { - doCommit(std::move(chunk)); - })); -} - -void -Domain::doCommit(std::unique_ptr chunk) { - const Packet & packet = chunk->getPacket(); - if (packet.empty()) return; - +Domain::append(const Packet & packet, Writer::DoneCallback onDone) +{ + (void) onDone; vespalib::nbostream_longlivedbuf is(packet.getHandle().data(), packet.getHandle().size()); Packet::Entry entry; entry.deserialize(is); DomainPart::SP dp = optionallyRotateFile(entry.serial()); dp->commit(entry.serial(), packet); - if (_config.getFSyncOnCommit()) { - dp->sync(); - } cleanSessions(); - LOG(debug, "Releasing %zu acks and %zu entries and %zu bytes.", - chunk->getNumCallBacks(), chunk->getPacket().size(), chunk->sizeBytes()); } bool diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h index 041ec27cf23..7e77e6ef0ef 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h @@ -56,11 +56,6 @@ public: uint64_t size() const; Domain & setConfig(const DomainConfig & cfg); private: - void commitIfFull(const vespalib::MonitorGuard & guard); - - std::unique_ptr grabCurrentChunk(const vespalib::MonitorGuard & guard); - void commitChunk(std::unique_ptr chunk, const vespalib::MonitorGuard & chunkOrderGuard); - void doCommit(std::unique_ptr chunk); SerialNum begin(const vespalib::LockGuard & guard) const; SerialNum end(const vespalib::LockGuard & guard) const; size_t byteSize(const vespalib::LockGuard & guard) const; diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp index b7e02894e6b..8855183226d 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp @@ -252,7 +252,7 @@ DomainPart::buildPacketMapping(bool allowTruncate) DomainPart::DomainPart(const string & name, const string & baseDir, SerialNum s, Encoding encoding, uint8_t compressionLevel, const FileHeaderContext &fileHeaderContext, bool allowTruncate) - : _encoding(encoding), + : _encoding(encoding.getCrc(), Encoding::Compression::none), //TODO We do not yet support compression _compressionLevel(compressionLevel), _lock(), _fileLock(), @@ -396,19 +396,16 @@ DomainPart::commit(SerialNum firstSerial, const Packet &packet) if (_range.from() == 0) { _range.from(firstSerial); } - IChunk::UP chunk = IChunk::create(_encoding, _compressionLevel); for (size_t i(0); h.size() > 0; i++) { //LOG(spam, //"Pos(%d) Len(%d), Lim(%d), Remaining(%d)", //h.getPos(), h.getLength(), h.getLimit(), h.getRemaining()); + IChunk::UP chunk = IChunk::create(_encoding, _compressionLevel); Packet::Entry entry; entry.deserialize(h); if (_range.to() < entry.serial()) { chunk->add(entry); - if (_encoding.getCompression() == Encoding::Compression::none) { - write(*_transLog, *chunk); - chunk = IChunk::create(_encoding, _compressionLevel); - } + write(*_transLog, *chunk); _sz++; _range.to(entry.serial()); } else { @@ -416,9 +413,6 @@ DomainPart::commit(SerialNum firstSerial, const Packet &packet) entry.serial(), _range.to())); } } - if ( ! chunk->getEntries().empty()) { - write(*_transLog, *chunk); - } bool merged(false); LockGuard guard(_lock); diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index 0c0c9186e12..7be3dd708a5 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -572,11 +572,8 @@ TransLogServer::domainCommit(FRT_RPCRequest *req) Packet packet(params[1]._data._buf, params[1]._data._len); try { vespalib::Gate gate; - { - // Need to scope in order to drain out all the callbacks. - domain->append(packet, make_shared(gate)); - auto keep = domain->startCommit(make_shared()); - } + domain->append(packet, make_shared(gate)); + auto keep = domain->startCommit(make_shared()); gate.await(); ret.AddInt32(0); ret.AddString("ok"); -- cgit v1.2.3 From 6d0897b79ff7ba6d2f801d6d094382b4d2ff2aa8 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 22 Sep 2020 18:51:44 +0200 Subject: Allow no progress for 4h during upgrades --- .../controller/deployment/InternalStepRunner.java | 2 +- .../JobControllerApiHandlerHelperTest.java | 2 +- .../responses/deployment-overview-2.json | 28 +++++----- .../responses/dev-aws-us-east-2a-runs.json | 4 +- .../responses/overview-user-instance.json | 4 +- .../application/responses/staging-runs.json | 12 ++--- .../application/responses/staging-test-log.json | 62 +++++++++++----------- 7 files changed, 57 insertions(+), 57 deletions(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 3ebc8240889..af9acc7c3a8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -987,7 +987,7 @@ public class InternalStepRunner implements StepRunner { Duration endpointCertificate() { return Duration.ofMinutes(20); } Duration tester() { return Duration.ofMinutes(30); } Duration nodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 60); } - Duration noNodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 120); } + Duration noNodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 240); } Duration testerCertificate() { return Duration.ofMinutes(300); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index 3d583f12a1c..182af3e47ab 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -88,7 +88,7 @@ public class JobControllerApiHandlerHelperTest { assertEquals(deploymentFailed, tester.jobs().last(app.instanceId(), productionUsEast3).get().status()); tester.runner().run(); - tester.clock().advance(Duration.ofHours(2).plusSeconds(1)); + tester.clock().advance(Duration.ofHours(4).plusSeconds(1)); tester.runner().run(); assertEquals(installationFailed, tester.jobs().last(app.instanceId(), productionUsWest1).get().status()); assertEquals(revision2, app.deployment(productionUsCentral1.zone(tester.controller().system())).applicationVersion()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json index e45bd190d5f..b8c48eb3d0c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json @@ -128,8 +128,8 @@ { "id": 3, "url": "https://some.url:43/instance/default/job/system-test/run/3", - "start": 7203000, - "end": 7203000, + "start": 14403000, + "end": 14403000, "status": "success", "versions": { "targetPlatform": "6.1.0", @@ -320,9 +320,9 @@ "dependencies": [], "declared": true, "instance": "default", - "readyAt": 7953000, - "delayedUntil": 7953000, - "coolingDownUntil": 7953000, + "readyAt": 15153000, + "delayedUntil": 15153000, + "coolingDownUntil": 15153000, "jobName": "staging-test", "url": "https://some.url:43/instance/default/job/staging-test", "environment": "staging", @@ -351,8 +351,8 @@ { "id": 5, "url": "https://some.url:43/instance/default/job/staging-test/run/5", - "start": 7303000, - "end": 7303000, + "start": 14503000, + "end": 14503000, "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", @@ -432,8 +432,8 @@ { "id": 4, "url": "https://some.url:43/instance/default/job/staging-test/run/4", - "start": 7203000, - "end": 7203000, + "start": 14403000, + "end": 14403000, "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", @@ -513,8 +513,8 @@ { "id": 3, "url": "https://some.url:43/instance/default/job/staging-test/run/3", - "start": 7203000, - "end": 7203000, + "start": 14403000, + "end": 14403000, "status": "success", "versions": { "targetPlatform": "6.1.0", @@ -756,7 +756,7 @@ ], "declared": true, "instance": "default", - "readyAt": 7203000, + "readyAt": 14403000, "jobName": "production-us-central-1", "url": "https://some.url:43/instance/default/job/production-us-central-1", "environment": "prod", @@ -773,7 +773,7 @@ { "id": 3, "url": "https://some.url:43/instance/default/job/production-us-central-1/run/3", - "start": 7203000, + "start": 14403000, "status": "running", "versions": { "targetPlatform": "6.1.0", @@ -1050,7 +1050,7 @@ "id": 2, "url": "https://some.url:43/instance/default/job/production-us-west-1/run/2", "start": 1000, - "end": 7202000, + "end": 14402000, "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json index 2053b5a80b1..3a78f8c44a0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json @@ -2,8 +2,8 @@ "1": { "id": 1, "status": "success", - "start": 7303000, - "end": 7303000, + "start": 14503000, + "end": 14503000, "wantedPlatform": "7.1", "wantedApplication": { "hash": "unknown" diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json index 285e7f14e8f..2601937faee 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json @@ -8,8 +8,8 @@ "targetApplication": {}, "targetPlatform": "7.1.0" }, - "start": 7303000, - "end": 7303000, + "start": 14503000, + "end": 14503000, "id": 1, "steps": [ { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json index 2ad35968732..37ae9e4b56b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json @@ -94,8 +94,8 @@ "3": { "id": 3, "status": "success", - "start": 7203000, - "end": 7203000, + "start": 14403000, + "end": 14403000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -146,8 +146,8 @@ "4": { "id": 4, "status": "installationFailed", - "start": 7203000, - "end": 7203000, + "start": 14403000, + "end": 14403000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -194,8 +194,8 @@ "5": { "id": 5, "status": "installationFailed", - "start": 7303000, - "end": 7303000, + "start": 14503000, + "end": 14503000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json index a941b824b53..e8c8e57aa9f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json @@ -4,127 +4,127 @@ "log": { "deployTester": [ { - "at": 7303000, + "at": 14503000, "type": "info", "message": "No services requiring restart." }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "Deployment successful." }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "foo" } ], "installTester": [ { - "at": 7303000, + "at": 14503000, "type": "info", "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- platform dockerImage:6.1" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- platform dockerImage:6.1" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- platform dockerImage:6.1" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" } ], "deployInitialReal": [ { - "at": 7303000, + "at": 14503000, "type": "info", "message": "Deploying platform version 6.1 and application version 1.0.1-commit1 ..." }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "No services requiring restart." }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "Deployment successful." }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "foo" } ], "installInitialReal": [ { - "at": 7303000, + "at": 14503000, "type": "info", "message": "######## Details for all nodes ########" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "host-tenant:application:default-staging.us-east-3: unorchestrated" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- platform dockerImage:6.1" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "Deployment expired before installation was successful." } ], "deactivateReal": [ { - "at": 7303000, + "at": 14503000, "type": "info", "message": "Deactivating deployment of tenant.application in staging.us-east-3 ..." } ], "deactivateTester": [ { - "at": 7303000, + "at": 14503000, "type": "info", "message": "Deactivating tester of tenant.application in staging.us-east-3 ..." } @@ -134,19 +134,19 @@ "steps": { "deployTester": { "status": "succeeded", - "startMillis": 7303000 + "startMillis": 14503000 }, "installTester": { "status": "unfinished", - "startMillis": 7303000 + "startMillis": 14503000 }, "deployInitialReal": { "status": "succeeded", - "startMillis": 7303000 + "startMillis": 14503000 }, "installInitialReal": { "status": "failed", - "startMillis": 7303000, + "startMillis": 14503000, "convergence": { "nodes": 1, "down": 0, @@ -182,19 +182,19 @@ }, "copyVespaLogs": { "status": "succeeded", - "startMillis": 7303000 + "startMillis": 14503000 }, "deactivateReal": { "status": "succeeded", - "startMillis": 7303000 + "startMillis": 14503000 }, "deactivateTester": { "status": "succeeded", - "startMillis": 7303000 + "startMillis": 14503000 }, "report": { "status": "succeeded", - "startMillis": 7303000 + "startMillis": 14503000 } } } -- cgit v1.2.3 From 08b9abe47a4a2315a91d999d6ce3d4aceacd25ef Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 22 Sep 2020 17:48:45 +0000 Subject: Add feature flag control over fsync and compression type in the backend TLS. --- .../com/yahoo/config/model/api/ModelContext.java | 2 ++ .../yahoo/config/model/deploy/TestProperties.java | 14 ++++++++ .../vespa/model/content/ContentSearchCluster.java | 2 +- .../vespa/model/search/TransactionLogServer.java | 19 ++++++++-- .../vespa/model/search/test/SearchNodeTest.java | 40 +++++++++++++++++++++- .../config/server/deploy/ModelContextImpl.java | 9 ++++- .../src/main/java/com/yahoo/vespa/flags/Flags.java | 12 +++++++ .../src/vespa/searchlib/config/translogserver.def | 4 +-- 8 files changed, 95 insertions(+), 7 deletions(-) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index e4c203fda6b..6a7a9ed16ae 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -84,6 +84,8 @@ public interface ModelContext { boolean skipCommunicationManagerThread(); boolean skipMbusRequestThread(); boolean skipMbusReplyThread(); + boolean tlsUseFSync(); + String tlsCompressionType(); double visibilityDelay(); boolean useContentNodeBtreeDb(); diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index d9cebd2b1d2..31c72e1be69 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -38,6 +38,8 @@ public class TestProperties implements ModelContext.Properties { private boolean useContentNodeBtreeDb = false; private boolean useThreePhaseUpdates = false; private boolean useDirectStorageApiRpc = false; + private boolean tlsUseFSync = false; + private String tlsCompressionType = "NONE"; private double defaultTermwiseLimit = 1.0; private double threadPoolSizeFactor = 0.0; private double queueSizeFactor = 0.0; @@ -85,6 +87,8 @@ public class TestProperties implements ModelContext.Properties { @Override public boolean skipMbusReplyThread() { return false; } @Override public Quota quota() { return quota; } @Override public double visibilityDelay() { return visibilityDelay; } + @Override public boolean tlsUseFSync() { return tlsUseFSync; } + @Override public String tlsCompressionType() { return tlsCompressionType; } public TestProperties setJvmGCOptions(String gcOptions) { jvmGCOptions = gcOptions; @@ -141,6 +145,16 @@ public class TestProperties implements ModelContext.Properties { return this; } + public TestProperties setTlsUseFSync(boolean useFSync) { + this.tlsUseFSync = useFSync; + return this; + } + + public TestProperties setTlsCompressionType(String type) { + this.tlsCompressionType = type; + return this; + } + public TestProperties setVisibilityDelay(double visibilityDelay) { this.visibilityDelay = visibilityDelay; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 50a6054d1b9..2198b6e278b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -268,7 +268,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot searchNode.setHostResource(node.getHostResource()); searchNode.initService(deployState.getDeployLogger()); - tls = new TransactionLogServer(searchNode, clusterName); + tls = new TransactionLogServer(searchNode, clusterName, deployState.getProperties()); tls.setHostResource(searchNode.getHostResource()); tls.initService(deployState.getDeployLogger()); } else { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java b/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java index 2c457940f24..347250176e5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.search; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.searchlib.TranslogserverConfig; import com.yahoo.config.model.producer.AbstractConfigProducer; @@ -15,12 +16,24 @@ import org.w3c.dom.Element; public class TransactionLogServer extends AbstractService { private static final long serialVersionUID = 1L; + private final boolean useFSync; + private final TranslogserverConfig.Compression.Type.Enum compressionType; - public TransactionLogServer(AbstractConfigProducer searchNode, String clusterName) { + private static TranslogserverConfig.Compression.Type.Enum convertCompressionType(String type) { + try { + return TranslogserverConfig.Compression.Type.Enum.valueOf(type); + } catch (Throwable t) { + return TranslogserverConfig.Compression.Type.NONE; + } + } + + public TransactionLogServer(AbstractConfigProducer searchNode, String clusterName, ModelContext.Properties featureFlags) { super(searchNode, "transactionlogserver"); portsMeta.on(0).tag("tls"); setProp("clustername", clusterName); setProp("clustertype", "search"); + useFSync = featureFlags.tlsUseFSync(); + compressionType = convertCompressionType(featureFlags.tlsCompressionType()); } public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder { @@ -31,7 +44,7 @@ public class TransactionLogServer extends AbstractService { @Override protected TransactionLogServer doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element producerSpec) { - return new TransactionLogServer(ancestor, clusterName); + return new TransactionLogServer(ancestor, clusterName, deployState.getProperties()); } } @@ -65,6 +78,8 @@ public class TransactionLogServer extends AbstractService { public void getConfig(TranslogserverConfig.Builder builder) { builder.listenport(getTlsPort()).basedir(getTlsDir()); + builder.usefsync(useFSync); + builder.compression.type(compressionType); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java index 9c69ba8f212..a04a4f196cf 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java @@ -1,8 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.search.test; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.test.MockRoot; +import com.yahoo.searchlib.TranslogserverConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.Host; @@ -36,7 +40,7 @@ public class SearchNodeTest { private void prepare(MockRoot root, SearchNode node) { Host host = new Host(root, "mockhost"); - TransactionLogServer tls = new TransactionLogServer(root, "mycluster"); + TransactionLogServer tls = new TransactionLogServer(root, "mycluster", root.getDeployState().getProperties()); tls.setHostResource(new HostResource(host)); tls.setBasePort(100); tls.initService(root.deployLogger()); @@ -52,6 +56,10 @@ public class SearchNodeTest { return SearchNode.create(parent, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown, Optional.empty(), Optional.empty(), isHosted, combined); } + private static SearchNode createSearchNode(MockRoot root) { + return createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), true, true, false); + } + @Test public void requireThatBasedirIsCorrectForElasticMode() { MockRoot root = new MockRoot(""); @@ -80,4 +88,34 @@ public class SearchNodeTest { CoreMatchers.containsString("vespa-proton-cmd " + node.getRpcPort() + " prepareRestart")); } + private MockRoot createRoot(ModelContext.Properties properties) { + return new MockRoot("", new DeployState.Builder().properties(properties).build()); + } + + private TranslogserverConfig getTlsConfig(ModelContext.Properties properties) { + MockRoot root = createRoot(properties); + SearchNode node = createSearchNode(root); + prepare(root, node); + TranslogserverConfig.Builder tlsBuilder = new TranslogserverConfig.Builder(); + node.getConfig(tlsBuilder); + return tlsBuilder.build(); + } + + @Test + public void requireThaFeatureFlagCanControlTlsUseFSync() { + assertFalse(getTlsConfig(new TestProperties()).usefsync()); + assertFalse(getTlsConfig(new TestProperties().setTlsUseFSync(false)).usefsync()); + assertTrue(getTlsConfig(new TestProperties().setTlsUseFSync(true)).usefsync()); + } + + @Test + public void requireThaFeatureFlagCanControlCompressionType() { + assertEquals(TranslogserverConfig.Compression.Type.NONE, getTlsConfig(new TestProperties()).compression().type()); + assertEquals(TranslogserverConfig.Compression.Type.NONE, getTlsConfig(new TestProperties().setTlsCompressionType("NONE")).compression().type()); + assertEquals(TranslogserverConfig.Compression.Type.NONE_MULTI, getTlsConfig(new TestProperties().setTlsCompressionType("NONE_MULTI")).compression().type()); + assertEquals(TranslogserverConfig.Compression.Type.ZSTD, getTlsConfig(new TestProperties().setTlsCompressionType("ZSTD")).compression().type()); + assertEquals(TranslogserverConfig.Compression.Type.LZ4, getTlsConfig(new TestProperties().setTlsCompressionType("LZ4")).compression().type()); + assertEquals(TranslogserverConfig.Compression.Type.NONE, getTlsConfig(new TestProperties().setTlsCompressionType("zstd")).compression().type()); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 4ac20f6220f..2c6b30a2e6e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -166,6 +166,8 @@ public class ModelContextImpl implements ModelContext { private final double feedCoreThreadPoolSizeFactor; private final double visibilityDelay; private final Quota quota; + private final boolean tlsUseFSync; + private final String tlsCompressionType; public Properties(ApplicationId applicationId, boolean multitenantFromConfig, @@ -207,6 +209,10 @@ public class ModelContextImpl implements ModelContext { .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); visibilityDelay = Flags.VISIBILITY_DELAY.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + tlsCompressionType = Flags.TLS_COMPRESSION_TYPE.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + tlsUseFSync = Flags.TLS_USE_FSYNC.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); queueSizefactor = Flags.DEFAULT_QUEUE_SIZE_FACTOR.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); jvmGCOPtions = Flags.JVM_GC_OPTIONS.bindTo(flagSource) @@ -317,7 +323,8 @@ public class ModelContextImpl implements ModelContext { @Override public boolean skipMbusReplyThread() { return skipMbusReplyThread; } @Override public double feedCoreThreadPoolSizeFactor() { return feedCoreThreadPoolSizeFactor; } @Override public double visibilityDelay() { return visibilityDelay; } - + @Override public boolean tlsUseFSync() { return tlsUseFSync; } + @Override public String tlsCompressionType() { return tlsCompressionType; } @Override public Quota quota() { return quota; } } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index a6748667b7e..560ec2a271b 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -196,6 +196,18 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundStringFlag TLS_COMPRESSION_TYPE = defineStringFlag( + "tls-compression-type", "NONE", + "Selects type of compression, valid values are NONE, NONE_MULTI, LZ4, ZSTD", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + + public static final UnboundBooleanFlag TLS_USE_FSYNC = defineFeatureFlag( + "tls-use-fsync", false, + "Whether to use fsync when writing to the TLS.", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundDoubleFlag VISIBILITY_DELAY = defineDoubleFlag( "visibility-delay", 0.0, "Default visibility-delay", diff --git a/searchlib/src/vespa/searchlib/config/translogserver.def b/searchlib/src/vespa/searchlib/config/translogserver.def index 38741745773..540895b2404 100644 --- a/searchlib/src/vespa/searchlib/config/translogserver.def +++ b/searchlib/src/vespa/searchlib/config/translogserver.def @@ -24,12 +24,12 @@ maxthreads int default=4 restart crcmethod enum {ccitt_crc32, xxh64} default=xxh64 ## Control compression type. -compression.type enum {NONE, NONE_MULTI, LZ4, ZSTD} default=LZ4 +compression.type enum {NONE, NONE_MULTI, LZ4, ZSTD} default=NONE ## Control compression level ## LZ4 has normal range 1..9 while ZSTD has range 1..19 ## 9 is a reasonable default for both -compression.level int default=9 +compression.level int default=3 ## How large a chunk can grow in memory before beeing flushed chunk.sizelimit int default = 256000 # 256k -- cgit v1.2.3 From 619f3240aee9398afef7707d85f0c1e9734c8c3b Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 22 Sep 2020 21:12:58 +0200 Subject: Revert "Allow no progress for 4h during upgrades" --- .../controller/deployment/InternalStepRunner.java | 2 +- .../JobControllerApiHandlerHelperTest.java | 2 +- .../responses/deployment-overview-2.json | 28 +++++----- .../responses/dev-aws-us-east-2a-runs.json | 4 +- .../responses/overview-user-instance.json | 4 +- .../application/responses/staging-runs.json | 12 ++--- .../application/responses/staging-test-log.json | 62 +++++++++++----------- 7 files changed, 57 insertions(+), 57 deletions(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index af9acc7c3a8..3ebc8240889 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -987,7 +987,7 @@ public class InternalStepRunner implements StepRunner { Duration endpointCertificate() { return Duration.ofMinutes(20); } Duration tester() { return Duration.ofMinutes(30); } Duration nodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 60); } - Duration noNodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 240); } + Duration noNodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 120); } Duration testerCertificate() { return Duration.ofMinutes(300); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index 182af3e47ab..3d583f12a1c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -88,7 +88,7 @@ public class JobControllerApiHandlerHelperTest { assertEquals(deploymentFailed, tester.jobs().last(app.instanceId(), productionUsEast3).get().status()); tester.runner().run(); - tester.clock().advance(Duration.ofHours(4).plusSeconds(1)); + tester.clock().advance(Duration.ofHours(2).plusSeconds(1)); tester.runner().run(); assertEquals(installationFailed, tester.jobs().last(app.instanceId(), productionUsWest1).get().status()); assertEquals(revision2, app.deployment(productionUsCentral1.zone(tester.controller().system())).applicationVersion()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json index b8c48eb3d0c..e45bd190d5f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json @@ -128,8 +128,8 @@ { "id": 3, "url": "https://some.url:43/instance/default/job/system-test/run/3", - "start": 14403000, - "end": 14403000, + "start": 7203000, + "end": 7203000, "status": "success", "versions": { "targetPlatform": "6.1.0", @@ -320,9 +320,9 @@ "dependencies": [], "declared": true, "instance": "default", - "readyAt": 15153000, - "delayedUntil": 15153000, - "coolingDownUntil": 15153000, + "readyAt": 7953000, + "delayedUntil": 7953000, + "coolingDownUntil": 7953000, "jobName": "staging-test", "url": "https://some.url:43/instance/default/job/staging-test", "environment": "staging", @@ -351,8 +351,8 @@ { "id": 5, "url": "https://some.url:43/instance/default/job/staging-test/run/5", - "start": 14503000, - "end": 14503000, + "start": 7303000, + "end": 7303000, "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", @@ -432,8 +432,8 @@ { "id": 4, "url": "https://some.url:43/instance/default/job/staging-test/run/4", - "start": 14403000, - "end": 14403000, + "start": 7203000, + "end": 7203000, "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", @@ -513,8 +513,8 @@ { "id": 3, "url": "https://some.url:43/instance/default/job/staging-test/run/3", - "start": 14403000, - "end": 14403000, + "start": 7203000, + "end": 7203000, "status": "success", "versions": { "targetPlatform": "6.1.0", @@ -756,7 +756,7 @@ ], "declared": true, "instance": "default", - "readyAt": 14403000, + "readyAt": 7203000, "jobName": "production-us-central-1", "url": "https://some.url:43/instance/default/job/production-us-central-1", "environment": "prod", @@ -773,7 +773,7 @@ { "id": 3, "url": "https://some.url:43/instance/default/job/production-us-central-1/run/3", - "start": 14403000, + "start": 7203000, "status": "running", "versions": { "targetPlatform": "6.1.0", @@ -1050,7 +1050,7 @@ "id": 2, "url": "https://some.url:43/instance/default/job/production-us-west-1/run/2", "start": 1000, - "end": 14402000, + "end": 7202000, "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json index 3a78f8c44a0..2053b5a80b1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json @@ -2,8 +2,8 @@ "1": { "id": 1, "status": "success", - "start": 14503000, - "end": 14503000, + "start": 7303000, + "end": 7303000, "wantedPlatform": "7.1", "wantedApplication": { "hash": "unknown" diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json index 2601937faee..285e7f14e8f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json @@ -8,8 +8,8 @@ "targetApplication": {}, "targetPlatform": "7.1.0" }, - "start": 14503000, - "end": 14503000, + "start": 7303000, + "end": 7303000, "id": 1, "steps": [ { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json index 37ae9e4b56b..2ad35968732 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json @@ -94,8 +94,8 @@ "3": { "id": 3, "status": "success", - "start": 14403000, - "end": 14403000, + "start": 7203000, + "end": 7203000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -146,8 +146,8 @@ "4": { "id": 4, "status": "installationFailed", - "start": 14403000, - "end": 14403000, + "start": 7203000, + "end": 7203000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -194,8 +194,8 @@ "5": { "id": 5, "status": "installationFailed", - "start": 14503000, - "end": 14503000, + "start": 7303000, + "end": 7303000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json index e8c8e57aa9f..a941b824b53 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json @@ -4,127 +4,127 @@ "log": { "deployTester": [ { - "at": 14503000, + "at": 7303000, "type": "info", "message": "No services requiring restart." }, { - "at": 14503000, + "at": 7303000, "type": "info", "message": "Deployment successful." }, { - "at": 14503000, + "at": 7303000, "type": "info", "message": "foo" } ], "installTester": [ { - "at": 14503000, + "at": 7303000, "type": "info", "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" }, { - "at": 14503000, + "at": 7303000, "type": "info", "message": "--- platform dockerImage:6.1" }, { - "at": 14503000, + "at": 7303000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": 14503000, + "at": 7303000, "type": "info", "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" }, { - "at": 14503000, + "at": 7303000, "type": "info", "message": "--- platform dockerImage:6.1" }, { - "at": 14503000, + "at": 7303000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": 14503000, + "at": 7303000, "type": "info", "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" }, { - "at": 14503000, + "at": 7303000, "type": "info", "message": "--- platform dockerImage:6.1" }, { - "at": 14503000, + "at": 7303000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" } ], "deployInitialReal": [ { - "at": 14503000, + "at": 7303000, "type": "info", "message": "Deploying platform version 6.1 and application version 1.0.1-commit1 ..." }, { - "at": 14503000, + "at": 7303000, "type": "info", "message": "No services requiring restart." }, { - "at": 14503000, + "at": 7303000, "type": "info", "message": "Deployment successful." }, { - "at": 14503000, + "at": 7303000, "type": "info", "message": "foo" } ], "installInitialReal": [ { - "at": 14503000, + "at": 7303000, "type": "info", "message": "######## Details for all nodes ########" }, { - "at": 14503000, + "at": 7303000, "type": "info", "message": "host-tenant:application:default-staging.us-east-3: unorchestrated" }, { - "at": 14503000, + "at": 7303000, "type": "info", "message": "--- platform dockerImage:6.1" }, { - "at": 14503000, + "at": 7303000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": 14503000, + "at": 7303000, "type": "info", "message": "Deployment expired before installation was successful." } ], "deactivateReal": [ { - "at": 14503000, + "at": 7303000, "type": "info", "message": "Deactivating deployment of tenant.application in staging.us-east-3 ..." } ], "deactivateTester": [ { - "at": 14503000, + "at": 7303000, "type": "info", "message": "Deactivating tester of tenant.application in staging.us-east-3 ..." } @@ -134,19 +134,19 @@ "steps": { "deployTester": { "status": "succeeded", - "startMillis": 14503000 + "startMillis": 7303000 }, "installTester": { "status": "unfinished", - "startMillis": 14503000 + "startMillis": 7303000 }, "deployInitialReal": { "status": "succeeded", - "startMillis": 14503000 + "startMillis": 7303000 }, "installInitialReal": { "status": "failed", - "startMillis": 14503000, + "startMillis": 7303000, "convergence": { "nodes": 1, "down": 0, @@ -182,19 +182,19 @@ }, "copyVespaLogs": { "status": "succeeded", - "startMillis": 14503000 + "startMillis": 7303000 }, "deactivateReal": { "status": "succeeded", - "startMillis": 14503000 + "startMillis": 7303000 }, "deactivateTester": { "status": "succeeded", - "startMillis": 14503000 + "startMillis": 7303000 }, "report": { "status": "succeeded", - "startMillis": 14503000 + "startMillis": 7303000 } } } -- cgit v1.2.3 From 36bffe2d20a7223cf4e6e6b52ff25ee0f94cd7c4 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Tue, 22 Sep 2020 22:07:55 +0200 Subject: Update tests --- .../config/server/ApplicationRepositoryTest.java | 1 + .../config/server/ConfigServerBootstrapTest.java | 1 + .../vespa/config/server/deploy/DeployTester.java | 3 ++- .../config/server/deploy/HostedDeployTest.java | 1 + .../server/http/HttpGetConfigHandlerTest.java | 11 ++++++---- .../server/http/HttpListConfigsHandlerTest.java | 11 ++++++---- .../http/v2/ApplicationContentHandlerTest.java | 23 ++++++++++++++++++--- .../server/http/v2/ApplicationHandlerTest.java | 17 +++++++++++++-- .../config/server/http/v2/HostHandlerTest.java | 15 +++++++++++++- .../server/http/v2/HttpGetConfigHandlerTest.java | 11 ++++++---- .../server/http/v2/HttpListConfigsHandlerTest.java | 11 ++++++---- .../server/http/v2/SessionActiveHandlerTest.java | 10 ++++++++- .../server/http/v2/SessionContentHandlerTest.java | 21 ++++++++++++++++--- .../server/http/v2/SessionPrepareHandlerTest.java | 24 ++++++++++++++++++---- .../config/server/http/v2/TenantHandlerTest.java | 20 ++++++++++++++++-- .../server/maintenance/MaintainerTester.java | 1 + .../yahoo/vespa/config/server/rpc/RpcTester.java | 3 ++- .../config/server/session/SessionPreparerTest.java | 7 ++++++- 18 files changed, 156 insertions(+), 35 deletions(-) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 4770b0797eb..3789e540fc3 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -382,6 +382,7 @@ public class ApplicationRepositoryTest { new ConfigserverConfig(new ConfigserverConfig.Builder() .configServerDBDir(serverdb.getAbsolutePath()) .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder("filedistribution").getAbsolutePath()) .sessionLifetime(60)); DeployTester tester = new DeployTester(configserverConfig, clock); tester.deployApp("src/test/apps/app", clock.instant()); // session 2 (numbering starts at 2) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index 23323d11f76..bda17faec12 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -220,6 +220,7 @@ public class ConfigServerBootstrapTest { return new ConfigserverConfig(new ConfigserverConfig.Builder() .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath()) .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder("filedistribution").getAbsolutePath()) .hostedVespa(hosted) .multitenant(hosted) .maxDurationOfBootstrap(2) /* seconds */ diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index d92245bf5c1..b657ff2a5f8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -78,7 +78,8 @@ public class DeployTester { this(modelFactories, new ConfigserverConfig(new ConfigserverConfig.Builder() .configServerDBDir(uncheck(() -> Files.createTempDirectory("serverdb")).toString()) - .configDefinitionsDir(uncheck(() -> Files.createTempDirectory("configdefinitions")).toString())), + .configDefinitionsDir(uncheck(() -> Files.createTempDirectory("configdefinitions")).toString()) + .fileReferencesDir(uncheck(() -> Files.createTempDirectory("configdefinitions")).toString())), Clock.systemUTC()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index 254fe62cba8..5b995ce55e6 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -393,6 +393,7 @@ public class HostedDeployTest { return new ConfigserverConfig(new ConfigserverConfig.Builder() .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) .hostedVespa(true) .multitenant(true) .region(zone.region().value()) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java index 40aee72e71c..ea896469f03 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java @@ -48,12 +48,14 @@ public class HttpGetConfigHandlerTest { @Before public void setUp() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .configDefinitionRepo(new TestConfigDefinitionRepo()) - .configServerConfig(new ConfigserverConfig.Builder() - .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) - .build()) + .configServerConfig(configserverConfig) .build(); TenantRepository tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenant); @@ -61,6 +63,7 @@ public class HttpGetConfigHandlerTest { .withTenantRepository(tenantRepository) .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) + .withConfigserverConfig(configserverConfig) .build(); handler = new HttpGetConfigHandler(HttpGetConfigHandler.testOnlyContext(), tenantRepository); applicationRepository.deploy(testApp, prepareParams()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java index 906716125c2..9cf8f7960a3 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java @@ -53,12 +53,14 @@ public class HttpListConfigsHandlerTest { @Before public void setUp() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .configDefinitionRepo(new TestConfigDefinitionRepo()) - .configServerConfig(new ConfigserverConfig.Builder() - .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) - .build()) + .configServerConfig(configserverConfig) .build(); TenantRepository tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenant); @@ -66,6 +68,7 @@ public class HttpListConfigsHandlerTest { .withTenantRepository(tenantRepository) .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) + .withConfigserverConfig(configserverConfig) .build(); applicationRepository.deploy(testApp, prepareParams()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java index 67ac0b02133..ab5303b221e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; @@ -17,7 +18,9 @@ import com.yahoo.vespa.config.server.session.Session; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; @@ -35,8 +38,7 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { private static final File testApp = new File("src/test/apps/content"); private static final File testApp2 = new File("src/test/apps/content2"); - private final TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().build(); - private final Clock clock = componentRegistry.getClock(); + private final TenantName tenantName1 = TenantName.from("mofet"); private final TenantName tenantName2 = TenantName.from("bla"); @@ -48,8 +50,22 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { private ApplicationRepository applicationRepository; private ApplicationHandler handler; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before - public void setupHandler() { + public void setupHandler() throws IOException { + + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); + TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() + .configServerConfig(configserverConfig) + .build(); + Clock clock = componentRegistry.getClock(); + TenantRepository tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenantName1); tenantRepository.addTenant(tenantName2); @@ -59,6 +75,7 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withClock(clock) + .withConfigserverConfig(configserverConfig) .build(); applicationRepository.deploy(testApp, prepareParams(appId1)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index d84b81bd8e7..9e4ca8dfb2c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.http.v2; import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.config.model.api.ModelFactory; import com.yahoo.config.provision.ApplicationId; @@ -31,7 +32,9 @@ import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import org.junit.After; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import javax.ws.rs.client.Client; import java.io.ByteArrayInputStream; @@ -73,12 +76,21 @@ public class ApplicationHandlerTest { private SessionHandlerTest.MockProvisioner provisioner; private OrchestratorMock orchestrator; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before - public void setup() { + public void setup() throws IOException { List modelFactories = List.of(DeployTester.createModelFactory(vespaVersion)); + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .provisioner(provisioner) .modelFactoryRegistry(new ModelFactoryRegistry(modelFactories)) + .configServerConfig(configserverConfig) .build(); tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(mytenantName); @@ -91,6 +103,7 @@ public class ApplicationHandlerTest { .withClock(componentRegistry.getClock()) .withTesterClient(testerClient) .withLogRetriever(logRetriever) + .withConfigserverConfig(configserverConfig) .build(); } @@ -357,7 +370,7 @@ public class ApplicationHandlerTest { assertEquals(200, response.getStatus()); String renderedString = SessionHandlerTest.getRenderedString(response); assertEquals("{\"generation\":" + expectedGeneration + - ",\"applicationPackageFileReference\":\"\"" + + ",\"applicationPackageFileReference\":\"./\"" + ",\"modelVersions\":[\"" + expectedVersion.toFullString() + "\"]}", renderedString); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java index 4d64721f7dd..6b9abf5d7ba 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; @@ -19,7 +20,9 @@ import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; @@ -40,10 +43,19 @@ public class HostHandlerTest { private final static Zone zone = Zone.defaultZone(); private ApplicationRepository applicationRepository; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before - public void setup() { + public void setup() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .zone(zone) + .configServerConfig(configserverConfig) .build(); TenantRepository tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(mytenant); @@ -51,6 +63,7 @@ public class HostHandlerTest { .withTenantRepository(tenantRepository) .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) + .withConfigserverConfig(configserverConfig) .build(); handler = new HostHandler(HostHandler.testOnlyContext(), applicationRepository); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java index 14d7a0743a5..9bb113875b4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java @@ -57,12 +57,14 @@ public class HttpGetConfigHandlerTest { @Before public void setUp() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .configDefinitionRepo(new TestConfigDefinitionRepo()) - .configServerConfig(new ConfigserverConfig.Builder() - .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) - .build()) + .configServerConfig(configserverConfig) .build(); TenantRepository tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenant); @@ -70,6 +72,7 @@ public class HttpGetConfigHandlerTest { .withTenantRepository(tenantRepository) .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) + .withConfigserverConfig(configserverConfig) .build(); handler = new HttpGetConfigHandler(HttpGetConfigHandler.testOnlyContext(), tenantRepository); applicationRepository.deploy(testApp, prepareParams()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java index 785c9977fd2..c1adec3336d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java @@ -61,12 +61,14 @@ public class HttpListConfigsHandlerTest { @Before public void setUp() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .configDefinitionRepo(new TestConfigDefinitionRepo()) - .configServerConfig(new ConfigserverConfig.Builder() - .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) - .build()) + .configServerConfig(configserverConfig) .build(); TenantRepository tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenant); @@ -74,6 +76,7 @@ public class HttpListConfigsHandlerTest { .withTenantRepository(tenantRepository) .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) + .withConfigserverConfig(configserverConfig) .build(); applicationRepository.deploy(testApp, prepareParams()); handler = new HttpListConfigsHandler(HttpListConfigsHandler.testOnlyContext(), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index c3a7e82dff5..4ac1d633e75 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.provision.ApplicationId; @@ -65,12 +66,18 @@ public class SessionActiveHandlerTest { public TemporaryFolder temporaryFolder = new TemporaryFolder(); @Before - public void setup() { + public void setup() throws IOException { VespaModelFactory modelFactory = new TestModelFactory(Version.fromString("7.222.2")); hostProvisioner = new SessionHandlerTest.MockProvisioner(); + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); componentRegistry = new TestComponentRegistry.Builder() .curator(new MockCurator()) .modelFactoryRegistry(new ModelFactoryRegistry(List.of((modelFactory)))) + .configServerConfig(configserverConfig) .build(); TenantRepository tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenantName); @@ -79,6 +86,7 @@ public class SessionActiveHandlerTest { .withProvisioner(hostProvisioner) .withOrchestrator(new OrchestratorMock()) .withClock(componentRegistry.getClock()) + .withConfigserverConfig(configserverConfig) .build(); handler = createHandler(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java index d28404d8d72..6de85f12765 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; @@ -19,7 +20,9 @@ import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import org.junit.Before; import org.junit.Ignore; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.ByteArrayInputStream; import java.io.File; @@ -37,14 +40,25 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase { private static final TenantName tenantName = TenantName.from("contenttest"); private static final File testApp = new File("src/test/apps/content"); - private final TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().build(); - + private TestComponentRegistry componentRegistry; private TenantRepository tenantRepository; private SessionContentHandler handler = null; private long sessionId; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before - public void setupHandler() { + public void setupHandler() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); + componentRegistry = new TestComponentRegistry.Builder() + .configServerConfig(configserverConfig) + .build(); + tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenantName); @@ -52,6 +66,7 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase { .withTenantRepository(tenantRepository) .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) + .withConfigserverConfig(configserverConfig) .build(); applicationRepository.deploy(testApp, new PrepareParams.Builder().applicationId(applicationId()).build()); Tenant tenant = applicationRepository.getTenant(applicationId()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java index cc4f39b0789..f1abddba63c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java @@ -25,7 +25,9 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.ByteArrayOutputStream; import java.io.File; @@ -54,17 +56,30 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { private static final File app = new File("src/test/resources/deploy/validapp"); private final Curator curator = new MockCurator(); - private final TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().curator(curator).build(); - private final Clock clock = componentRegistry.getClock(); - private final TimeoutBudget timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(10)); + private TimeoutBudget timeoutBudget; private ApplicationRepository applicationRepository; + private TestComponentRegistry componentRegistry; private String preparedMessage = " prepared.\"}"; private String tenantMessage = ""; private TenantRepository tenantRepository; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before - public void setupRepo() { + public void setupRepo() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); + componentRegistry = new TestComponentRegistry.Builder() + .curator(curator) + .configServerConfig(configserverConfig) + .build(); + Clock clock = componentRegistry.getClock(); + timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(10)); tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenant); applicationRepository = new ApplicationRepository.Builder() @@ -72,6 +87,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withClock(clock) + .withConfigserverConfig(configserverConfig) .build(); pathPrefix = "/application/v2/tenant/" + tenant + "/session/"; preparedMessage = " for tenant '" + tenant + "' prepared.\""; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java index ecab121e547..e2eeb68a565 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java @@ -10,6 +10,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.Clock; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; @@ -25,12 +26,14 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.After; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.vespa.config.server.http.BadRequestException; import com.yahoo.vespa.config.server.http.NotFoundException; +import org.junit.rules.TemporaryFolder; public class TenantHandlerTest { @@ -41,13 +44,26 @@ public class TenantHandlerTest { private TenantHandler handler; private final TenantName a = TenantName.from("a"); + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before - public void setup() { - tenantRepository = new TenantRepository(new TestComponentRegistry.Builder().curator(new MockCurator()).build()); + public void setup() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); + tenantRepository = new TenantRepository(new TestComponentRegistry.Builder() + .curator(new MockCurator()) + .configServerConfig(configserverConfig) + .build()); + applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) + .withConfigserverConfig(configserverConfig) .build(); handler = new TenantHandler(TenantHandler.testOnlyContext(), applicationRepository); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java index 78d69b75d59..0bc23b4d442 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java @@ -47,6 +47,7 @@ class MaintainerTester { .hostedVespa(true) .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) .build(); GlobalComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .curator(curator) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java index eb06f2f7017..47217491e3c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java @@ -82,7 +82,8 @@ public class RpcTester implements AutoCloseable { spec = createSpec(port); configBuilder.rpcport(port) .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()); + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()); configserverConfig = new ConfigserverConfig(configBuilder); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .configDefinitionRepo(new TestConfigDefinitionRepo()) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index b181ad3e8d6..c1698718ad1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.application.provider.BaseDeployLogger; @@ -65,6 +66,7 @@ import java.time.temporal.ChronoUnit; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.logging.Level; @@ -181,7 +183,10 @@ public class SessionPreparerTest { .dryRun(true) .build(), 1); - assertTrue(result.getFileRegistries().get(version321).export().isEmpty()); + Map fileRegistries = result.getFileRegistries(); + System.out.println(fileRegistries); + assertEquals(1, fileRegistries.get(version321).export().size()); + assertEquals("./", fileRegistries.get(version321).export().get(0).reference.value()); } @Test -- cgit v1.2.3 From 9905ffa957edab94cbe9e0129ccbcb4c4052801f Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 22 Sep 2020 19:51:01 +0200 Subject: Revert "Revert "- Group commits to the TLS and sync to disk before acking."" --- .../vespa/searchcore/proton/server/feedhandler.cpp | 32 ++++++++++- .../vespa/searchcore/proton/server/feedhandler.h | 5 ++ .../searchcore/proton/server/i_operation_storer.h | 4 ++ .../src/vespa/searchlib/config/translogserver.def | 2 +- .../src/vespa/searchlib/transactionlog/domain.cpp | 66 ++++++++++++++++++++-- .../src/vespa/searchlib/transactionlog/domain.h | 5 ++ .../vespa/searchlib/transactionlog/domainpart.cpp | 12 +++- .../searchlib/transactionlog/translogserver.cpp | 7 ++- 8 files changed, 121 insertions(+), 12 deletions(-) diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 37dfddf0c2c..209a35ce4a2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -402,6 +402,8 @@ FeedHandler::FeedHandler(IThreadingService &writeService, _tlsReplayProgress(), _serialNum(0), _prunedSerialNum(0), + _numPendingCommit(0), + _numCommitsCompleted(0), _delayedPrune(false), _feedLock(), _feedState(make_shared(getDocTypeName())), @@ -494,12 +496,40 @@ FeedHandler::getTransactionLogReplayDone() const { return _tlsMgr.getReplayDone(); } +void +FeedHandler::onCommitDone(uint64_t numPendingAtStart) { + assert(numPendingAtStart <= _numPendingCommit); + _numPendingCommit -= numPendingAtStart; + if (_numPendingCommit > 0) { + enqueCommitTask(); + } +} + +void FeedHandler::enqueCommitTask() { + _writeService.master().execute(makeLambdaTask([this]() { initiateCommit(); })); +} + +void +FeedHandler::initiateCommit() { + auto commitResult = _tlsWriter->startCommit(std::make_shared( + _writeService.master(), + makeLambdaTask([this, numPendingAtStart=_numPendingCommit]() { + onCommitDone(numPendingAtStart); + }))); + if (_activeFeedView) { + _activeFeedView->forceCommit(_serialNum, std::make_shared>(std::move(commitResult))); + } +} + void FeedHandler::appendOperation(const FeedOperation &op, TlsWriter::DoneCallback onDone) { if (!op.getSerialNum()) { const_cast(op).setSerialNum(incSerialNum()); } _tlsWriter->appendOperation(op, std::move(onDone)); + if (++_numPendingCommit == 1) { + enqueCommitTask(); + } } FeedHandler::CommitResult @@ -510,7 +540,7 @@ FeedHandler::startCommit(DoneCallback onDone) { void FeedHandler::storeOperationSync(const FeedOperation &op) { vespalib::Gate gate; - appendOperation(op, make_shared(gate)); + appendAndCommitOperation(op, make_shared(gate)); gate.await(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index 4807c596130..97629bfc018 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -76,6 +76,8 @@ private: // the serial num of the last message in the transaction log SerialNum _serialNum; SerialNum _prunedSerialNum; + size_t _numPendingCommit; + size_t _numCommitsCompleted; bool _delayedPrune; mutable std::shared_mutex _feedLock; FeedStateSP _feedState; @@ -125,6 +127,9 @@ private: FeedStateSP getFeedState() const; void changeFeedState(FeedStateSP newState); void doChangeFeedState(FeedStateSP newState); + void onCommitDone(uint64_t numPendingAtStart); + void initiateCommit(); + void enqueCommitTask(); public: FeedHandler(const FeedHandler &) = delete; FeedHandler & operator = (const FeedHandler &) = delete; diff --git a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h index b276779c2ee..c3b76a9db75 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h @@ -22,6 +22,10 @@ struct IOperationStorer */ virtual void appendOperation(const FeedOperation &op, DoneCallback onDone) = 0; [[nodiscard]] virtual CommitResult startCommit(DoneCallback onDone) = 0; + void appendAndCommitOperation(const FeedOperation &op, DoneCallback onDone) { + appendOperation(op, onDone); + (void) startCommit(std::move(onDone)); + } }; } // namespace proton diff --git a/searchlib/src/vespa/searchlib/config/translogserver.def b/searchlib/src/vespa/searchlib/config/translogserver.def index 540895b2404..defce8c3421 100644 --- a/searchlib/src/vespa/searchlib/config/translogserver.def +++ b/searchlib/src/vespa/searchlib/config/translogserver.def @@ -15,7 +15,7 @@ basedir string default="tmp" restart ## Use fsync after each commit. ## If not the below interval is used. -usefsync bool default=false restart +usefsync bool default=false ##Number of threads available for visiting/subscription. maxthreads int default=4 restart diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index 9e0f1a8a1aa..bd7feec0598 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -113,7 +113,12 @@ Domain::addPart(SerialNum partId, bool isLastPart) { } } -Domain::~Domain() { } +Domain::~Domain() { + MonitorGuard guard(_currentChunkMonitor); + guard.broadcast(); + commitChunk(grabCurrentChunk(guard), guard); + _singleCommitter->shutdown().sync(); +} DomainInfo Domain::getDomainInfo() const @@ -318,22 +323,73 @@ Domain::optionallyRotateFile(SerialNum serialNum) { return dp; } +void +Domain::append(const Packet & packet, Writer::DoneCallback onDone) { + vespalib::MonitorGuard guard(_currentChunkMonitor); + if (_lastSerial >= packet.range().from()) { + throw runtime_error(fmt("Incoming serial number(%" PRIu64 ") must be bigger than the last one (%" PRIu64 ").", + packet.range().from(), _lastSerial)); + } else { + _lastSerial = packet.range().to(); + } + _currentChunk->add(packet, std::move(onDone)); + commitIfFull(guard); +} + Domain::CommitResult Domain::startCommit(DoneCallback onDone) { - (void) onDone; + vespalib::MonitorGuard guard(_currentChunkMonitor); + if ( !_currentChunk->empty() ) { + auto completed = grabCurrentChunk(guard); + completed->setCommitDoneCallback(std::move(onDone)); + CommitResult result(completed->createCommitResult()); + commitChunk(std::move(completed), guard); + return result; + } return CommitResult(); } void -Domain::append(const Packet & packet, Writer::DoneCallback onDone) -{ - (void) onDone; +Domain::commitIfFull(const vespalib::MonitorGuard &guard) { + if (_currentChunk->sizeBytes() > _config.getChunkSizeLimit()) { + auto completed = std::move(_currentChunk); + _currentChunk = std::make_unique(_config.getChunkSizeLimit(), completed->stealCallbacks()); + commitChunk(std::move(completed), guard); + } +} + +std::unique_ptr +Domain::grabCurrentChunk(const vespalib::MonitorGuard & guard) { + assert(guard.monitors(_currentChunkMonitor)); + auto chunk = std::move(_currentChunk); + _currentChunk = createCommitChunk(_config); + return chunk; +} + +void +Domain::commitChunk(std::unique_ptr chunk, const vespalib::MonitorGuard & chunkOrderGuard) { + assert(chunkOrderGuard.monitors(_currentChunkMonitor)); + _singleCommitter->execute( makeLambdaTask([this, chunk = std::move(chunk)]() mutable { + doCommit(std::move(chunk)); + })); +} + +void +Domain::doCommit(std::unique_ptr chunk) { + const Packet & packet = chunk->getPacket(); + if (packet.empty()) return; + vespalib::nbostream_longlivedbuf is(packet.getHandle().data(), packet.getHandle().size()); Packet::Entry entry; entry.deserialize(is); DomainPart::SP dp = optionallyRotateFile(entry.serial()); dp->commit(entry.serial(), packet); + if (_config.getFSyncOnCommit()) { + dp->sync(); + } cleanSessions(); + LOG(debug, "Releasing %zu acks and %zu entries and %zu bytes.", + chunk->getNumCallBacks(), chunk->getPacket().size(), chunk->sizeBytes()); } bool diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h index 7e77e6ef0ef..041ec27cf23 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h @@ -56,6 +56,11 @@ public: uint64_t size() const; Domain & setConfig(const DomainConfig & cfg); private: + void commitIfFull(const vespalib::MonitorGuard & guard); + + std::unique_ptr grabCurrentChunk(const vespalib::MonitorGuard & guard); + void commitChunk(std::unique_ptr chunk, const vespalib::MonitorGuard & chunkOrderGuard); + void doCommit(std::unique_ptr chunk); SerialNum begin(const vespalib::LockGuard & guard) const; SerialNum end(const vespalib::LockGuard & guard) const; size_t byteSize(const vespalib::LockGuard & guard) const; diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp index 8855183226d..b7e02894e6b 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp @@ -252,7 +252,7 @@ DomainPart::buildPacketMapping(bool allowTruncate) DomainPart::DomainPart(const string & name, const string & baseDir, SerialNum s, Encoding encoding, uint8_t compressionLevel, const FileHeaderContext &fileHeaderContext, bool allowTruncate) - : _encoding(encoding.getCrc(), Encoding::Compression::none), //TODO We do not yet support compression + : _encoding(encoding), _compressionLevel(compressionLevel), _lock(), _fileLock(), @@ -396,16 +396,19 @@ DomainPart::commit(SerialNum firstSerial, const Packet &packet) if (_range.from() == 0) { _range.from(firstSerial); } + IChunk::UP chunk = IChunk::create(_encoding, _compressionLevel); for (size_t i(0); h.size() > 0; i++) { //LOG(spam, //"Pos(%d) Len(%d), Lim(%d), Remaining(%d)", //h.getPos(), h.getLength(), h.getLimit(), h.getRemaining()); - IChunk::UP chunk = IChunk::create(_encoding, _compressionLevel); Packet::Entry entry; entry.deserialize(h); if (_range.to() < entry.serial()) { chunk->add(entry); - write(*_transLog, *chunk); + if (_encoding.getCompression() == Encoding::Compression::none) { + write(*_transLog, *chunk); + chunk = IChunk::create(_encoding, _compressionLevel); + } _sz++; _range.to(entry.serial()); } else { @@ -413,6 +416,9 @@ DomainPart::commit(SerialNum firstSerial, const Packet &packet) entry.serial(), _range.to())); } } + if ( ! chunk->getEntries().empty()) { + write(*_transLog, *chunk); + } bool merged(false); LockGuard guard(_lock); diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index 7be3dd708a5..0c0c9186e12 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -572,8 +572,11 @@ TransLogServer::domainCommit(FRT_RPCRequest *req) Packet packet(params[1]._data._buf, params[1]._data._len); try { vespalib::Gate gate; - domain->append(packet, make_shared(gate)); - auto keep = domain->startCommit(make_shared()); + { + // Need to scope in order to drain out all the callbacks. + domain->append(packet, make_shared(gate)); + auto keep = domain->startCommit(make_shared()); + } gate.await(); ret.AddInt32(0); ret.AddString("ok"); -- cgit v1.2.3 From fee532dfc40215cd2d814874b9bedd6bfeda048c Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 22 Sep 2020 19:08:06 +0000 Subject: Must append AND commit when doing a synchronous operation. --- .../src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp | 2 +- .../src/vespa/searchcore/proton/server/lid_space_compaction_job.h | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp index d423e095ad9..468850b4409 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp @@ -82,7 +82,7 @@ LidSpaceCompactionJob::compactLidSpace(const LidUsageStats &stats) uint32_t wantedLidLimit = stats.getHighestUsedLid() + 1; CompactLidSpaceOperation op(_handler.getSubDbId(), wantedLidLimit); vespalib::Gate gate; - _opStorer.appendOperation(op, std::make_shared(gate)); + _opStorer.appendAndCommitOperation(op, std::make_shared(gate)); gate.await(); _handler.handleCompactLidSpace(op); EventLogger::lidSpaceCompactionComplete(_handler.getName(), wantedLidLimit); diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h index 37497eaa998..35549f21471 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h @@ -63,13 +63,13 @@ public: ~LidSpaceCompactionJob(); // Implements IDiskMemUsageListener - virtual void notifyDiskMemUsage(DiskMemUsageState state) override; + void notifyDiskMemUsage(DiskMemUsageState state) override; // Implements IClusterStateChangedNofifier - virtual void notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) override; + void notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) override; // Implements IMaintenanceJob - virtual bool run() override; + bool run() override; }; } // namespace proton -- cgit v1.2.3 From e0bc3258030fed738fa39014572b7ca82e22b1d2 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Wed, 23 Sep 2020 08:48:59 +0200 Subject: Relax test to verify meta is stored, but not its content --- .../java/com/yahoo/vespa/hosted/controller/ControllerTest.java | 7 +++---- document/src/main/java/com/yahoo/document/DocumentUtil.java | 2 +- document/src/main/java/com/yahoo/document/json/JsonFeedReader.java | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 942c9ac037b..f3b4d1f6457 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -185,10 +185,9 @@ public class ControllerTest { assertNull("Deployment job was removed", context.instanceJobs().get(productionUsWest1)); // Submission has stored application meta. - assertArrayEquals(applicationPackage.metaDataZip(), - tester.controllerTester().serviceRegistry().applicationStore() - .getMeta(context.instanceId()) - .get(tester.clock().instant())); + assertNotNull(tester.controllerTester().serviceRegistry().applicationStore() + .getMeta(context.instanceId()) + .get(tester.clock().instant())); // Meta data tombstone placed on delete tester.clock().advance(Duration.ofSeconds(1)); diff --git a/document/src/main/java/com/yahoo/document/DocumentUtil.java b/document/src/main/java/com/yahoo/document/DocumentUtil.java index 6e781ce9011..78fe748dac8 100644 --- a/document/src/main/java/com/yahoo/document/DocumentUtil.java +++ b/document/src/main/java/com/yahoo/document/DocumentUtil.java @@ -3,7 +3,7 @@ package com.yahoo.document; /** * Class containing static utility function related to documents. - * @author einarmr + * @author Einar M Rosenvinge * @since 5.1.9 */ public class DocumentUtil { diff --git a/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java b/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java index 5da6b58ac98..3b275d571e0 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java +++ b/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java @@ -23,7 +23,7 @@ import com.yahoo.vespaxmlparser.RemoveFeedOperation; * The feed reader will take ownership of the input stream and close it when the * last parseable document has been read. * - * @author steinar + * @author Steinar Knutsen */ public class JsonFeedReader implements FeedReader { -- cgit v1.2.3 From 32ae03339b525a01fb2cdb82f542f6b4071770ed Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Wed, 23 Sep 2020 09:01:51 +0200 Subject: Allow 4h without progress, with fixes to clock sensitive test --- .../controller/deployment/InternalStepRunner.java | 2 +- .../controller/maintenance/UpgraderTest.java | 6 +-- .../JobControllerApiHandlerHelperTest.java | 2 +- .../responses/deployment-overview-2.json | 28 +++++----- .../responses/dev-aws-us-east-2a-runs.json | 4 +- .../responses/overview-user-instance.json | 4 +- .../application/responses/staging-runs.json | 12 ++--- .../application/responses/staging-test-log.json | 62 +++++++++++----------- 8 files changed, 60 insertions(+), 60 deletions(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 3ebc8240889..af9acc7c3a8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -987,7 +987,7 @@ public class InternalStepRunner implements StepRunner { Duration endpointCertificate() { return Duration.ofMinutes(20); } Duration tester() { return Duration.ofMinutes(30); } Duration nodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 60); } - Duration noNodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 120); } + Duration noNodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 240); } Duration testerCertificate() { return Duration.ofMinutes(300); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index cc92c6cd271..522d44ed667 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -116,7 +116,7 @@ public class UpgraderTest { assertEquals("New system version: Should upgrade Canaries", 4, tester.jobs().active().size()); canary0.runJob(systemTest); - canary0.timeOutUpgrade(stagingTest); + canary0.failDeployment(stagingTest); tester.controllerTester().computeVersionStatus(); assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence()); @@ -153,7 +153,7 @@ public class UpgraderTest { assertEquals("Canaries done: Should upgrade defaults", 6, tester.jobs().active().size()); default0.runJob(systemTest); - default0.timeOutConvergence(stagingTest); + default0.failDeployment(stagingTest); default1.deployPlatform(version3); default2.deployPlatform(version3); @@ -1035,7 +1035,7 @@ public class UpgraderTest { assertEquals(v3, application.instanceJobs().get(stagingTest).lastSuccess().get().versions().targetPlatform()); // First deployment fails and then successfully upgrades to v3 - application.timeOutUpgrade(productionUsCentral1); + application.failDeployment(productionUsCentral1); application.runJob(productionUsCentral1); // Deployments are now on 3 versions diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index 3d583f12a1c..182af3e47ab 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -88,7 +88,7 @@ public class JobControllerApiHandlerHelperTest { assertEquals(deploymentFailed, tester.jobs().last(app.instanceId(), productionUsEast3).get().status()); tester.runner().run(); - tester.clock().advance(Duration.ofHours(2).plusSeconds(1)); + tester.clock().advance(Duration.ofHours(4).plusSeconds(1)); tester.runner().run(); assertEquals(installationFailed, tester.jobs().last(app.instanceId(), productionUsWest1).get().status()); assertEquals(revision2, app.deployment(productionUsCentral1.zone(tester.controller().system())).applicationVersion()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json index e45bd190d5f..b8c48eb3d0c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json @@ -128,8 +128,8 @@ { "id": 3, "url": "https://some.url:43/instance/default/job/system-test/run/3", - "start": 7203000, - "end": 7203000, + "start": 14403000, + "end": 14403000, "status": "success", "versions": { "targetPlatform": "6.1.0", @@ -320,9 +320,9 @@ "dependencies": [], "declared": true, "instance": "default", - "readyAt": 7953000, - "delayedUntil": 7953000, - "coolingDownUntil": 7953000, + "readyAt": 15153000, + "delayedUntil": 15153000, + "coolingDownUntil": 15153000, "jobName": "staging-test", "url": "https://some.url:43/instance/default/job/staging-test", "environment": "staging", @@ -351,8 +351,8 @@ { "id": 5, "url": "https://some.url:43/instance/default/job/staging-test/run/5", - "start": 7303000, - "end": 7303000, + "start": 14503000, + "end": 14503000, "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", @@ -432,8 +432,8 @@ { "id": 4, "url": "https://some.url:43/instance/default/job/staging-test/run/4", - "start": 7203000, - "end": 7203000, + "start": 14403000, + "end": 14403000, "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", @@ -513,8 +513,8 @@ { "id": 3, "url": "https://some.url:43/instance/default/job/staging-test/run/3", - "start": 7203000, - "end": 7203000, + "start": 14403000, + "end": 14403000, "status": "success", "versions": { "targetPlatform": "6.1.0", @@ -756,7 +756,7 @@ ], "declared": true, "instance": "default", - "readyAt": 7203000, + "readyAt": 14403000, "jobName": "production-us-central-1", "url": "https://some.url:43/instance/default/job/production-us-central-1", "environment": "prod", @@ -773,7 +773,7 @@ { "id": 3, "url": "https://some.url:43/instance/default/job/production-us-central-1/run/3", - "start": 7203000, + "start": 14403000, "status": "running", "versions": { "targetPlatform": "6.1.0", @@ -1050,7 +1050,7 @@ "id": 2, "url": "https://some.url:43/instance/default/job/production-us-west-1/run/2", "start": 1000, - "end": 7202000, + "end": 14402000, "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json index 2053b5a80b1..3a78f8c44a0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json @@ -2,8 +2,8 @@ "1": { "id": 1, "status": "success", - "start": 7303000, - "end": 7303000, + "start": 14503000, + "end": 14503000, "wantedPlatform": "7.1", "wantedApplication": { "hash": "unknown" diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json index 285e7f14e8f..2601937faee 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json @@ -8,8 +8,8 @@ "targetApplication": {}, "targetPlatform": "7.1.0" }, - "start": 7303000, - "end": 7303000, + "start": 14503000, + "end": 14503000, "id": 1, "steps": [ { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json index 2ad35968732..37ae9e4b56b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json @@ -94,8 +94,8 @@ "3": { "id": 3, "status": "success", - "start": 7203000, - "end": 7203000, + "start": 14403000, + "end": 14403000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -146,8 +146,8 @@ "4": { "id": 4, "status": "installationFailed", - "start": 7203000, - "end": 7203000, + "start": 14403000, + "end": 14403000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -194,8 +194,8 @@ "5": { "id": 5, "status": "installationFailed", - "start": 7303000, - "end": 7303000, + "start": 14503000, + "end": 14503000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json index a941b824b53..e8c8e57aa9f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json @@ -4,127 +4,127 @@ "log": { "deployTester": [ { - "at": 7303000, + "at": 14503000, "type": "info", "message": "No services requiring restart." }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "Deployment successful." }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "foo" } ], "installTester": [ { - "at": 7303000, + "at": 14503000, "type": "info", "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- platform dockerImage:6.1" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- platform dockerImage:6.1" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- platform dockerImage:6.1" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" } ], "deployInitialReal": [ { - "at": 7303000, + "at": 14503000, "type": "info", "message": "Deploying platform version 6.1 and application version 1.0.1-commit1 ..." }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "No services requiring restart." }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "Deployment successful." }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "foo" } ], "installInitialReal": [ { - "at": 7303000, + "at": 14503000, "type": "info", "message": "######## Details for all nodes ########" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "host-tenant:application:default-staging.us-east-3: unorchestrated" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- platform dockerImage:6.1" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "Deployment expired before installation was successful." } ], "deactivateReal": [ { - "at": 7303000, + "at": 14503000, "type": "info", "message": "Deactivating deployment of tenant.application in staging.us-east-3 ..." } ], "deactivateTester": [ { - "at": 7303000, + "at": 14503000, "type": "info", "message": "Deactivating tester of tenant.application in staging.us-east-3 ..." } @@ -134,19 +134,19 @@ "steps": { "deployTester": { "status": "succeeded", - "startMillis": 7303000 + "startMillis": 14503000 }, "installTester": { "status": "unfinished", - "startMillis": 7303000 + "startMillis": 14503000 }, "deployInitialReal": { "status": "succeeded", - "startMillis": 7303000 + "startMillis": 14503000 }, "installInitialReal": { "status": "failed", - "startMillis": 7303000, + "startMillis": 14503000, "convergence": { "nodes": 1, "down": 0, @@ -182,19 +182,19 @@ }, "copyVespaLogs": { "status": "succeeded", - "startMillis": 7303000 + "startMillis": 14503000 }, "deactivateReal": { "status": "succeeded", - "startMillis": 7303000 + "startMillis": 14503000 }, "deactivateTester": { "status": "succeeded", - "startMillis": 7303000 + "startMillis": 14503000 }, "report": { "status": "succeeded", - "startMillis": 7303000 + "startMillis": 14503000 } } } -- cgit v1.2.3 From d2d607e0368ea38f1f23ec9dde004d11bc1dd297 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Wed, 23 Sep 2020 10:30:47 +0200 Subject: Add zone dimension to use-config-server-vip flag --- flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 0cc4777320e..a9f34efe96c 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -390,7 +390,8 @@ public class Flags { "use-config-server-vip", false, "Whether the controller should use a config server VIP or not", - "Takes effect immediately" + "Takes effect immediately", + ZONE_ID ); /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ -- cgit v1.2.3 From 16063c1ee66fcf1d9621ec0edae646f33485a41e Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Wed, 23 Sep 2020 10:44:30 +0200 Subject: Use zone id as dimension when getting flag --- .../hosted/controller/restapi/zone/v2/ZoneApiHandler.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java index f5e6354da05..36a46f72d34 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java @@ -11,7 +11,8 @@ import com.yahoo.restapi.Path; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; -import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; @@ -35,14 +36,14 @@ public class ZoneApiHandler extends AuditLoggingRequestHandler { private final ZoneRegistry zoneRegistry; private final ConfigServerRestExecutor proxy; - private final BooleanFlag useConfigServerVip; + private final FlagSource flagSource; public ZoneApiHandler(LoggingRequestHandler.Context parentCtx, ServiceRegistry serviceRegistry, ConfigServerRestExecutor proxy, Controller controller) { super(parentCtx, controller.auditLogger()); this.zoneRegistry = serviceRegistry.zoneRegistry(); this.proxy = proxy; - this.useConfigServerVip = Flags.USE_CONFIG_SERVER_VIP.bindTo(controller.flagSource()); + this.flagSource = controller.flagSource(); } @Override @@ -112,8 +113,11 @@ public class ZoneApiHandler extends AuditLoggingRequestHandler { } private ProxyRequest proxyRequest(ZoneId zoneId, String path, HttpRequest request) { + boolean useConfigServerVip = Flags.USE_CONFIG_SERVER_VIP.bindTo(flagSource) + .with(FetchVector.Dimension.ZONE_ID, zoneId.value()).value(); + // TODO: Still need to hardcode AWS since flag cannot be set until flag has been rolled out - if (zoneId.region().value().startsWith("aws-") || useConfigServerVip.value()) { + if (zoneId.region().value().startsWith("aws-") || useConfigServerVip) { return ProxyRequest.tryOne(zoneRegistry.getConfigServerVipUri(zoneId), path, request); } return ProxyRequest.tryAll(zoneRegistry.getConfigServerUris(zoneId), path, request); -- cgit v1.2.3 From dbe1d53b29e2fc699bd98fd9cf393be2e1987809 Mon Sep 17 00:00:00 2001 From: smorgrav Date: Wed, 23 Sep 2020 10:44:41 +0200 Subject: Add optional prefix for 443 access --- .../vespa/hosted/controller/restapi/routing/RoutingApiHandler.java | 3 ++- .../vespa/hosted/controller/restapi/routing/RoutingApiTest.java | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java index ba40f9c2085..114a2967e9a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java @@ -42,6 +42,7 @@ import java.util.stream.Collectors; */ public class RoutingApiHandler extends AuditLoggingRequestHandler { + private static final String OPTIONAL_PREFIX = "/api"; private final Controller controller; public RoutingApiHandler(Context ctx, Controller controller) { @@ -52,7 +53,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { @Override public HttpResponse auditAndHandle(HttpRequest request) { try { - var path = new Path(request.getUri()); + var path = new Path(request.getUri(), OPTIONAL_PREFIX); switch (request.getMethod()) { case GET: return get(path, request); case POST: return post(path); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java index fefd23eb67c..25d21dd702d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java @@ -55,6 +55,11 @@ public class RoutingApiTest extends ControllerContainerTest { Request.Method.GET), new File("discovery/root.json")); + // GET root with api prefix + tester.assertResponse(operatorRequest("http://localhost:8080/api/routing/v1/", "", + Request.Method.GET), + new File("discovery/root.json")); + // GET tenant tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1", "", Request.Method.GET), -- cgit v1.2.3 From 8d6708dcee8a8041917bac842b45645d01dbaa25 Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Wed, 23 Sep 2020 12:42:41 +0200 Subject: Remove the use of optional in stream expression (#14501) No longer use optional in validator by providing a identity value for .reduce(). Also put throw + message in separate method. --- .../model/application/validation/QuotaValidator.java | 17 ++++++++++------- .../application/validation/QuotaValidatorTest.java | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java index f6108f5ac77..7cd5e6b9b07 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.VespaModel; -import java.util.Optional; import java.util.stream.Collectors; /** @@ -23,16 +22,15 @@ public class QuotaValidator extends Validator { } private void validateBudget(int budget, VespaModel model) { - Optional spend = model.allClusters().stream() + var spend = model.allClusters().stream() .map(clusterId -> model.provisioned().all().get(clusterId)) .map(Capacity::maxResources) .map(clusterCapacity -> clusterCapacity.nodeResources().cost() * clusterCapacity.nodes()) - .reduce(Double::sum); + .reduce(0.0, Double::sum); - if(spend.isPresent() && spend.get() > budget) - throw new IllegalArgumentException( - String.format("Hourly spend for maximum specified resources ($%.2f) exceeds budget from quota ($%d)!", - spend.get(), budget)); + if (spend > budget) { + throwBudgetExceeded(spend, budget); + } } /** Check that all clusters in the application do not exceed the quota max cluster size. */ @@ -51,4 +49,9 @@ public class QuotaValidator extends Validator { throw new IllegalArgumentException("Clusters " + clusterNames + " exceeded max cluster size of " + maxClusterSize); } } + + private void throwBudgetExceeded(double spend, double budget) { + var message = String.format("Hourly spend for maximum specified resources ($%.2f) exceeds budget from quota ($%.2f)!", spend, budget); + throw new IllegalArgumentException(message); + } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java index 0a1c0dcb8d8..10199bfe6b9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java @@ -42,7 +42,7 @@ public class QuotaValidatorTest { tester.deploy(null, getServices("testCluster", 10), Environment.prod, null); fail(); } catch (RuntimeException e) { - assertEquals("Hourly spend for maximum specified resources ($1.60) exceeds budget from quota ($1)!", e.getMessage()); + assertEquals("Hourly spend for maximum specified resources ($1.60) exceeds budget from quota ($1.00)!", e.getMessage()); } } -- cgit v1.2.3 From 93e49cad49b0d003dc71a179592e55b7bfd76413 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Wed, 23 Sep 2020 13:14:47 +0200 Subject: Add switch hostname to node entity --- .../hosted/controller/api/integration/entity/NodeEntity.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/NodeEntity.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/NodeEntity.java index 80f09688ec8..bb0613428f5 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/NodeEntity.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/NodeEntity.java @@ -14,11 +14,13 @@ public class NodeEntity { private final String hostname; private final Optional model; private final Optional manufacturer; + private final Optional switchHostname; - public NodeEntity(String hostname, String model, String manufacturer) { + public NodeEntity(String hostname, String model, String manufacturer, String switchHostname) { this.hostname = Objects.requireNonNull(hostname); this.model = nonBlank(model); this.manufacturer = nonBlank(manufacturer); + this.switchHostname = nonBlank(switchHostname); } public String hostname() { @@ -35,6 +37,11 @@ public class NodeEntity { return manufacturer; } + /** The hostname of network switch this node is connected to */ + public Optional switchHostname() { + return switchHostname; + } + private static Optional nonBlank(String s) { return Optional.ofNullable(s).filter(v -> !v.isBlank()); } -- cgit v1.2.3 From a1a76e86064fda3c93b40cfb3d79e8f02c9a88be Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Wed, 23 Sep 2020 13:28:56 +0200 Subject: Wait only a short time for the lock --- .../hosted/provision/maintenance/ScalingSuggestionsMaintainer.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java index b0c52d10f7d..db0e5f03097 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java @@ -62,7 +62,8 @@ public class ScalingSuggestionsMaintainer extends NodeRepositoryMaintainer { Optional cluster = application.cluster(clusterId); if (cluster.isEmpty()) return; Optional suggestion = autoscaler.suggest(cluster.get(), clusterNodes); - try (Mutex lock = nodeRepository().lock(applicationId)) { + // Wait only a short time for the lock to avoid interfering with change deployments + try (Mutex lock = nodeRepository().lock(applicationId, Duration.ofSeconds(1))) { applications().get(applicationId).ifPresent(a -> storeSuggestion(suggestion, clusterId, a, lock)); } } -- cgit v1.2.3 From bee1848654dcdd541001bbb4bf86159cb7d3fc6b Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Wed, 23 Sep 2020 13:52:12 +0200 Subject: Add maintainer for debugging info about zookeeper locks --- .../maintenance/ConfigServerMaintenance.java | 3 + .../config/server/maintenance/LocksMaintainer.java | 65 ++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/LocksMaintainer.java diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java index 0e75d683478..751e983770a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java @@ -27,6 +27,7 @@ public class ConfigServerMaintenance extends AbstractComponent { private final FileDistributionMaintainer fileDistributionMaintainer; private final SessionsMaintainer sessionsMaintainer; private final ApplicationPackageMaintainer applicationPackageMaintainer; + private final LocksMaintainer locksMaintainer; @Inject public ConfigServerMaintenance(ConfigServerBootstrap configServerBootstrap, @@ -40,6 +41,7 @@ public class ConfigServerMaintenance extends AbstractComponent { fileDistributionMaintainer = new FileDistributionMaintainer(applicationRepository, curator, defaults.defaultInterval, flagSource); sessionsMaintainer = new SessionsMaintainer(applicationRepository, curator, Duration.ofSeconds(30), flagSource); applicationPackageMaintainer = new ApplicationPackageMaintainer(applicationRepository, curator, Duration.ofSeconds(30), flagSource); + locksMaintainer = new LocksMaintainer(applicationRepository, curator, Duration.ofSeconds(10), flagSource); } @Override @@ -48,6 +50,7 @@ public class ConfigServerMaintenance extends AbstractComponent { sessionsMaintainer.close(); applicationPackageMaintainer.close(); tenantsMaintainer.close(); + locksMaintainer.close(); } /* diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/LocksMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/LocksMaintainer.java new file mode 100644 index 00000000000..b43d65d4c0a --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/LocksMaintainer.java @@ -0,0 +1,65 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.maintenance; + +import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.flags.FlagSource; +import org.apache.zookeeper.data.Stat; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.logging.Level; + +/** + * Debug maintainer for logging info about node repository locks. Logs with level FINE, so to actually log something + * one needs to change log levels with vespa-logctl + * + * @author hmusum + */ +public class LocksMaintainer extends ConfigServerMaintainer { + + private final boolean hostedVespa; + private final Curator curator; + + LocksMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval, FlagSource flagSource) { + super(applicationRepository, curator, flagSource, Duration.ZERO, interval); + this.hostedVespa = applicationRepository.configserverConfig().hostedVespa(); + this.curator = curator; + } + + @Override + protected boolean maintain() { + if (! hostedVespa) return true; + + Path unallocatedLockPath = Path.fromString("/provision/v1/locks/unallocatedLock"); + logLockInfo(unallocatedLockPath); + + applicationRepository.listApplications().forEach(applicationId -> { + Path applicationLockPath = Path.fromString("/provision/v1/locks").append(applicationId.tenant().value()) + .append(applicationId.application().value()) + .append(applicationId.instance().value()); + logLockInfo(applicationLockPath); + }); + + return true; + } + + private void logLockInfo(Path path) { + List children = curator.getChildren(path); + if (children.size() > 0) + log.log(Level.FINE, path + " has " + children.size() + " locks "); + children.forEach(lockString -> { + Optional createTime = Optional.empty(); + Path lockPath = path.append(lockString); + Optional stat = curator.getStat(lockPath); + if (stat.isPresent()) { + createTime = Optional.of(Instant.ofEpochMilli(stat.get().getCtime())); + } + log.log(Level.FINE, "Lock /" + lockPath + " created " + createTime.map(Instant::toString).orElse(" at unknown time")); + }); + } + +} -- cgit v1.2.3 From 3f9c2f64823b4b65b5164ec28e36e13912817f18 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Wed, 23 Sep 2020 06:46:30 +0000 Subject: add new_encode and new_decode * move common encode/decode logic from simple_tensor.cpp into codec.h / codec.cpp * encoding and decoding using new Value API is in value_codec.h, both for binary format and to/from TensorSpec * will move implementation of TensorSpec conversions later * extend unit test for tensor serialization to also test new_encode and new_decode * add hack in CreateValueFromTensorSpec to handle a spec without any cells but no mapped dimensions, this was used by unit test --- .../tests/eval/simple_value/simple_value_test.cpp | 1 + .../tensor/packed_mappings/packed_mixed_test.cpp | 1 + .../tensor_serialization_test.cpp | 21 ++++ eval/src/vespa/eval/eval/CMakeLists.txt | 2 + eval/src/vespa/eval/eval/codec.cpp | 46 +++++++ eval/src/vespa/eval/eval/codec.h | 111 +++++++++++++++++ eval/src/vespa/eval/eval/simple_tensor.cpp | 136 +-------------------- eval/src/vespa/eval/eval/simple_value.cpp | 7 +- eval/src/vespa/eval/eval/simple_value.h | 11 -- eval/src/vespa/eval/eval/value_codec.cpp | 115 +++++++++++++++++ eval/src/vespa/eval/eval/value_codec.h | 33 +++++ 11 files changed, 339 insertions(+), 145 deletions(-) create mode 100644 eval/src/vespa/eval/eval/codec.cpp create mode 100644 eval/src/vespa/eval/eval/codec.h create mode 100644 eval/src/vespa/eval/eval/value_codec.cpp create mode 100644 eval/src/vespa/eval/eval/value_codec.h diff --git a/eval/src/tests/eval/simple_value/simple_value_test.cpp b/eval/src/tests/eval/simple_value/simple_value_test.cpp index 8ac553680f7..233c09f7e2b 100644 --- a/eval/src/tests/eval/simple_value/simple_value_test.cpp +++ b/eval/src/tests/eval/simple_value/simple_value_test.cpp @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include +#include #include #include #include diff --git a/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp b/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp index 48c72c3c591..eb7607c13b8 100644 --- a/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp +++ b/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include +#include #include #include #include diff --git a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp index d1491e4f758..c00b0e5d35b 100644 --- a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp +++ b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp @@ -10,6 +10,7 @@ #include #include #include +#include using namespace vespalib::tensor; using vespalib::eval::TensorSpec; @@ -47,6 +48,24 @@ void verify_cells_only(const ExpBuffer &exp, const TensorSpec &spec) { ASSERT_EQUAL(i, cells.size()); } +TensorSpec verify_new_value_serialized(const ExpBuffer &exp, const TensorSpec &spec) { + auto factory = vespalib::eval::SimpleValueBuilderFactory(); + auto new_value = vespalib::eval::value_from_spec(spec, factory); + auto new_value_spec = vespalib::eval::spec_from_value(*new_value); + nbostream actual; + vespalib::eval::new_encode(*new_value, actual); + ASSERT_EQUAL(exp, actual); + auto new_decoded = vespalib::eval::new_decode(actual, factory); + auto new_decoded_spec = vespalib::eval::spec_from_value(*new_decoded); + EXPECT_EQUAL(0u, actual.size()); + EXPECT_EQUAL(new_value_spec, new_decoded_spec); + if (new_value->type().is_dense()) { + TEST_DO(verify_cells_only(exp, new_value_spec)); + TEST_DO(verify_cells_only(exp, new_value_spec)); + } + return new_decoded_spec; +} + void verify_serialized(const ExpBuffer &exp, const TensorSpec &spec) { auto &engine = DefaultTensorEngine::ref(); auto value = engine.from_spec(spec); @@ -62,6 +81,8 @@ void verify_serialized(const ExpBuffer &exp, const TensorSpec &spec) { TEST_DO(verify_cells_only(exp, value_spec)); TEST_DO(verify_cells_only(exp, value_spec)); } + auto new_value_spec = verify_new_value_serialized(exp, spec); + EXPECT_EQUAL(value_spec, new_value_spec); } //----------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/eval/CMakeLists.txt b/eval/src/vespa/eval/eval/CMakeLists.txt index 973245607de..4660a2ac9bf 100644 --- a/eval/src/vespa/eval/eval/CMakeLists.txt +++ b/eval/src/vespa/eval/eval/CMakeLists.txt @@ -4,6 +4,7 @@ vespa_add_library(eval_eval OBJECT aggr.cpp basic_nodes.cpp call_nodes.cpp + codec.cpp compile_tensor_function.cpp delete_node.cpp fast_forest.cpp @@ -28,6 +29,7 @@ vespa_add_library(eval_eval OBJECT tensor_nodes.cpp tensor_spec.cpp value.cpp + value_codec.cpp value_type.cpp value_type_spec.cpp visit_stuff.cpp diff --git a/eval/src/vespa/eval/eval/codec.cpp b/eval/src/vespa/eval/eval/codec.cpp new file mode 100644 index 00000000000..1cf60f476a1 --- /dev/null +++ b/eval/src/vespa/eval/eval/codec.cpp @@ -0,0 +1,46 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "codec.h" + +namespace vespalib::eval::codec { + +void encode_type(nbostream &output, const Format &format, const ValueType &type, const TypeMeta &meta) { + maybe_encode_cell_type(output, format, meta); + if (format.is_sparse) { + output.putInt1_4Bytes(meta.mapped.size()); + for (size_t idx: meta.mapped) { + output.writeSmallString(type.dimensions()[idx].name); + } + } + if (format.is_dense) { + output.putInt1_4Bytes(meta.indexed.size()); + for (size_t idx: meta.indexed) { + output.writeSmallString(type.dimensions()[idx].name); + output.putInt1_4Bytes(type.dimensions()[idx].size); + } + } +} + +ValueType decode_type(nbostream &input, const Format &format) { + CellType cell_type = maybe_decode_cell_type(input, format); + std::vector dim_list; + if (format.is_sparse) { + size_t cnt = input.getInt1_4Bytes(); + for (size_t i = 0; i < cnt; ++i) { + vespalib::string name; + input.readSmallString(name); + dim_list.emplace_back(name); + } + } + if (format.is_dense) { + size_t cnt = input.getInt1_4Bytes(); + for (size_t i = 0; i < cnt; ++i) { + vespalib::string name; + input.readSmallString(name); + dim_list.emplace_back(name, input.getInt1_4Bytes()); + } + } + return ValueType::tensor_type(std::move(dim_list), cell_type); +} + +} // namespace vespalib::eval::codec diff --git a/eval/src/vespa/eval/eval/codec.h b/eval/src/vespa/eval/eval/codec.h new file mode 100644 index 00000000000..fad370f6d8b --- /dev/null +++ b/eval/src/vespa/eval/eval/codec.h @@ -0,0 +1,111 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "simple_tensor.h" +#include +#include + +namespace vespalib::eval::codec { + +using CellType = ValueType::CellType; +using IndexList = std::vector; + +constexpr uint32_t DOUBLE_CELL_TYPE = 0; +constexpr uint32_t FLOAT_CELL_TYPE = 1; + +inline uint32_t cell_type_to_id(CellType cell_type) { + switch (cell_type) { + case CellType::DOUBLE: return DOUBLE_CELL_TYPE; + case CellType::FLOAT: return FLOAT_CELL_TYPE; + } + abort(); +} + +inline CellType id_to_cell_type(uint32_t id) { + switch (id) { + case DOUBLE_CELL_TYPE: return CellType::DOUBLE; + case FLOAT_CELL_TYPE: return CellType::FLOAT; + } + abort(); +} + +/** + * Meta information about how a type can be decomposed into mapped and + * indexed dimensions and also how large each block is. A block is a + * dense-subspace consisting of all indexed dimensions that is + * uniquely specified by the labels of all mapped dimensions. + **/ +struct TypeMeta { + IndexList mapped; + IndexList indexed; + size_t block_size; + CellType cell_type; + explicit TypeMeta(const ValueType &type) + : mapped(), + indexed(), + block_size(1), + cell_type(type.cell_type()) + { + for (size_t i = 0; i < type.dimensions().size(); ++i) { + const auto &dimension = type.dimensions()[i]; + if (dimension.is_mapped()) { + mapped.push_back(i); + } else { + block_size *= dimension.size; + indexed.push_back(i); + } + } + } + ~TypeMeta() {} +}; + +struct Format { + bool is_sparse; + bool is_dense; + bool with_cell_type; + uint32_t tag; + explicit Format(const TypeMeta &meta) + : is_sparse(meta.mapped.size() > 0), + is_dense((meta.indexed.size() > 0) || !is_sparse), + with_cell_type(meta.cell_type != CellType::DOUBLE), + tag((is_sparse ? 0x1 : 0) | (is_dense ? 0x2 : 0) | (with_cell_type ? 0x4 : 0)) {} + explicit Format(uint32_t tag_in) + : is_sparse((tag_in & 0x1) != 0), + is_dense((tag_in & 0x2) != 0), + with_cell_type((tag_in & 0x4) != 0), + tag(tag_in) {} + ~Format() {} +}; + +inline void maybe_encode_cell_type(nbostream &output, const Format &format, const TypeMeta &meta) { + if (format.with_cell_type) { + output.putInt1_4Bytes(cell_type_to_id(meta.cell_type)); + } +} + +void encode_type(nbostream &output, const Format &format, const ValueType &type, const TypeMeta &meta); + +inline void maybe_encode_num_blocks(nbostream &output, const TypeMeta &meta, size_t num_blocks) { + if ((meta.mapped.size() > 0)) { + output.putInt1_4Bytes(num_blocks); + } +} + +inline CellType maybe_decode_cell_type(nbostream &input, const Format &format) { + if (format.with_cell_type) { + return id_to_cell_type(input.getInt1_4Bytes()); + } + return CellType::DOUBLE; +} + +ValueType decode_type(nbostream &input, const Format &format); + +inline size_t maybe_decode_num_blocks(nbostream &input, const TypeMeta &meta, const Format &format) { + if ((meta.mapped.size() > 0) || !format.is_dense) { + return input.getInt1_4Bytes(); + } + return 1; +} + +} // namespace vespalib::eval::codec diff --git a/eval/src/vespa/eval/eval/simple_tensor.cpp b/eval/src/vespa/eval/eval/simple_tensor.cpp index 64b2b6f8865..498119f0733 100644 --- a/eval/src/vespa/eval/eval/simple_tensor.cpp +++ b/eval/src/vespa/eval/eval/simple_tensor.cpp @@ -3,44 +3,26 @@ #include "simple_tensor.h" #include "simple_tensor_engine.h" #include "operation.h" +#include "codec.h" #include #include #include #include #include +using namespace vespalib::eval::codec; + namespace vespalib { namespace eval { using Address = SimpleTensor::Address; using Cell = SimpleTensor::Cell; using Cells = SimpleTensor::Cells; -using IndexList = std::vector; using Label = SimpleTensor::Label; using CellRef = std::reference_wrapper; -using CellType = ValueType::CellType; namespace { -constexpr uint32_t DOUBLE_CELL_TYPE = 0; -constexpr uint32_t FLOAT_CELL_TYPE = 1; - -uint32_t cell_type_to_id(CellType cell_type) { - switch (cell_type) { - case CellType::DOUBLE: return DOUBLE_CELL_TYPE; - case CellType::FLOAT: return FLOAT_CELL_TYPE; - } - abort(); -} - -CellType id_to_cell_type(uint32_t id) { - switch (id) { - case DOUBLE_CELL_TYPE: return CellType::DOUBLE; - case FLOAT_CELL_TYPE: return CellType::FLOAT; - } - abort(); -} - void assert_type(const ValueType &type) { (void) type; assert(type.is_double() || type.is_tensor()); @@ -105,35 +87,6 @@ const vespalib::string &reverse_rename(const vespalib::string &name, return name; } -/** - * Meta information about how a type can be decomposed into mapped and - * indexed dimensions and also how large each block is. A block is a - * dense-subspace consisting of all indexed dimensions that is - * uniquely specified by the labels of all mapped dimensions. - **/ -struct TypeMeta { - IndexList mapped; - IndexList indexed; - size_t block_size; - CellType cell_type; - explicit TypeMeta(const ValueType &type) - : mapped(), - indexed(), - block_size(1), - cell_type(type.cell_type()) - { - for (size_t i = 0; i < type.dimensions().size(); ++i) { - const auto &dimension = type.dimensions()[i]; - if (dimension.is_mapped()) { - mapped.push_back(i); - } else { - block_size *= dimension.size; - indexed.push_back(i); - } - } - } - ~TypeMeta() {} -}; /** * Helper class used when building SimpleTensors. While a tensor @@ -438,95 +391,12 @@ public: } }; -struct Format { - bool is_sparse; - bool is_dense; - bool with_cell_type; - uint32_t tag; - explicit Format(const TypeMeta &meta) - : is_sparse(meta.mapped.size() > 0), - is_dense((meta.indexed.size() > 0) || !is_sparse), - with_cell_type(meta.cell_type != CellType::DOUBLE), - tag((is_sparse ? 0x1 : 0) | (is_dense ? 0x2 : 0) | (with_cell_type ? 0x4 : 0)) {} - explicit Format(uint32_t tag_in) - : is_sparse((tag_in & 0x1) != 0), - is_dense((tag_in & 0x2) != 0), - with_cell_type((tag_in & 0x4) != 0), - tag(tag_in) {} - ~Format() {} -}; - -void maybe_encode_cell_type(nbostream &output, const Format &format, const TypeMeta &meta) { - if (format.with_cell_type) { - output.putInt1_4Bytes(cell_type_to_id(meta.cell_type)); - } -} - -void encode_type(nbostream &output, const Format &format, const ValueType &type, const TypeMeta &meta) { - maybe_encode_cell_type(output, format, meta); - if (format.is_sparse) { - output.putInt1_4Bytes(meta.mapped.size()); - for (size_t idx: meta.mapped) { - output.writeSmallString(type.dimensions()[idx].name); - } - } - if (format.is_dense) { - output.putInt1_4Bytes(meta.indexed.size()); - for (size_t idx: meta.indexed) { - output.writeSmallString(type.dimensions()[idx].name); - output.putInt1_4Bytes(type.dimensions()[idx].size); - } - } -} - -void maybe_encode_num_blocks(nbostream &output, const TypeMeta &meta, size_t num_blocks) { - if ((meta.mapped.size() > 0)) { - output.putInt1_4Bytes(num_blocks); - } -} - void encode_mapped_labels(nbostream &output, const TypeMeta &meta, const Address &addr) { for (size_t idx: meta.mapped) { output.writeSmallString(addr[idx].name); } } -CellType maybe_decode_cell_type(nbostream &input, const Format &format) { - if (format.with_cell_type) { - return id_to_cell_type(input.getInt1_4Bytes()); - } - return CellType::DOUBLE; -} - -ValueType decode_type(nbostream &input, const Format &format) { - CellType cell_type = maybe_decode_cell_type(input, format); - std::vector dim_list; - if (format.is_sparse) { - size_t cnt = input.getInt1_4Bytes(); - for (size_t i = 0; i < cnt; ++i) { - vespalib::string name; - input.readSmallString(name); - dim_list.emplace_back(name); - } - } - if (format.is_dense) { - size_t cnt = input.getInt1_4Bytes(); - for (size_t i = 0; i < cnt; ++i) { - vespalib::string name; - input.readSmallString(name); - dim_list.emplace_back(name, input.getInt1_4Bytes()); - } - } - return ValueType::tensor_type(std::move(dim_list), cell_type); -} - -size_t maybe_decode_num_blocks(nbostream &input, const TypeMeta &meta, const Format &format) { - if ((meta.mapped.size() > 0) || !format.is_dense) { - return input.getInt1_4Bytes(); - } - return 1; -} - void decode_mapped_labels(nbostream &input, const TypeMeta &meta, Address &addr) { for (size_t idx: meta.mapped) { vespalib::string name; diff --git a/eval/src/vespa/eval/eval/simple_value.cpp b/eval/src/vespa/eval/eval/simple_value.cpp index 4daa78375e5..2c958f2150a 100644 --- a/eval/src/vespa/eval/eval/simple_value.cpp +++ b/eval/src/vespa/eval/eval/simple_value.cpp @@ -3,6 +3,7 @@ #include "simple_value.h" #include "tensor_spec.h" #include "inline_operation.h" +#include "codec.h" #include #include #include @@ -45,6 +46,10 @@ struct CreateValueFromTensorSpec { } map[sparse_key][dense_key] = entry.second; } + // hack for passing some (invalid?) unit tests + if (spec.cells().empty() && type.count_mapped_dimensions() == 0) { + map[SparseKey()][0] = 0; + } auto builder = factory.create_value_builder(type, type.count_mapped_dimensions(), type.dense_subspace_size(), map.size()); for (const auto &entry: map) { auto subspace = builder->add_subspace(entry.first); @@ -410,4 +415,4 @@ TensorSpec spec_from_value(const Value &value) { //----------------------------------------------------------------------------- -} +} // namespace diff --git a/eval/src/vespa/eval/eval/simple_value.h b/eval/src/vespa/eval/eval/simple_value.h index 4c7bccfa01c..14ba5df835e 100644 --- a/eval/src/vespa/eval/eval/simple_value.h +++ b/eval/src/vespa/eval/eval/simple_value.h @@ -203,15 +203,4 @@ struct SparseJoinPlan { using join_fun_t = double (*)(double, double); std::unique_ptr new_join(const Value &a, const Value &b, join_fun_t function, const ValueBuilderFactory &factory); -/** - * Make a value from a tensor spec using a value builder factory - * interface, making it work with any value implementation. - **/ -std::unique_ptr value_from_spec(const TensorSpec &spec, const ValueBuilderFactory &factory); - -/** - * Convert a generic value to a tensor spec. - **/ -TensorSpec spec_from_value(const Value &value); - } diff --git a/eval/src/vespa/eval/eval/value_codec.cpp b/eval/src/vespa/eval/eval/value_codec.cpp new file mode 100644 index 00000000000..5aad3f91b6a --- /dev/null +++ b/eval/src/vespa/eval/eval/value_codec.cpp @@ -0,0 +1,115 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "value_codec.h" +#include "codec.h" +#include + +using namespace vespalib::eval::codec; + +namespace vespalib::eval { + +void encode_mapped_labels(nbostream &output, size_t num_mapped_dims, const std::vector &addr) { + for (size_t i = 0; i < num_mapped_dims; ++i) { + output.writeSmallString(addr[i]); + } +} + +void decode_mapped_labels(nbostream &input, size_t num_mapped_dims, std::vector &addr) { + for (size_t i = 0; i < num_mapped_dims; ++i) { + vespalib::string name; + input.readSmallString(name); + addr[i] = name; + } +} + +void new_encode(const Value &value, nbostream &output) { + TypeMeta meta(value.type()); + Format format(meta); + output.putInt1_4Bytes(format.tag); + encode_type(output, format, value.type(), meta); + maybe_encode_num_blocks(output, meta, value.cells().size / meta.block_size); + std::vector address(meta.mapped.size()); + std::vector a_refs(meta.mapped.size()); + for (size_t i = 0; i < meta.mapped.size(); ++i) { + a_refs[i] = &address[i]; + } + auto view = value.index().create_view({}); + view->lookup({}); + size_t subspace; + while (view->next_result(a_refs, subspace)) { + encode_mapped_labels(output, meta.mapped.size(), address); + if (meta.cell_type == CellType::FLOAT) { + auto iter = value.cells().typify().begin(); + iter += (subspace * meta.block_size); + for (size_t i = 0; i < meta.block_size; ++i) { + output << (float) *iter++; + } + } else { + auto iter = value.cells().typify().begin(); + iter += (subspace * meta.block_size); + for (size_t i = 0; i < meta.block_size; ++i) { + output << *iter++; + } + } + } +} + +template +void decode_cells(nbostream &input, size_t num_cells, ArrayRef dst) +{ + T value; + for (size_t i = 0; i < num_cells; ++i) { + input >> value; + dst[i] = value; + } +} + +struct DecodeState { + const ValueType &type; + const size_t block_size; + const size_t num_blocks; + const size_t num_mapped_dims; + std::vector address; + std::vector addr_refs; + DecodeState(const ValueType &type_in, + size_t block_size_in, + size_t num_blocks_in, + size_t num_mapped_dims_in) + : type(type_in), + block_size(block_size_in), + num_blocks(num_blocks_in), + num_mapped_dims(num_mapped_dims_in), + address(num_mapped_dims_in), + addr_refs(num_mapped_dims_in) + {} + void fix_refs() { + for (size_t j = 0; j < num_mapped_dims; ++j) { + addr_refs[j] = address[j]; + } + } +}; + +struct ContentDecoder { + template + static std::unique_ptr invoke(nbostream &input, DecodeState &state, const ValueBuilderFactory &factory) { + auto builder = factory.create_value_builder(state.type, state.num_mapped_dims, state.block_size, state.num_blocks); + for (size_t i = 0; i < state.num_blocks; ++i) { + decode_mapped_labels(input, state.num_mapped_dims, state.address); + state.fix_refs(); + auto block_cells = builder->add_subspace(state.addr_refs); + decode_cells(input, state.block_size, block_cells); + } + return builder->build(std::move(builder)); + } +}; + +std::unique_ptr new_decode(nbostream &input, const ValueBuilderFactory &factory) { + Format format(input.getInt1_4Bytes()); + ValueType type = decode_type(input, format); + TypeMeta meta(type); + const size_t num_blocks = maybe_decode_num_blocks(input, meta, format); + DecodeState state(type, meta.block_size, num_blocks, meta.mapped.size()); + return typify_invoke<1,TypifyCellType,ContentDecoder>(meta.cell_type, input, state, factory); +} + +} // namespace diff --git a/eval/src/vespa/eval/eval/value_codec.h b/eval/src/vespa/eval/eval/value_codec.h new file mode 100644 index 00000000000..08b0ac73cc3 --- /dev/null +++ b/eval/src/vespa/eval/eval/value_codec.h @@ -0,0 +1,33 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "simple_value.h" +#include + +namespace vespalib { class nbostream; } + +namespace vespalib::eval { + +/** + * encode a value to binary format + **/ +void new_encode(const Value &value, nbostream &output); + +/** + * decode a value from binary format + **/ +std::unique_ptr new_decode(nbostream &input, const ValueBuilderFactory &factory); + +/** + * Make a value from a tensor spec using a value builder factory + * interface, making it work with any value implementation. + **/ +std::unique_ptr value_from_spec(const TensorSpec &spec, const ValueBuilderFactory &factory); + +/** + * Convert a generic value to a tensor spec. + **/ +TensorSpec spec_from_value(const Value &value); + +} -- cgit v1.2.3 From dff267e0d6323c3ad1ae10d3193e4c18c823a1b8 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Wed, 23 Sep 2020 11:54:28 +0000 Subject: peek into nbostream instead of copying strings --- eval/src/vespa/eval/eval/value_codec.cpp | 95 ++++++++++++++------------------ 1 file changed, 41 insertions(+), 54 deletions(-) diff --git a/eval/src/vespa/eval/eval/value_codec.cpp b/eval/src/vespa/eval/eval/value_codec.cpp index 5aad3f91b6a..90a3016d0ab 100644 --- a/eval/src/vespa/eval/eval/value_codec.cpp +++ b/eval/src/vespa/eval/eval/value_codec.cpp @@ -8,20 +8,56 @@ using namespace vespalib::eval::codec; namespace vespalib::eval { +namespace { + void encode_mapped_labels(nbostream &output, size_t num_mapped_dims, const std::vector &addr) { for (size_t i = 0; i < num_mapped_dims; ++i) { output.writeSmallString(addr[i]); } } -void decode_mapped_labels(nbostream &input, size_t num_mapped_dims, std::vector &addr) { +void decode_mapped_labels(nbostream &input, size_t num_mapped_dims, std::vector &addr) { for (size_t i = 0; i < num_mapped_dims; ++i) { - vespalib::string name; - input.readSmallString(name); - addr[i] = name; + size_t strSize = input.getInt1_4Bytes(); + addr[i] = vespalib::stringref(input.peek(), strSize); + input.adjustReadPos(strSize); } } + +template +void decode_cells(nbostream &input, size_t num_cells, ArrayRef dst) +{ + T value; + for (size_t i = 0; i < num_cells; ++i) { + input >> value; + dst[i] = value; + } +} + +struct DecodeState { + const ValueType &type; + const size_t block_size; + const size_t num_blocks; + const size_t num_mapped_dims; +}; + +struct ContentDecoder { + template + static std::unique_ptr invoke(nbostream &input, const DecodeState &state, const ValueBuilderFactory &factory) { + std::vector address(state.num_mapped_dims); + auto builder = factory.create_value_builder(state.type, state.num_mapped_dims, state.block_size, state.num_blocks); + for (size_t i = 0; i < state.num_blocks; ++i) { + decode_mapped_labels(input, state.num_mapped_dims, address); + auto block_cells = builder->add_subspace(address); + decode_cells(input, state.block_size, block_cells); + } + return builder->build(std::move(builder)); + } +}; + +} // namespace + void new_encode(const Value &value, nbostream &output) { TypeMeta meta(value.type()); Format format(meta); @@ -54,61 +90,12 @@ void new_encode(const Value &value, nbostream &output) { } } -template -void decode_cells(nbostream &input, size_t num_cells, ArrayRef dst) -{ - T value; - for (size_t i = 0; i < num_cells; ++i) { - input >> value; - dst[i] = value; - } -} - -struct DecodeState { - const ValueType &type; - const size_t block_size; - const size_t num_blocks; - const size_t num_mapped_dims; - std::vector address; - std::vector addr_refs; - DecodeState(const ValueType &type_in, - size_t block_size_in, - size_t num_blocks_in, - size_t num_mapped_dims_in) - : type(type_in), - block_size(block_size_in), - num_blocks(num_blocks_in), - num_mapped_dims(num_mapped_dims_in), - address(num_mapped_dims_in), - addr_refs(num_mapped_dims_in) - {} - void fix_refs() { - for (size_t j = 0; j < num_mapped_dims; ++j) { - addr_refs[j] = address[j]; - } - } -}; - -struct ContentDecoder { - template - static std::unique_ptr invoke(nbostream &input, DecodeState &state, const ValueBuilderFactory &factory) { - auto builder = factory.create_value_builder(state.type, state.num_mapped_dims, state.block_size, state.num_blocks); - for (size_t i = 0; i < state.num_blocks; ++i) { - decode_mapped_labels(input, state.num_mapped_dims, state.address); - state.fix_refs(); - auto block_cells = builder->add_subspace(state.addr_refs); - decode_cells(input, state.block_size, block_cells); - } - return builder->build(std::move(builder)); - } -}; - std::unique_ptr new_decode(nbostream &input, const ValueBuilderFactory &factory) { Format format(input.getInt1_4Bytes()); ValueType type = decode_type(input, format); TypeMeta meta(type); const size_t num_blocks = maybe_decode_num_blocks(input, meta, format); - DecodeState state(type, meta.block_size, num_blocks, meta.mapped.size()); + DecodeState state{type, meta.block_size, num_blocks, meta.mapped.size()}; return typify_invoke<1,TypifyCellType,ContentDecoder>(meta.cell_type, input, state, factory); } -- cgit v1.2.3 From 71b3946b52bb69ca611cfedd12175e58bca45f02 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Wed, 23 Sep 2020 14:24:33 +0200 Subject: Add feature flag for skipping maintenance deployments --- flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index a9f34efe96c..ce9063dbcce 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -157,21 +157,25 @@ public class Flags { "Selects type of sequenced executor used for feeding, valid values are LATENCY, ADAPTIVE, THROUGHPUT", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundStringFlag RESPONSE_SEQUENCER_TYPE = defineStringFlag( "response-sequencer-type", "ADAPTIVE", "Selects type of sequenced executor used for mbus responses, valid values are LATENCY, ADAPTIVE, THROUGHPUT", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundIntFlag RESPONSE_NUM_THREADS = defineIntFlag( "response-num-threads", 2, "Number of threads used for mbus responses, default is 2, negative number = numcores/4", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag SKIP_COMMUNICATIONMANAGER_THREAD = defineFeatureFlag( "skip-communicatiomanager-thread", false, "Should we skip the communicationmanager thread", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag SKIP_MBUS_REQUEST_THREAD = defineFeatureFlag( "skip-mbus-request-thread", false, "Should we skip the mbus request thread", @@ -288,7 +292,6 @@ public class Flags { "Whether to provision and use endpoint certs for apps in shared routing zones", "Takes effect on next deployment of the application", APPLICATION_ID); - public static final UnboundBooleanFlag USE_CLOUD_INIT_FORMAT = defineFeatureFlag( "use-cloud-init", false, "Use the cloud-init format when provisioning hosts", @@ -394,6 +397,13 @@ public class Flags { ZONE_ID ); + public static final UnboundBooleanFlag SKIP_MAINTENANCE_DEPLOYMENT = defineFeatureFlag( + "node-repository-skip-maintenance-deployment", + false, + "Whether PeriodicApplicationMaintainer should skip deployment for an application", + "Takes effect at next run of maintainer", + APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { -- cgit v1.2.3 From cf2070eeca6a91be150a100cd32ec97129627459 Mon Sep 17 00:00:00 2001 From: smorgrav Date: Wed, 23 Sep 2020 14:53:24 +0200 Subject: Add bindings to controller tester and test api prefix --- .../hosted/controller/restapi/ControllerContainerTest.java | 1 + .../hosted/controller/restapi/routing/RoutingApiTest.java | 13 ++++++++----- .../restapi/routing/responses/discovery/instance_api.json | 10 ++++++++++ 3 files changed, 19 insertions(+), 5 deletions(-) create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance_api.json diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index f79051f2edb..4b4a8415d69 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -106,6 +106,7 @@ public class ControllerContainerTest { " \n" + " \n" + " http://*/routing/v1/*\n" + + " http://*/api/routing/v1/*\n" + " \n" + variablePartXml() + ""; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java index 25d21dd702d..b2e9fc956ec 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java @@ -55,11 +55,6 @@ public class RoutingApiTest extends ControllerContainerTest { Request.Method.GET), new File("discovery/root.json")); - // GET root with api prefix - tester.assertResponse(operatorRequest("http://localhost:8080/api/routing/v1/", "", - Request.Method.GET), - new File("discovery/root.json")); - // GET tenant tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1", "", Request.Method.GET), @@ -81,6 +76,14 @@ public class RoutingApiTest extends ControllerContainerTest { tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/", "", Request.Method.GET), new File("discovery/environment.json")); + + + // GET instance with api prefix (test that the /api prefix works) + tester.assertResponse(authenticatedRequest("http://localhost:8080/api/routing/v1/status/tenant/t1/application/a1/instance/default/", + "", + Request.Method.GET), + new File("discovery/instance_api.json")); + } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance_api.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance_api.json new file mode 100644 index 00000000000..a9e789d9fe9 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance_api.json @@ -0,0 +1,10 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/api/routing/v1/status/tenant/t1/application/a1/instance/default/environment/prod/region/us-east-3/" + }, + { + "url": "http://localhost:8080/api/routing/v1/status/tenant/t1/application/a1/instance/default/environment/prod/region/us-west-1/" + } + ] +} \ No newline at end of file -- cgit v1.2.3 From 87537becd85dacc4dcd7eb36605acd755da5189c Mon Sep 17 00:00:00 2001 From: smorgrav Date: Wed, 23 Sep 2020 14:54:18 +0200 Subject: Remove redundant newlines --- .../yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java index b2e9fc956ec..b0549662ab0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java @@ -77,13 +77,11 @@ public class RoutingApiTest extends ControllerContainerTest { Request.Method.GET), new File("discovery/environment.json")); - // GET instance with api prefix (test that the /api prefix works) tester.assertResponse(authenticatedRequest("http://localhost:8080/api/routing/v1/status/tenant/t1/application/a1/instance/default/", "", Request.Method.GET), new File("discovery/instance_api.json")); - } @Test -- cgit v1.2.3 From efc62ebb67af59aa6410e9604961722d7f04958e Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Wed, 23 Sep 2020 14:56:30 +0200 Subject: Run some maintainers with higher interval When deployments exist on all config servers each server will deploy, so do it with 3 times higher interval --- .../provision/maintenance/NodeRepositoryMaintenance.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 0890908dc80..02ceee37e51 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -9,7 +9,9 @@ import com.yahoo.config.provision.HostLivenessTracker; import com.yahoo.config.provision.InfraDeployer; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics; import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; @@ -70,7 +72,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { Zone zone, Clock clock, Orchestrator orchestrator, Metric metric, ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource, NodeMetrics nodeMetrics, NodeMetricsDb nodeMetricsDb) { - DefaultTimes defaults = new DefaultTimes(zone); + DefaultTimes defaults = new DefaultTimes(zone, Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.bindTo(flagSource).value()); nodeFailer = new NodeFailer(deployer, hostLivenessTracker, serviceMonitor, nodeRepository, defaults.failGrace, clock, orchestrator, throttlePolicyFromEnv().orElse(defaults.throttlePolicy), metric); periodicApplicationMaintainer = new PeriodicApplicationMaintainer(deployer, metric, nodeRepository, defaults.redeployMaintainerInterval, defaults.periodicRedeployInterval); @@ -167,7 +169,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final NodeFailer.ThrottlePolicy throttlePolicy; - DefaultTimes(Zone zone) { + DefaultTimes(Zone zone, boolean deploymentExistsOnAllConfigServers) { autoscalingInterval = Duration.ofMinutes(5); dynamicProvisionerInterval = Duration.ofMinutes(5); failedExpirerInterval = Duration.ofMinutes(10); @@ -178,14 +180,14 @@ public class NodeRepositoryMaintenance extends AbstractComponent { nodeMetricsCollectionInterval = Duration.ofMinutes(1); operatorChangeRedeployInterval = Duration.ofMinutes(1); osUpgradeActivatorInterval = zone.system().isCd() ? Duration.ofSeconds(30) : Duration.ofMinutes(5); - periodicRedeployInterval = Duration.ofMinutes(30); + periodicRedeployInterval = deploymentExistsOnAllConfigServers ? Duration.ofMinutes(90) : Duration.ofMinutes(30); provisionedExpiry = Duration.ofHours(4); - rebalancerInterval = Duration.ofMinutes(40); + rebalancerInterval = deploymentExistsOnAllConfigServers ? Duration.ofMinutes(120) : Duration.ofMinutes(40); redeployMaintainerInterval = Duration.ofMinutes(1); // Need to be long enough for deployment to be finished for all config model versions // Should be equal to timeout for deployments reservationExpiry = zone.system().isCd() ? Duration.ofMinutes(5) : Duration.ofMinutes(30); - scalingSuggestionsInterval = Duration.ofMinutes(31); + scalingSuggestionsInterval = deploymentExistsOnAllConfigServers ? Duration.ofMinutes(91) : Duration.ofMinutes(31); spareCapacityMaintenanceInterval = Duration.ofMinutes(10); throttlePolicy = NodeFailer.ThrottlePolicy.hosted; -- cgit v1.2.3 From b81b7f5e10eaefb611616fa516b4a0851cd1e645 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Wed, 23 Sep 2020 13:20:40 +0000 Subject: revert moving code into codec.h --- eval/src/vespa/eval/eval/CMakeLists.txt | 1 - eval/src/vespa/eval/eval/codec.cpp | 46 ---------- eval/src/vespa/eval/eval/codec.h | 111 ----------------------- eval/src/vespa/eval/eval/simple_tensor.cpp | 136 +++++++++++++++++++++++++++- eval/src/vespa/eval/eval/simple_value.cpp | 1 - eval/src/vespa/eval/eval/value_codec.cpp | 139 ++++++++++++++++++++++++++++- 6 files changed, 269 insertions(+), 165 deletions(-) delete mode 100644 eval/src/vespa/eval/eval/codec.cpp delete mode 100644 eval/src/vespa/eval/eval/codec.h diff --git a/eval/src/vespa/eval/eval/CMakeLists.txt b/eval/src/vespa/eval/eval/CMakeLists.txt index 4660a2ac9bf..84cebe8cfd0 100644 --- a/eval/src/vespa/eval/eval/CMakeLists.txt +++ b/eval/src/vespa/eval/eval/CMakeLists.txt @@ -4,7 +4,6 @@ vespa_add_library(eval_eval OBJECT aggr.cpp basic_nodes.cpp call_nodes.cpp - codec.cpp compile_tensor_function.cpp delete_node.cpp fast_forest.cpp diff --git a/eval/src/vespa/eval/eval/codec.cpp b/eval/src/vespa/eval/eval/codec.cpp deleted file mode 100644 index 1cf60f476a1..00000000000 --- a/eval/src/vespa/eval/eval/codec.cpp +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "codec.h" - -namespace vespalib::eval::codec { - -void encode_type(nbostream &output, const Format &format, const ValueType &type, const TypeMeta &meta) { - maybe_encode_cell_type(output, format, meta); - if (format.is_sparse) { - output.putInt1_4Bytes(meta.mapped.size()); - for (size_t idx: meta.mapped) { - output.writeSmallString(type.dimensions()[idx].name); - } - } - if (format.is_dense) { - output.putInt1_4Bytes(meta.indexed.size()); - for (size_t idx: meta.indexed) { - output.writeSmallString(type.dimensions()[idx].name); - output.putInt1_4Bytes(type.dimensions()[idx].size); - } - } -} - -ValueType decode_type(nbostream &input, const Format &format) { - CellType cell_type = maybe_decode_cell_type(input, format); - std::vector dim_list; - if (format.is_sparse) { - size_t cnt = input.getInt1_4Bytes(); - for (size_t i = 0; i < cnt; ++i) { - vespalib::string name; - input.readSmallString(name); - dim_list.emplace_back(name); - } - } - if (format.is_dense) { - size_t cnt = input.getInt1_4Bytes(); - for (size_t i = 0; i < cnt; ++i) { - vespalib::string name; - input.readSmallString(name); - dim_list.emplace_back(name, input.getInt1_4Bytes()); - } - } - return ValueType::tensor_type(std::move(dim_list), cell_type); -} - -} // namespace vespalib::eval::codec diff --git a/eval/src/vespa/eval/eval/codec.h b/eval/src/vespa/eval/eval/codec.h deleted file mode 100644 index fad370f6d8b..00000000000 --- a/eval/src/vespa/eval/eval/codec.h +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "simple_tensor.h" -#include -#include - -namespace vespalib::eval::codec { - -using CellType = ValueType::CellType; -using IndexList = std::vector; - -constexpr uint32_t DOUBLE_CELL_TYPE = 0; -constexpr uint32_t FLOAT_CELL_TYPE = 1; - -inline uint32_t cell_type_to_id(CellType cell_type) { - switch (cell_type) { - case CellType::DOUBLE: return DOUBLE_CELL_TYPE; - case CellType::FLOAT: return FLOAT_CELL_TYPE; - } - abort(); -} - -inline CellType id_to_cell_type(uint32_t id) { - switch (id) { - case DOUBLE_CELL_TYPE: return CellType::DOUBLE; - case FLOAT_CELL_TYPE: return CellType::FLOAT; - } - abort(); -} - -/** - * Meta information about how a type can be decomposed into mapped and - * indexed dimensions and also how large each block is. A block is a - * dense-subspace consisting of all indexed dimensions that is - * uniquely specified by the labels of all mapped dimensions. - **/ -struct TypeMeta { - IndexList mapped; - IndexList indexed; - size_t block_size; - CellType cell_type; - explicit TypeMeta(const ValueType &type) - : mapped(), - indexed(), - block_size(1), - cell_type(type.cell_type()) - { - for (size_t i = 0; i < type.dimensions().size(); ++i) { - const auto &dimension = type.dimensions()[i]; - if (dimension.is_mapped()) { - mapped.push_back(i); - } else { - block_size *= dimension.size; - indexed.push_back(i); - } - } - } - ~TypeMeta() {} -}; - -struct Format { - bool is_sparse; - bool is_dense; - bool with_cell_type; - uint32_t tag; - explicit Format(const TypeMeta &meta) - : is_sparse(meta.mapped.size() > 0), - is_dense((meta.indexed.size() > 0) || !is_sparse), - with_cell_type(meta.cell_type != CellType::DOUBLE), - tag((is_sparse ? 0x1 : 0) | (is_dense ? 0x2 : 0) | (with_cell_type ? 0x4 : 0)) {} - explicit Format(uint32_t tag_in) - : is_sparse((tag_in & 0x1) != 0), - is_dense((tag_in & 0x2) != 0), - with_cell_type((tag_in & 0x4) != 0), - tag(tag_in) {} - ~Format() {} -}; - -inline void maybe_encode_cell_type(nbostream &output, const Format &format, const TypeMeta &meta) { - if (format.with_cell_type) { - output.putInt1_4Bytes(cell_type_to_id(meta.cell_type)); - } -} - -void encode_type(nbostream &output, const Format &format, const ValueType &type, const TypeMeta &meta); - -inline void maybe_encode_num_blocks(nbostream &output, const TypeMeta &meta, size_t num_blocks) { - if ((meta.mapped.size() > 0)) { - output.putInt1_4Bytes(num_blocks); - } -} - -inline CellType maybe_decode_cell_type(nbostream &input, const Format &format) { - if (format.with_cell_type) { - return id_to_cell_type(input.getInt1_4Bytes()); - } - return CellType::DOUBLE; -} - -ValueType decode_type(nbostream &input, const Format &format); - -inline size_t maybe_decode_num_blocks(nbostream &input, const TypeMeta &meta, const Format &format) { - if ((meta.mapped.size() > 0) || !format.is_dense) { - return input.getInt1_4Bytes(); - } - return 1; -} - -} // namespace vespalib::eval::codec diff --git a/eval/src/vespa/eval/eval/simple_tensor.cpp b/eval/src/vespa/eval/eval/simple_tensor.cpp index 498119f0733..64b2b6f8865 100644 --- a/eval/src/vespa/eval/eval/simple_tensor.cpp +++ b/eval/src/vespa/eval/eval/simple_tensor.cpp @@ -3,26 +3,44 @@ #include "simple_tensor.h" #include "simple_tensor_engine.h" #include "operation.h" -#include "codec.h" #include #include #include #include #include -using namespace vespalib::eval::codec; - namespace vespalib { namespace eval { using Address = SimpleTensor::Address; using Cell = SimpleTensor::Cell; using Cells = SimpleTensor::Cells; +using IndexList = std::vector; using Label = SimpleTensor::Label; using CellRef = std::reference_wrapper; +using CellType = ValueType::CellType; namespace { +constexpr uint32_t DOUBLE_CELL_TYPE = 0; +constexpr uint32_t FLOAT_CELL_TYPE = 1; + +uint32_t cell_type_to_id(CellType cell_type) { + switch (cell_type) { + case CellType::DOUBLE: return DOUBLE_CELL_TYPE; + case CellType::FLOAT: return FLOAT_CELL_TYPE; + } + abort(); +} + +CellType id_to_cell_type(uint32_t id) { + switch (id) { + case DOUBLE_CELL_TYPE: return CellType::DOUBLE; + case FLOAT_CELL_TYPE: return CellType::FLOAT; + } + abort(); +} + void assert_type(const ValueType &type) { (void) type; assert(type.is_double() || type.is_tensor()); @@ -87,6 +105,35 @@ const vespalib::string &reverse_rename(const vespalib::string &name, return name; } +/** + * Meta information about how a type can be decomposed into mapped and + * indexed dimensions and also how large each block is. A block is a + * dense-subspace consisting of all indexed dimensions that is + * uniquely specified by the labels of all mapped dimensions. + **/ +struct TypeMeta { + IndexList mapped; + IndexList indexed; + size_t block_size; + CellType cell_type; + explicit TypeMeta(const ValueType &type) + : mapped(), + indexed(), + block_size(1), + cell_type(type.cell_type()) + { + for (size_t i = 0; i < type.dimensions().size(); ++i) { + const auto &dimension = type.dimensions()[i]; + if (dimension.is_mapped()) { + mapped.push_back(i); + } else { + block_size *= dimension.size; + indexed.push_back(i); + } + } + } + ~TypeMeta() {} +}; /** * Helper class used when building SimpleTensors. While a tensor @@ -391,12 +438,95 @@ public: } }; +struct Format { + bool is_sparse; + bool is_dense; + bool with_cell_type; + uint32_t tag; + explicit Format(const TypeMeta &meta) + : is_sparse(meta.mapped.size() > 0), + is_dense((meta.indexed.size() > 0) || !is_sparse), + with_cell_type(meta.cell_type != CellType::DOUBLE), + tag((is_sparse ? 0x1 : 0) | (is_dense ? 0x2 : 0) | (with_cell_type ? 0x4 : 0)) {} + explicit Format(uint32_t tag_in) + : is_sparse((tag_in & 0x1) != 0), + is_dense((tag_in & 0x2) != 0), + with_cell_type((tag_in & 0x4) != 0), + tag(tag_in) {} + ~Format() {} +}; + +void maybe_encode_cell_type(nbostream &output, const Format &format, const TypeMeta &meta) { + if (format.with_cell_type) { + output.putInt1_4Bytes(cell_type_to_id(meta.cell_type)); + } +} + +void encode_type(nbostream &output, const Format &format, const ValueType &type, const TypeMeta &meta) { + maybe_encode_cell_type(output, format, meta); + if (format.is_sparse) { + output.putInt1_4Bytes(meta.mapped.size()); + for (size_t idx: meta.mapped) { + output.writeSmallString(type.dimensions()[idx].name); + } + } + if (format.is_dense) { + output.putInt1_4Bytes(meta.indexed.size()); + for (size_t idx: meta.indexed) { + output.writeSmallString(type.dimensions()[idx].name); + output.putInt1_4Bytes(type.dimensions()[idx].size); + } + } +} + +void maybe_encode_num_blocks(nbostream &output, const TypeMeta &meta, size_t num_blocks) { + if ((meta.mapped.size() > 0)) { + output.putInt1_4Bytes(num_blocks); + } +} + void encode_mapped_labels(nbostream &output, const TypeMeta &meta, const Address &addr) { for (size_t idx: meta.mapped) { output.writeSmallString(addr[idx].name); } } +CellType maybe_decode_cell_type(nbostream &input, const Format &format) { + if (format.with_cell_type) { + return id_to_cell_type(input.getInt1_4Bytes()); + } + return CellType::DOUBLE; +} + +ValueType decode_type(nbostream &input, const Format &format) { + CellType cell_type = maybe_decode_cell_type(input, format); + std::vector dim_list; + if (format.is_sparse) { + size_t cnt = input.getInt1_4Bytes(); + for (size_t i = 0; i < cnt; ++i) { + vespalib::string name; + input.readSmallString(name); + dim_list.emplace_back(name); + } + } + if (format.is_dense) { + size_t cnt = input.getInt1_4Bytes(); + for (size_t i = 0; i < cnt; ++i) { + vespalib::string name; + input.readSmallString(name); + dim_list.emplace_back(name, input.getInt1_4Bytes()); + } + } + return ValueType::tensor_type(std::move(dim_list), cell_type); +} + +size_t maybe_decode_num_blocks(nbostream &input, const TypeMeta &meta, const Format &format) { + if ((meta.mapped.size() > 0) || !format.is_dense) { + return input.getInt1_4Bytes(); + } + return 1; +} + void decode_mapped_labels(nbostream &input, const TypeMeta &meta, Address &addr) { for (size_t idx: meta.mapped) { vespalib::string name; diff --git a/eval/src/vespa/eval/eval/simple_value.cpp b/eval/src/vespa/eval/eval/simple_value.cpp index 2c958f2150a..343e61fb8fc 100644 --- a/eval/src/vespa/eval/eval/simple_value.cpp +++ b/eval/src/vespa/eval/eval/simple_value.cpp @@ -3,7 +3,6 @@ #include "simple_value.h" #include "tensor_spec.h" #include "inline_operation.h" -#include "codec.h" #include #include #include diff --git a/eval/src/vespa/eval/eval/value_codec.cpp b/eval/src/vespa/eval/eval/value_codec.cpp index 90a3016d0ab..a9121bb4aa9 100644 --- a/eval/src/vespa/eval/eval/value_codec.cpp +++ b/eval/src/vespa/eval/eval/value_codec.cpp @@ -1,15 +1,148 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "value_codec.h" -#include "codec.h" +#include #include -using namespace vespalib::eval::codec; - namespace vespalib::eval { namespace { +using CellType = ValueType::CellType; +using IndexList = std::vector; + +constexpr uint32_t DOUBLE_CELL_TYPE = 0; +constexpr uint32_t FLOAT_CELL_TYPE = 1; + +inline uint32_t cell_type_to_id(CellType cell_type) { + switch (cell_type) { + case CellType::DOUBLE: return DOUBLE_CELL_TYPE; + case CellType::FLOAT: return FLOAT_CELL_TYPE; + } + abort(); +} + +inline CellType id_to_cell_type(uint32_t id) { + switch (id) { + case DOUBLE_CELL_TYPE: return CellType::DOUBLE; + case FLOAT_CELL_TYPE: return CellType::FLOAT; + } + abort(); +} + +/** + * Meta information about how a type can be decomposed into mapped and + * indexed dimensions and also how large each block is. A block is a + * dense-subspace consisting of all indexed dimensions that is + * uniquely specified by the labels of all mapped dimensions. + **/ +struct TypeMeta { + IndexList mapped; + IndexList indexed; + size_t block_size; + CellType cell_type; + explicit TypeMeta(const ValueType &type) + : mapped(), + indexed(), + block_size(1), + cell_type(type.cell_type()) + { + for (size_t i = 0; i < type.dimensions().size(); ++i) { + const auto &dimension = type.dimensions()[i]; + if (dimension.is_mapped()) { + mapped.push_back(i); + } else { + block_size *= dimension.size; + indexed.push_back(i); + } + } + } + ~TypeMeta() {} +}; + +struct Format { + bool is_sparse; + bool is_dense; + bool with_cell_type; + uint32_t tag; + explicit Format(const TypeMeta &meta) + : is_sparse(meta.mapped.size() > 0), + is_dense((meta.indexed.size() > 0) || !is_sparse), + with_cell_type(meta.cell_type != CellType::DOUBLE), + tag((is_sparse ? 0x1 : 0) | (is_dense ? 0x2 : 0) | (with_cell_type ? 0x4 : 0)) {} + explicit Format(uint32_t tag_in) + : is_sparse((tag_in & 0x1) != 0), + is_dense((tag_in & 0x2) != 0), + with_cell_type((tag_in & 0x4) != 0), + tag(tag_in) {} + ~Format() {} +}; + +void maybe_encode_cell_type(nbostream &output, const Format &format, const TypeMeta &meta) { + if (format.with_cell_type) { + output.putInt1_4Bytes(cell_type_to_id(meta.cell_type)); + } +} + +void encode_type(nbostream &output, const Format &format, const ValueType &type, const TypeMeta &meta) { + maybe_encode_cell_type(output, format, meta); + if (format.is_sparse) { + output.putInt1_4Bytes(meta.mapped.size()); + for (size_t idx: meta.mapped) { + output.writeSmallString(type.dimensions()[idx].name); + } + } + if (format.is_dense) { + output.putInt1_4Bytes(meta.indexed.size()); + for (size_t idx: meta.indexed) { + output.writeSmallString(type.dimensions()[idx].name); + output.putInt1_4Bytes(type.dimensions()[idx].size); + } + } +} + +void maybe_encode_num_blocks(nbostream &output, const TypeMeta &meta, size_t num_blocks) { + if ((meta.mapped.size() > 0)) { + output.putInt1_4Bytes(num_blocks); + } +} + +CellType maybe_decode_cell_type(nbostream &input, const Format &format) { + if (format.with_cell_type) { + return id_to_cell_type(input.getInt1_4Bytes()); + } + return CellType::DOUBLE; +} + +ValueType decode_type(nbostream &input, const Format &format) { + CellType cell_type = maybe_decode_cell_type(input, format); + std::vector dim_list; + if (format.is_sparse) { + size_t cnt = input.getInt1_4Bytes(); + for (size_t i = 0; i < cnt; ++i) { + vespalib::string name; + input.readSmallString(name); + dim_list.emplace_back(name); + } + } + if (format.is_dense) { + size_t cnt = input.getInt1_4Bytes(); + for (size_t i = 0; i < cnt; ++i) { + vespalib::string name; + input.readSmallString(name); + dim_list.emplace_back(name, input.getInt1_4Bytes()); + } + } + return ValueType::tensor_type(std::move(dim_list), cell_type); +} + +size_t maybe_decode_num_blocks(nbostream &input, const TypeMeta &meta, const Format &format) { + if ((meta.mapped.size() > 0) || !format.is_dense) { + return input.getInt1_4Bytes(); + } + return 1; +} + void encode_mapped_labels(nbostream &output, size_t num_mapped_dims, const std::vector &addr) { for (size_t i = 0; i < num_mapped_dims; ++i) { output.writeSmallString(addr[i]); -- cgit v1.2.3 From e6bb3101ca84e92e2335d1710e42c04ac562b1d2 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Wed, 23 Sep 2020 13:30:05 +0000 Subject: move Spec conversion to value_codec.cpp --- eval/src/vespa/eval/eval/simple_value.cpp | 89 ------------------------------ eval/src/vespa/eval/eval/value_codec.cpp | 90 +++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 89 deletions(-) diff --git a/eval/src/vespa/eval/eval/simple_value.cpp b/eval/src/vespa/eval/eval/simple_value.cpp index 343e61fb8fc..3bf5351f0d2 100644 --- a/eval/src/vespa/eval/eval/simple_value.cpp +++ b/eval/src/vespa/eval/eval/simple_value.cpp @@ -1,7 +1,6 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "simple_value.h" -#include "tensor_spec.h" #include "inline_operation.h" #include #include @@ -25,80 +24,6 @@ struct CreateSimpleValueBuilderBase { } }; -struct CreateValueFromTensorSpec { - template static std::unique_ptr invoke(const ValueType &type, const TensorSpec &spec, const ValueBuilderFactory &factory) { - using SparseKey = std::vector; - using DenseMap = std::map; - std::map map; - for (const auto &entry: spec.cells()) { - SparseKey sparse_key; - size_t dense_key = 0; - for (const auto &dim: type.dimensions()) { - auto pos = entry.first.find(dim.name); - assert(pos != entry.first.end()); - assert(pos->second.is_mapped() == dim.is_mapped()); - if (dim.is_mapped()) { - sparse_key.emplace_back(pos->second.name); - } else { - dense_key = (dense_key * dim.size) + pos->second.index; - } - } - map[sparse_key][dense_key] = entry.second; - } - // hack for passing some (invalid?) unit tests - if (spec.cells().empty() && type.count_mapped_dimensions() == 0) { - map[SparseKey()][0] = 0; - } - auto builder = factory.create_value_builder(type, type.count_mapped_dimensions(), type.dense_subspace_size(), map.size()); - for (const auto &entry: map) { - auto subspace = builder->add_subspace(entry.first); - for (const auto &cell: entry.second) { - subspace[cell.first] = cell.second; - } - } - return builder->build(std::move(builder)); - } -}; - -struct CreateTensorSpecFromValue { - template static TensorSpec invoke(const Value &value) { - auto cells = value.cells().typify(); - TensorSpec spec(value.type().to_spec()); - size_t subspace_id = 0; - size_t subspace_size = value.type().dense_subspace_size(); - std::vector labels(value.type().count_mapped_dimensions()); - std::vector label_refs; - for (auto &label: labels) { - label_refs.push_back(&label); - } - auto view = value.index().create_view({}); - view->lookup({}); - while (view->next_result(label_refs, subspace_id)) { - size_t label_idx = 0; - TensorSpec::Address addr; - for (const auto &dim: value.type().dimensions()) { - if (dim.is_mapped()) { - addr.emplace(dim.name, labels[label_idx++]); - } - } - for (size_t i = 0; i < subspace_size; ++i) { - size_t dense_key = i; - for (auto dim = value.type().dimensions().rbegin(); - dim != value.type().dimensions().rend(); ++dim) - { - if (dim->is_indexed()) { - size_t label = dense_key % dim->size; - addr.emplace(dim->name, label).first->second = TensorSpec::Label(label); - dense_key /= dim->size; - } - } - spec.add(addr, cells[(subspace_size * subspace_id) + i]); - } - } - return spec; - } -}; - class SimpleValueView : public Value::Index::View { private: using Addr = std::vector; @@ -400,18 +325,4 @@ std::unique_ptr new_join(const Value &a, const Value &b, join_fun_t funct //----------------------------------------------------------------------------- -std::unique_ptr value_from_spec(const TensorSpec &spec, const ValueBuilderFactory &factory) { - ValueType type = ValueType::from_spec(spec.type()); - assert(!type.is_error()); - return typify_invoke<1,TypifyCellType,CreateValueFromTensorSpec>(type.cell_type(), type, spec, factory); -} - -//----------------------------------------------------------------------------- - -TensorSpec spec_from_value(const Value &value) { - return typify_invoke<1,TypifyCellType,CreateTensorSpecFromValue>(value.type().cell_type(), value); -} - -//----------------------------------------------------------------------------- - } // namespace diff --git a/eval/src/vespa/eval/eval/value_codec.cpp b/eval/src/vespa/eval/eval/value_codec.cpp index a9121bb4aa9..f642f90c292 100644 --- a/eval/src/vespa/eval/eval/value_codec.cpp +++ b/eval/src/vespa/eval/eval/value_codec.cpp @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "value_codec.h" +#include "tensor_spec.h" #include #include @@ -189,6 +190,80 @@ struct ContentDecoder { } }; +struct CreateValueFromTensorSpec { + template static std::unique_ptr invoke(const ValueType &type, const TensorSpec &spec, const ValueBuilderFactory &factory) { + using SparseKey = std::vector; + using DenseMap = std::map; + std::map map; + for (const auto &entry: spec.cells()) { + SparseKey sparse_key; + size_t dense_key = 0; + for (const auto &dim: type.dimensions()) { + auto pos = entry.first.find(dim.name); + assert(pos != entry.first.end()); + assert(pos->second.is_mapped() == dim.is_mapped()); + if (dim.is_mapped()) { + sparse_key.emplace_back(pos->second.name); + } else { + dense_key = (dense_key * dim.size) + pos->second.index; + } + } + map[sparse_key][dense_key] = entry.second; + } + // hack for passing some (invalid?) unit tests + if (spec.cells().empty() && type.count_mapped_dimensions() == 0) { + map[SparseKey()][0] = 0; + } + auto builder = factory.create_value_builder(type, type.count_mapped_dimensions(), type.dense_subspace_size(), map.size()); + for (const auto &entry: map) { + auto subspace = builder->add_subspace(entry.first); + for (const auto &cell: entry.second) { + subspace[cell.first] = cell.second; + } + } + return builder->build(std::move(builder)); + } +}; + +struct CreateTensorSpecFromValue { + template static TensorSpec invoke(const Value &value) { + auto cells = value.cells().typify(); + TensorSpec spec(value.type().to_spec()); + size_t subspace_id = 0; + size_t subspace_size = value.type().dense_subspace_size(); + std::vector labels(value.type().count_mapped_dimensions()); + std::vector label_refs; + for (auto &label: labels) { + label_refs.push_back(&label); + } + auto view = value.index().create_view({}); + view->lookup({}); + while (view->next_result(label_refs, subspace_id)) { + size_t label_idx = 0; + TensorSpec::Address addr; + for (const auto &dim: value.type().dimensions()) { + if (dim.is_mapped()) { + addr.emplace(dim.name, labels[label_idx++]); + } + } + for (size_t i = 0; i < subspace_size; ++i) { + size_t dense_key = i; + for (auto dim = value.type().dimensions().rbegin(); + dim != value.type().dimensions().rend(); ++dim) + { + if (dim->is_indexed()) { + size_t label = dense_key % dim->size; + addr.emplace(dim->name, label).first->second = TensorSpec::Label(label); + dense_key /= dim->size; + } + } + spec.add(addr, cells[(subspace_size * subspace_id) + i]); + } + } + return spec; + } +}; + } // namespace void new_encode(const Value &value, nbostream &output) { @@ -232,4 +307,19 @@ std::unique_ptr new_decode(nbostream &input, const ValueBuilderFactory &f return typify_invoke<1,TypifyCellType,ContentDecoder>(meta.cell_type, input, state, factory); } +//----------------------------------------------------------------------------- + +std::unique_ptr value_from_spec(const TensorSpec &spec, const ValueBuilderFactory &factory) { + ValueType type = ValueType::from_spec(spec.type()); + assert(!type.is_error()); + return typify_invoke<1,TypifyCellType,CreateValueFromTensorSpec>(type.cell_type(), type, spec, factory); +} + +TensorSpec spec_from_value(const Value &value) { + return typify_invoke<1,TypifyCellType,CreateTensorSpecFromValue>(value.type().cell_type(), value); +} + +//----------------------------------------------------------------------------- + + } // namespace -- cgit v1.2.3 From 10169943144990208e11529109d30febccb49756 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Wed, 23 Sep 2020 13:32:05 +0000 Subject: cleanup hack --- eval/src/vespa/eval/eval/value_codec.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eval/src/vespa/eval/eval/value_codec.cpp b/eval/src/vespa/eval/eval/value_codec.cpp index f642f90c292..fb202490ed9 100644 --- a/eval/src/vespa/eval/eval/value_codec.cpp +++ b/eval/src/vespa/eval/eval/value_codec.cpp @@ -210,9 +210,9 @@ struct CreateValueFromTensorSpec { } map[sparse_key][dense_key] = entry.second; } - // hack for passing some (invalid?) unit tests - if (spec.cells().empty() && type.count_mapped_dimensions() == 0) { - map[SparseKey()][0] = 0; + // if spec is missing the required dense space, add it here: + if (map.empty() && type.count_mapped_dimensions() == 0) { + map[{}][0] = 0; } auto builder = factory.create_value_builder(type, type.count_mapped_dimensions(), type.dense_subspace_size(), map.size()); for (const auto &entry: map) { -- cgit v1.2.3 From 05442b391e2feedae7cbea9dd53a01caf26da5f1 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Wed, 23 Sep 2020 13:57:05 +0000 Subject: add unit test for value_codec.h functions --- eval/CMakeLists.txt | 1 + eval/src/tests/eval/value_codec/CMakeLists.txt | 10 + .../tests/eval/value_codec/value_codec_test.cpp | 281 +++++++++++++++++++++ 3 files changed, 292 insertions(+) create mode 100644 eval/src/tests/eval/value_codec/CMakeLists.txt create mode 100644 eval/src/tests/eval/value_codec/value_codec_test.cpp diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index d4189fb3256..c69425d5387 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -29,6 +29,7 @@ vespa_define_module( src/tests/eval/tensor_lambda src/tests/eval/tensor_spec src/tests/eval/value_cache + src/tests/eval/value_codec src/tests/eval/value_type src/tests/gp/ponder_nov2017 src/tests/tensor/dense_add_dimension_optimizer diff --git a/eval/src/tests/eval/value_codec/CMakeLists.txt b/eval/src/tests/eval/value_codec/CMakeLists.txt new file mode 100644 index 00000000000..aa1adf10136 --- /dev/null +++ b/eval/src/tests/eval/value_codec/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +vespa_add_executable(eval_value_codec_test_app TEST + SOURCES + value_codec_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_value_codec_test_app COMMAND eval_value_codec_test_app) diff --git a/eval/src/tests/eval/value_codec/value_codec_test.cpp b/eval/src/tests/eval/value_codec/value_codec_test.cpp new file mode 100644 index 00000000000..43bbebfc094 --- /dev/null +++ b/eval/src/tests/eval/value_codec/value_codec_test.cpp @@ -0,0 +1,281 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include +#include +#include +#include +#include +#include +#include + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::eval::test; + +using vespalib::make_string_short::fmt; + +std::vector layouts = { + {}, + {x(3)}, + {x(3),y(5)}, + {x(3),y(5),z(7)}, + float_cells({x(3),y(5),z(7)}), + {x({"a","b","c"})}, + {x({"a","b","c"}),y({"foo","bar"})}, + {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), + {x(3),y({"foo", "bar"}),z(7)}, + {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) +}; + +struct Factory { + SimpleValueBuilderFactory simple; + std::unique_ptr from_spec(const TensorSpec &spec) { + return value_from_spec(spec, simple); + } + std::unique_ptr decode(nbostream &input) { + return new_decode(input, simple); + } +} simple_factory; + +TEST(ValueCodecTest, simple_values_can_be_converted_from_and_to_tensor_spec) { + for (const auto &layout: layouts) { + TensorSpec expect = spec(layout, N()); + std::unique_ptr value = simple_factory.from_spec(expect); + TensorSpec actual = spec_from_value(*value); + EXPECT_EQ(actual, expect); + } +} + + +TEST(ValueCodecTest, require_that_simple_tensors_can_be_built_using_tensor_spec) { + TensorSpec spec("tensor(w{},x[2],y{},z[2])"); + spec.add({{"w", "xxx"}, {"x", 0}, {"y", "xxx"}, {"z", 0}}, 1.0) + .add({{"w", "xxx"}, {"x", 0}, {"y", "yyy"}, {"z", 1}}, 2.0) + .add({{"w", "yyy"}, {"x", 1}, {"y", "xxx"}, {"z", 0}}, 3.0) + .add({{"w", "yyy"}, {"x", 1}, {"y", "yyy"}, {"z", 1}}, 4.0); + Value::UP tensor = simple_factory.from_spec(spec); + TensorSpec full_spec("tensor(w{},x[2],y{},z[2])"); + full_spec + .add({{"w", "xxx"}, {"x", 0}, {"y", "xxx"}, {"z", 0}}, 1.0) + .add({{"w", "xxx"}, {"x", 0}, {"y", "xxx"}, {"z", 1}}, 0.0) + .add({{"w", "xxx"}, {"x", 0}, {"y", "yyy"}, {"z", 0}}, 0.0) + .add({{"w", "xxx"}, {"x", 0}, {"y", "yyy"}, {"z", 1}}, 2.0) + .add({{"w", "xxx"}, {"x", 1}, {"y", "xxx"}, {"z", 0}}, 0.0) + .add({{"w", "xxx"}, {"x", 1}, {"y", "xxx"}, {"z", 1}}, 0.0) + .add({{"w", "xxx"}, {"x", 1}, {"y", "yyy"}, {"z", 0}}, 0.0) + .add({{"w", "xxx"}, {"x", 1}, {"y", "yyy"}, {"z", 1}}, 0.0) + .add({{"w", "yyy"}, {"x", 0}, {"y", "xxx"}, {"z", 0}}, 0.0) + .add({{"w", "yyy"}, {"x", 0}, {"y", "xxx"}, {"z", 1}}, 0.0) + .add({{"w", "yyy"}, {"x", 0}, {"y", "yyy"}, {"z", 0}}, 0.0) + .add({{"w", "yyy"}, {"x", 0}, {"y", "yyy"}, {"z", 1}}, 0.0) + .add({{"w", "yyy"}, {"x", 1}, {"y", "xxx"}, {"z", 0}}, 3.0) + .add({{"w", "yyy"}, {"x", 1}, {"y", "xxx"}, {"z", 1}}, 0.0) + .add({{"w", "yyy"}, {"x", 1}, {"y", "yyy"}, {"z", 0}}, 0.0) + .add({{"w", "yyy"}, {"x", 1}, {"y", "yyy"}, {"z", 1}}, 4.0); + Value::UP full_tensor = simple_factory.from_spec(full_spec); + EXPECT_EQUAL(full_spec, spec_from_value(*tensor)); + EXPECT_EQUAL(full_spec, spec_from_value(*full_tensor)); +}; + +//----------------------------------------------------------------------------- + +vespalib::string make_type_spec(bool use_float, const vespalib::string &dims) { + vespalib::string type_spec = "tensor"; + if (use_float) { + type_spec.append(""); + } + type_spec.append(dims); + return type_spec; +} + +struct TensorExample { + virtual ~TensorExample(); + virtual TensorSpec make_spec(bool use_float) const = 0; + virtual std::unique_ptr make_tensor(bool use_float) const = 0; + virtual void encode_default(nbostream &dst) const = 0; + virtual void encode_with_double(nbostream &dst) const = 0; + virtual void encode_with_float(nbostream &dst) const = 0; + void verify_encode_decode() const { + nbostream expect_default; + nbostream expect_double; + nbostream expect_float; + encode_default(expect_default); + encode_with_double(expect_double); + encode_with_float(expect_float); + nbostream data_double; + nbostream data_float; + new_encode(*make_tensor(false), data_double); + new_encode(*make_tensor(true), data_float); + EXPECT_EQ(Memory(data_double.peek(), data_double.size()), + Memory(expect_default.peek(), expect_default.size())); + EXPECT_EQ(Memory(data_float.peek(), data_float.size()), + Memory(expect_float.peek(), expect_float.size())); + EXPECT_EQ(spec_from_value(*simple_factory.decode(expect_default)), make_spec(false)); + EXPECT_EQ(spec_from_value(*simple_factory.decode(expect_double)), make_spec(false)); + EXPECT_EQ(spec_from_value(*simple_factory.decode(expect_float)), make_spec(true)); + } +}; +TensorExample::~TensorExample() = default; + +//----------------------------------------------------------------------------- + +struct SparseTensorExample : TensorExample { + TensorSpec make_spec(bool use_float) const override { + return TensorSpec(make_type_spec(use_float, "(x{},y{})")) + .add({{"x","a"},{"y","a"}}, 1) + .add({{"x","a"},{"y","b"}}, 2) + .add({{"x","b"},{"y","a"}}, 3); + } + std::unique_ptr make_tensor(bool use_float) const override { + return simple_factory.from_spec(make_spec(use_float)); + } + template + void encode_inner(nbostream &dst) const { + dst.putInt1_4Bytes(2); + dst.writeSmallString("x"); + dst.writeSmallString("y"); + dst.putInt1_4Bytes(3); + dst.writeSmallString("a"); + dst.writeSmallString("a"); + dst << (T) 1; + dst.writeSmallString("a"); + dst.writeSmallString("b"); + dst << (T) 2; + dst.writeSmallString("b"); + dst.writeSmallString("a"); + dst << (T) 3; + } + void encode_default(nbostream &dst) const override { + dst.putInt1_4Bytes(1); + encode_inner(dst); + } + void encode_with_double(nbostream &dst) const override { + dst.putInt1_4Bytes(5); + dst.putInt1_4Bytes(0); + encode_inner(dst); + } + void encode_with_float(nbostream &dst) const override { + dst.putInt1_4Bytes(5); + dst.putInt1_4Bytes(1); + encode_inner(dst); + } +}; + +TEST(ValueCodecTest, require_that_sparse_tensors_can_be_encoded_and_decoded) { + SparseTensorExample f1; + f1.verify_encode_decode(); +} + +//----------------------------------------------------------------------------- + +struct DenseTensorExample : TensorExample { + TensorSpec make_spec(bool use_float) const override { + return TensorSpec(make_type_spec(use_float, "(x[3],y[2])")) + .add({{"x",0},{"y",0}}, 1) + .add({{"x",0},{"y",1}}, 2) + .add({{"x",1},{"y",0}}, 3) + .add({{"x",1},{"y",1}}, 4) + .add({{"x",2},{"y",0}}, 5) + .add({{"x",2},{"y",1}}, 6); + } + std::unique_ptr make_tensor(bool use_float) const override { + return simple_factory.from_spec(make_spec(use_float)); + } + template + void encode_inner(nbostream &dst) const { + dst.putInt1_4Bytes(2); + dst.writeSmallString("x"); + dst.putInt1_4Bytes(3); + dst.writeSmallString("y"); + dst.putInt1_4Bytes(2); + dst << (T) 1; + dst << (T) 2; + dst << (T) 3; + dst << (T) 4; + dst << (T) 5; + dst << (T) 6; + } + void encode_default(nbostream &dst) const override { + dst.putInt1_4Bytes(2); + encode_inner(dst); + } + void encode_with_double(nbostream &dst) const override { + dst.putInt1_4Bytes(6); + dst.putInt1_4Bytes(0); + encode_inner(dst); + } + void encode_with_float(nbostream &dst) const override { + dst.putInt1_4Bytes(6); + dst.putInt1_4Bytes(1); + encode_inner(dst); + } +}; + +TEST(ValueCodecTest, require_that_dense_tensors_can_be_encoded_and_decoded) { + DenseTensorExample f1; + f1.verify_encode_decode(); +} + +//----------------------------------------------------------------------------- + +struct MixedTensorExample : TensorExample { + TensorSpec make_spec(bool use_float) const override { + return TensorSpec(make_type_spec(use_float, "(x{},y{},z[2])")) + .add({{"x","a"},{"y","a"},{"z",0}}, 1) + .add({{"x","a"},{"y","a"},{"z",1}}, 2) + .add({{"x","a"},{"y","b"},{"z",0}}, 3) + .add({{"x","a"},{"y","b"},{"z",1}}, 4) + .add({{"x","b"},{"y","a"},{"z",0}}, 5) + .add({{"x","b"},{"y","a"},{"z",1}}, 6); + } + std::unique_ptr make_tensor(bool use_float) const override { + return simple_factory.from_spec(make_spec(use_float)); + } + template + void encode_inner(nbostream &dst) const { + dst.putInt1_4Bytes(2); + dst.writeSmallString("x"); + dst.writeSmallString("y"); + dst.putInt1_4Bytes(1); + dst.writeSmallString("z"); + dst.putInt1_4Bytes(2); + dst.putInt1_4Bytes(3); + dst.writeSmallString("a"); + dst.writeSmallString("a"); + dst << (T) 1; + dst << (T) 2; + dst.writeSmallString("a"); + dst.writeSmallString("b"); + dst << (T) 3; + dst << (T) 4; + dst.writeSmallString("b"); + dst.writeSmallString("a"); + dst << (T) 5; + dst << (T) 6; + } + void encode_default(nbostream &dst) const override { + dst.putInt1_4Bytes(3); + encode_inner(dst); + } + void encode_with_double(nbostream &dst) const override { + dst.putInt1_4Bytes(7); + dst.putInt1_4Bytes(0); + encode_inner(dst); + } + void encode_with_float(nbostream &dst) const override { + dst.putInt1_4Bytes(7); + dst.putInt1_4Bytes(1); + encode_inner(dst); + } +}; + +TEST(ValueCodecTest, require_that_mixed_tensors_can_be_encoded_and_decoded) { + MixedTensorExample f1; + f1.verify_encode_decode(); +} + +//----------------------------------------------------------------------------- + +GTEST_MAIN_RUN_ALL_TESTS() -- cgit v1.2.3 From ae9af2ba9c3c476ef1e228a0bc36928c013ae3f3 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Wed, 23 Sep 2020 16:00:55 +0200 Subject: Redeploy interval should not change --- .../vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 02ceee37e51..dd5d9d040b5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.HostLivenessTracker; import com.yahoo.config.provision.InfraDeployer; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; -import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -180,7 +179,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { nodeMetricsCollectionInterval = Duration.ofMinutes(1); operatorChangeRedeployInterval = Duration.ofMinutes(1); osUpgradeActivatorInterval = zone.system().isCd() ? Duration.ofSeconds(30) : Duration.ofMinutes(5); - periodicRedeployInterval = deploymentExistsOnAllConfigServers ? Duration.ofMinutes(90) : Duration.ofMinutes(30); + periodicRedeployInterval = Duration.ofMinutes(30); provisionedExpiry = Duration.ofHours(4); rebalancerInterval = deploymentExistsOnAllConfigServers ? Duration.ofMinutes(120) : Duration.ofMinutes(40); redeployMaintainerInterval = Duration.ofMinutes(1); -- cgit v1.2.3 From b1181249596fdbff748aeaad958c3d329cb56c62 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Wed, 23 Sep 2020 14:08:53 +0000 Subject: simplify value_codec some --- eval/src/vespa/eval/eval/value_codec.cpp | 45 ++++++++++++++++---------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/eval/src/vespa/eval/eval/value_codec.cpp b/eval/src/vespa/eval/eval/value_codec.cpp index fb202490ed9..b5a1ccb92fa 100644 --- a/eval/src/vespa/eval/eval/value_codec.cpp +++ b/eval/src/vespa/eval/eval/value_codec.cpp @@ -40,12 +40,10 @@ inline CellType id_to_cell_type(uint32_t id) { struct TypeMeta { IndexList mapped; IndexList indexed; - size_t block_size; CellType cell_type; explicit TypeMeta(const ValueType &type) : mapped(), indexed(), - block_size(1), cell_type(type.cell_type()) { for (size_t i = 0; i < type.dimensions().size(); ++i) { @@ -53,7 +51,6 @@ struct TypeMeta { if (dimension.is_mapped()) { mapped.push_back(i); } else { - block_size *= dimension.size; indexed.push_back(i); } } @@ -88,7 +85,7 @@ void maybe_encode_cell_type(nbostream &output, const Format &format, const TypeM void encode_type(nbostream &output, const Format &format, const ValueType &type, const TypeMeta &meta) { maybe_encode_cell_type(output, format, meta); if (format.is_sparse) { - output.putInt1_4Bytes(meta.mapped.size()); + output.putInt1_4Bytes(type.count_mapped_dimensions()); for (size_t idx: meta.mapped) { output.writeSmallString(type.dimensions()[idx].name); } @@ -102,8 +99,8 @@ void encode_type(nbostream &output, const Format &format, const ValueType &type, } } -void maybe_encode_num_blocks(nbostream &output, const TypeMeta &meta, size_t num_blocks) { - if ((meta.mapped.size() > 0)) { +void maybe_encode_num_blocks(nbostream &output, bool has_mapped_dims, size_t num_blocks) { + if (has_mapped_dims) { output.putInt1_4Bytes(num_blocks); } } @@ -137,8 +134,8 @@ ValueType decode_type(nbostream &input, const Format &format) { return ValueType::tensor_type(std::move(dim_list), cell_type); } -size_t maybe_decode_num_blocks(nbostream &input, const TypeMeta &meta, const Format &format) { - if ((meta.mapped.size() > 0) || !format.is_dense) { +size_t maybe_decode_num_blocks(nbostream &input, bool has_mapped_dims, const Format &format) { + if (has_mapped_dims || !format.is_dense) { return input.getInt1_4Bytes(); } return 1; @@ -171,7 +168,7 @@ void decode_cells(nbostream &input, size_t num_cells, ArrayRef dst) struct DecodeState { const ValueType &type; - const size_t block_size; + const size_t subspace_size; const size_t num_blocks; const size_t num_mapped_dims; }; @@ -180,11 +177,11 @@ struct ContentDecoder { template static std::unique_ptr invoke(nbostream &input, const DecodeState &state, const ValueBuilderFactory &factory) { std::vector address(state.num_mapped_dims); - auto builder = factory.create_value_builder(state.type, state.num_mapped_dims, state.block_size, state.num_blocks); + auto builder = factory.create_value_builder(state.type, state.num_mapped_dims, state.subspace_size, state.num_blocks); for (size_t i = 0; i < state.num_blocks; ++i) { decode_mapped_labels(input, state.num_mapped_dims, address); auto block_cells = builder->add_subspace(address); - decode_cells(input, state.block_size, block_cells); + decode_cells(input, state.subspace_size, block_cells); } return builder->build(std::move(builder)); } @@ -268,30 +265,32 @@ struct CreateTensorSpecFromValue { void new_encode(const Value &value, nbostream &output) { TypeMeta meta(value.type()); + size_t num_mapped_dims = value.type().count_mapped_dimensions(); + size_t dense_subspace_size = value.type().dense_subspace_size(); Format format(meta); output.putInt1_4Bytes(format.tag); encode_type(output, format, value.type(), meta); - maybe_encode_num_blocks(output, meta, value.cells().size / meta.block_size); - std::vector address(meta.mapped.size()); - std::vector a_refs(meta.mapped.size()); - for (size_t i = 0; i < meta.mapped.size(); ++i) { + maybe_encode_num_blocks(output, (num_mapped_dims > 0), value.cells().size / dense_subspace_size); + std::vector address(num_mapped_dims); + std::vector a_refs(num_mapped_dims); + for (size_t i = 0; i < num_mapped_dims; ++i) { a_refs[i] = &address[i]; } auto view = value.index().create_view({}); view->lookup({}); size_t subspace; while (view->next_result(a_refs, subspace)) { - encode_mapped_labels(output, meta.mapped.size(), address); + encode_mapped_labels(output, num_mapped_dims, address); if (meta.cell_type == CellType::FLOAT) { auto iter = value.cells().typify().begin(); - iter += (subspace * meta.block_size); - for (size_t i = 0; i < meta.block_size; ++i) { + iter += (subspace * dense_subspace_size); + for (size_t i = 0; i < dense_subspace_size; ++i) { output << (float) *iter++; } } else { auto iter = value.cells().typify().begin(); - iter += (subspace * meta.block_size); - for (size_t i = 0; i < meta.block_size; ++i) { + iter += (subspace * dense_subspace_size); + for (size_t i = 0; i < dense_subspace_size; ++i) { output << *iter++; } } @@ -301,9 +300,11 @@ void new_encode(const Value &value, nbostream &output) { std::unique_ptr new_decode(nbostream &input, const ValueBuilderFactory &factory) { Format format(input.getInt1_4Bytes()); ValueType type = decode_type(input, format); + size_t num_mapped_dims = type.count_mapped_dimensions(); + size_t dense_subspace_size = type.dense_subspace_size(); TypeMeta meta(type); - const size_t num_blocks = maybe_decode_num_blocks(input, meta, format); - DecodeState state{type, meta.block_size, num_blocks, meta.mapped.size()}; + const size_t num_blocks = maybe_decode_num_blocks(input, (num_mapped_dims > 0), format); + DecodeState state{type, dense_subspace_size, num_blocks, num_mapped_dims}; return typify_invoke<1,TypifyCellType,ContentDecoder>(meta.cell_type, input, state, factory); } -- cgit v1.2.3 From 3a3a897fb1c2061438bbd999eaf5326dc07f0aaa Mon Sep 17 00:00:00 2001 From: Tor Brede Vekterli Date: Wed, 23 Sep 2020 14:14:32 +0000 Subject: Reorder equality checks so that common case is cheaper Node index and type are much more likely to differ than cluster. --- storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp index 8276587834a..1f1a2c602de 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp +++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp @@ -227,9 +227,9 @@ StorageMessageAddress::operator==(const StorageMessageAddress& other) const if (_protocol != other._protocol) return false; if (_type != other._type) return false; if (_type) { - if (_cluster != other._cluster) return false; if (_index != other._index) return false; if (_type != other._type) return false; + if (_cluster != other._cluster) return false; } return true; } -- cgit v1.2.3 From 135ac2fce54089d30ef1328efeea80ba87123627 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Wed, 23 Sep 2020 16:14:41 +0200 Subject: No need for different intervals, maintainer does not deploy --- .../vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index dd5d9d040b5..4f562d39607 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -186,7 +186,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { // Need to be long enough for deployment to be finished for all config model versions // Should be equal to timeout for deployments reservationExpiry = zone.system().isCd() ? Duration.ofMinutes(5) : Duration.ofMinutes(30); - scalingSuggestionsInterval = deploymentExistsOnAllConfigServers ? Duration.ofMinutes(91) : Duration.ofMinutes(31); + scalingSuggestionsInterval = Duration.ofMinutes(31); spareCapacityMaintenanceInterval = Duration.ofMinutes(10); throttlePolicy = NodeFailer.ThrottlePolicy.hosted; -- cgit v1.2.3 From 5ddaf3665febd32174cb54b4a43cd63b6631c1c2 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Wed, 23 Sep 2020 16:19:19 +0200 Subject: Check if we can deploy now outside of lock --- .../yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java index 0a94764d948..b3c76300173 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java @@ -79,9 +79,9 @@ public abstract class ApplicationMaintainer extends NodeRepositoryMaintainer { * @return whether it was successfully deployed */ protected final boolean deployWithLock(ApplicationId application) { + if ( ! canDeployNow(application)) return false; // redeployment is no longer needed try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, metric, nodeRepository())) { if ( ! deployment.isValid()) return false; // this will be done at another config server - if ( ! canDeployNow(application)) return false; // redeployment is no longer needed log.log(Level.INFO, application + " will be deployed, last deploy time " + getLastDeployTime(application)); return deployment.activate(); } finally { -- cgit v1.2.3 From 94b206e75e62b6e0878f66b5a4f1f071c187ff73 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Wed, 23 Sep 2020 16:25:03 +0200 Subject: Run some more maintainers with higher interval --- .../hosted/provision/maintenance/NodeRepositoryMaintenance.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 4f562d39607..7f57575974f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -169,7 +169,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final NodeFailer.ThrottlePolicy throttlePolicy; DefaultTimes(Zone zone, boolean deploymentExistsOnAllConfigServers) { - autoscalingInterval = Duration.ofMinutes(5); + autoscalingInterval = deploymentExistsOnAllConfigServers ? Duration.ofMinutes(15) : Duration.ofMinutes(5); dynamicProvisionerInterval = Duration.ofMinutes(5); failedExpirerInterval = Duration.ofMinutes(10); failGrace = Duration.ofMinutes(30); @@ -177,7 +177,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { loadBalancerExpirerInterval = Duration.ofMinutes(5); metricsInterval = Duration.ofMinutes(1); nodeMetricsCollectionInterval = Duration.ofMinutes(1); - operatorChangeRedeployInterval = Duration.ofMinutes(1); + operatorChangeRedeployInterval = deploymentExistsOnAllConfigServers ? Duration.ofMinutes(3) : Duration.ofMinutes(1); osUpgradeActivatorInterval = zone.system().isCd() ? Duration.ofSeconds(30) : Duration.ofMinutes(5); periodicRedeployInterval = Duration.ofMinutes(30); provisionedExpiry = Duration.ofHours(4); @@ -192,7 +192,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { if (zone.environment().equals(Environment.prod) && ! zone.system().isCd()) { inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy - retiredInterval = Duration.ofMinutes(10); + retiredInterval = deploymentExistsOnAllConfigServers ? Duration.ofMinutes(30) : Duration.ofMinutes(10); dirtyExpiry = Duration.ofHours(2); // enough time to clean the node retiredExpiry = Duration.ofDays(4); // give up migrating data after 4 days } else { -- cgit v1.2.3 From 6be01d9580160c186b9a4435258dfdf4fa788691 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Wed, 23 Sep 2020 16:54:17 +0200 Subject: Ignore force commit with serial number not greater than attribute vector create serial number. --- searchcore/src/tests/proton/attribute/attribute_test.cpp | 14 ++++++++++++++ .../vespa/searchcore/proton/attribute/attribute_writer.cpp | 4 +++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index 41de6827244..8711a21e5e6 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -192,6 +192,11 @@ public: void assertExecuteHistory(std::vector expExecuteHistory) { EXPECT_EQ(expExecuteHistory, _attributeFieldWriter->getExecuteHistory()); } + SerialNum test_force_commit(AttributeVector &attr, SerialNum serialNum) { + commit(serialNum); + _attributeFieldWriter->sync(); + return attr.getStatus().getLastSyncToken(); + } }; AttributeWriterTest::~AttributeWriterTest() = default; @@ -975,6 +980,15 @@ TEST_F(AttributeWriterTest, forceCommit_clears_search_cache_in_imported_attribut EXPECT_EQ(0u, _mgr->getImportedAttributes()->get("imported_b")->getSearchCache()->size()); } +TEST_F(AttributeWriterTest, ignores_force_commit_serial_not_greater_than_create_serial) +{ + auto a1 = addAttribute("a1"); + a1->setCreateSerialNum(100); + EXPECT_EQ(0u, test_force_commit(*a1, 50u)); + EXPECT_EQ(0u, test_force_commit(*a1, 100u)); + EXPECT_EQ(150u, test_force_commit(*a1, 150u)); +} + class StructWriterTestBase : public AttributeWriterTest { public: DocumentType _type; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index a49b27caf36..13695b969bf 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -212,7 +212,9 @@ void applyCommit(SerialNum serialNum, AttributeWriter::OnWriteDoneType , AttributeVector &attr) { if (attr.getStatus().getLastSyncToken() <= serialNum) { - attr.commit(serialNum, serialNum); + if (serialNum > attr.getCreateSerialNum()) { + attr.commit(serialNum, serialNum); + } } } -- cgit v1.2.3 From 6a1eb922191b8f7d2bfd966f4c3861d594b7414e Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Wed, 23 Sep 2020 15:15:25 +0000 Subject: Add option to set rpc network threads. --- .../apps/vespa-spi-feed-bm/vespa_spi_feed_bm.cpp | 25 +++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/searchcore/src/apps/vespa-spi-feed-bm/vespa_spi_feed_bm.cpp b/searchcore/src/apps/vespa-spi-feed-bm/vespa_spi_feed_bm.cpp index 1bebcee8a1d..06ad022a328 100644 --- a/searchcore/src/apps/vespa-spi-feed-bm/vespa_spi_feed_bm.cpp +++ b/searchcore/src/apps/vespa-spi-feed-bm/vespa_spi_feed_bm.cpp @@ -304,6 +304,7 @@ class BMParams { uint32_t _put_passes; uint32_t _update_passes; uint32_t _remove_passes; + uint32_t _rpc_network_threads; bool _enable_service_layer; uint32_t get_start(uint32_t thread_id) const { return (_documents / _threads) * thread_id + std::min(thread_id, _documents % _threads); @@ -315,6 +316,7 @@ public: _put_passes(2), _update_passes(1), _remove_passes(2), + _rpc_network_threads(1), _enable_service_layer(false) { } @@ -326,12 +328,14 @@ public: uint32_t get_put_passes() const { return _put_passes; } uint32_t get_update_passes() const { return _update_passes; } uint32_t get_remove_passes() const { return _remove_passes; } + uint32_t get_rpc_network_threads() const { return _rpc_network_threads; } bool get_enable_service_layer() const { return _enable_service_layer; } void set_documents(uint32_t documents_in) { _documents = documents_in; } void set_threads(uint32_t threads_in) { _threads = threads_in; } void set_put_passes(uint32_t put_passes_in) { _put_passes = put_passes_in; } void set_update_passes(uint32_t update_passes_in) { _update_passes = update_passes_in; } void set_remove_passes(uint32_t remove_passes_in) { _remove_passes = remove_passes_in; } + void set_rpc_network_threads(uint32_t threads_in) { _rpc_network_threads = threads_in; } void set_enable_service_layer(bool enable_service_layer_in) { _enable_service_layer = enable_service_layer_in; } bool check() const; }; @@ -355,6 +359,10 @@ BMParams::check() const std::cerr << "Put passes too low: " << _put_passes << std::endl; return false; } + if (_rpc_network_threads < 1) { + std::cerr << "Too few rpc network threads: " << _rpc_network_threads << std::endl; + return false; + } return true; } @@ -418,7 +426,7 @@ struct MyStorageConfig SlobroksConfigBuilder slobroks; MessagebusConfigBuilder messagebus; - MyStorageConfig(const vespalib::string& config_id_in, const DocumenttypesConfig& documenttypes_in, int slobrok_port, int status_port) + MyStorageConfig(const vespalib::string& config_id_in, const DocumenttypesConfig& documenttypes_in, int slobrok_port, int status_port, uint32_t rpc_network_threads) : config_id(config_id_in), documenttypes(documenttypes_in), persistence(), @@ -464,6 +472,7 @@ struct MyStorageConfig slobroks.slobrok.push_back(std::move(slobrok)); } stor_communicationmanager.useDirectStorageapiRpc = true; + stor_communicationmanager.rpc.numNetworkThreads = rpc_network_threads; stor_status.httpport = status_port; } @@ -623,7 +632,7 @@ struct PersistenceProviderFixture { std::unique_ptr _rpc_message_dispatcher; std::unique_ptr _rpc_client; - PersistenceProviderFixture(); + PersistenceProviderFixture(const BMParams& params); ~PersistenceProviderFixture(); void create_document_db(); uint32_t num_buckets() const { return (1u << _bucket_bits); } @@ -638,7 +647,7 @@ struct PersistenceProviderFixture { void send_rpc(std::shared_ptr cmd, MyPendingTracker& pending_tracker); }; -PersistenceProviderFixture::PersistenceProviderFixture() +PersistenceProviderFixture::PersistenceProviderFixture(const BMParams& params) : _document_types(make_document_type()), _repo(DocumentTypeRepoFactory::make(*_document_types)), _doc_type_name("test"), @@ -666,7 +675,7 @@ PersistenceProviderFixture::PersistenceProviderFixture() _persistence_engine(), _context(default_load_type, Priority(0), Trace::TraceLevel(0)), _bucket_bits(16), - _service_layer_config("bm-servicelayer", *_document_types, _slobrok_port, _status_port), + _service_layer_config("bm-servicelayer", *_document_types, _slobrok_port, _status_port, params.get_rpc_network_threads()), _rpc_client_config("bm-rpc-client", _slobrok_port), _config_set(), _config_context(std::make_shared(_config_set)), @@ -1017,7 +1026,7 @@ run_remove_async_tasks(PersistenceProviderFixture &f, vespalib::ThreadStackExecu void benchmark_async_spi(const BMParams &bm_params) { vespalib::rmdir(base_dir, true); - PersistenceProviderFixture f; + PersistenceProviderFixture f(bm_params); auto &provider = *f._persistence_engine; LOG(info, "start initialize"); provider.initialize(); @@ -1075,6 +1084,7 @@ App::usage() "[--put-passes put-passes]\n" "[--update-passes update-passes]\n" "[--remove-passes remove-passes]\n" + "[--rpc-network-threads threads]\n" "[--enable-service-layer]" << std::endl; } @@ -1090,6 +1100,7 @@ App::get_options() { "put-passes", 1, nullptr, 0 }, { "update-passes", 1, nullptr, 0 }, { "remove-passes", 1, nullptr, 0 }, + { "rpc-network-threads", 1, nullptr, 0 }, { "enable-service-layer", 0, nullptr, 0 } }; enum longopts_enum { @@ -1098,6 +1109,7 @@ App::get_options() LONGOPT_PUT_PASSES, LONGOPT_UPDATE_PASSES, LONGOPT_REMOVE_PASSES, + LONGOPT_RPC_NETWORK_THREADS, LONGOPT_ENABLE_SERVICE_LAYER }; int opt_index = 1; @@ -1121,6 +1133,9 @@ App::get_options() case LONGOPT_REMOVE_PASSES: _bm_params.set_remove_passes(atoi(opt_argument)); break; + case LONGOPT_RPC_NETWORK_THREADS: + _bm_params.set_rpc_network_threads(atoi(opt_argument)); + break; case LONGOPT_ENABLE_SERVICE_LAYER: _bm_params.set_enable_service_layer(true); break; -- cgit v1.2.3 From b632540821ed7c3657b93f926aa8719f1d7fc9e1 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Wed, 23 Sep 2020 15:22:24 +0000 Subject: refactor without TypeMeta --- eval/src/vespa/eval/eval/value_codec.cpp | 89 +++++++++++--------------------- eval/src/vespa/eval/eval/value_type.cpp | 23 +++++++++ eval/src/vespa/eval/eval/value_type.h | 2 + 3 files changed, 56 insertions(+), 58 deletions(-) diff --git a/eval/src/vespa/eval/eval/value_codec.cpp b/eval/src/vespa/eval/eval/value_codec.cpp index b5a1ccb92fa..88f7d2c9829 100644 --- a/eval/src/vespa/eval/eval/value_codec.cpp +++ b/eval/src/vespa/eval/eval/value_codec.cpp @@ -31,70 +31,45 @@ inline CellType id_to_cell_type(uint32_t id) { abort(); } -/** - * Meta information about how a type can be decomposed into mapped and - * indexed dimensions and also how large each block is. A block is a - * dense-subspace consisting of all indexed dimensions that is - * uniquely specified by the labels of all mapped dimensions. - **/ -struct TypeMeta { - IndexList mapped; - IndexList indexed; - CellType cell_type; - explicit TypeMeta(const ValueType &type) - : mapped(), - indexed(), - cell_type(type.cell_type()) - { - for (size_t i = 0; i < type.dimensions().size(); ++i) { - const auto &dimension = type.dimensions()[i]; - if (dimension.is_mapped()) { - mapped.push_back(i); - } else { - indexed.push_back(i); - } - } - } - ~TypeMeta() {} -}; - struct Format { - bool is_sparse; - bool is_dense; + bool has_sparse; + bool has_dense; bool with_cell_type; uint32_t tag; - explicit Format(const TypeMeta &meta) - : is_sparse(meta.mapped.size() > 0), - is_dense((meta.indexed.size() > 0) || !is_sparse), - with_cell_type(meta.cell_type != CellType::DOUBLE), - tag((is_sparse ? 0x1 : 0) | (is_dense ? 0x2 : 0) | (with_cell_type ? 0x4 : 0)) {} + explicit Format(const ValueType &type) + : has_sparse(type.count_mapped_dimensions() > 0), + has_dense((type.count_indexed_dimensions() > 0) || !has_sparse), + with_cell_type(type.cell_type() != CellType::DOUBLE), + tag((has_sparse ? 0x1 : 0) | (has_dense ? 0x2 : 0) | (with_cell_type ? 0x4 : 0)) {} explicit Format(uint32_t tag_in) - : is_sparse((tag_in & 0x1) != 0), - is_dense((tag_in & 0x2) != 0), + : has_sparse((tag_in & 0x1) != 0), + has_dense((tag_in & 0x2) != 0), with_cell_type((tag_in & 0x4) != 0), tag(tag_in) {} ~Format() {} }; -void maybe_encode_cell_type(nbostream &output, const Format &format, const TypeMeta &meta) { +void maybe_encode_cell_type(nbostream &output, const Format &format, CellType cell_type) { if (format.with_cell_type) { - output.putInt1_4Bytes(cell_type_to_id(meta.cell_type)); + output.putInt1_4Bytes(cell_type_to_id(cell_type)); } } -void encode_type(nbostream &output, const Format &format, const ValueType &type, const TypeMeta &meta) { - maybe_encode_cell_type(output, format, meta); - if (format.is_sparse) { - output.putInt1_4Bytes(type.count_mapped_dimensions()); - for (size_t idx: meta.mapped) { - output.writeSmallString(type.dimensions()[idx].name); +void encode_type(nbostream &output, const Format &format, const ValueType &type) { + maybe_encode_cell_type(output, format, type.cell_type()); + if (format.has_sparse) { + const auto & dims = type.mapped_dimensions(); + output.putInt1_4Bytes(dims.size()); + for (const auto & dim : dims) { + output.writeSmallString(dim.name); } } - if (format.is_dense) { - output.putInt1_4Bytes(meta.indexed.size()); - for (size_t idx: meta.indexed) { - output.writeSmallString(type.dimensions()[idx].name); - output.putInt1_4Bytes(type.dimensions()[idx].size); + if (format.has_dense) { + const auto & dims = type.indexed_dimensions(); + output.putInt1_4Bytes(dims.size()); + for (const auto & dim : dims) { + output.writeSmallString(dim.name); + output.putInt1_4Bytes(dim.size); } } } @@ -115,7 +90,7 @@ CellType maybe_decode_cell_type(nbostream &input, const Format &format) { ValueType decode_type(nbostream &input, const Format &format) { CellType cell_type = maybe_decode_cell_type(input, format); std::vector dim_list; - if (format.is_sparse) { + if (format.has_sparse) { size_t cnt = input.getInt1_4Bytes(); for (size_t i = 0; i < cnt; ++i) { vespalib::string name; @@ -123,7 +98,7 @@ ValueType decode_type(nbostream &input, const Format &format) { dim_list.emplace_back(name); } } - if (format.is_dense) { + if (format.has_dense) { size_t cnt = input.getInt1_4Bytes(); for (size_t i = 0; i < cnt; ++i) { vespalib::string name; @@ -135,7 +110,7 @@ ValueType decode_type(nbostream &input, const Format &format) { } size_t maybe_decode_num_blocks(nbostream &input, bool has_mapped_dims, const Format &format) { - if (has_mapped_dims || !format.is_dense) { + if (has_mapped_dims || !format.has_dense) { return input.getInt1_4Bytes(); } return 1; @@ -264,12 +239,11 @@ struct CreateTensorSpecFromValue { } // namespace void new_encode(const Value &value, nbostream &output) { - TypeMeta meta(value.type()); size_t num_mapped_dims = value.type().count_mapped_dimensions(); size_t dense_subspace_size = value.type().dense_subspace_size(); - Format format(meta); + Format format(value.type()); output.putInt1_4Bytes(format.tag); - encode_type(output, format, value.type(), meta); + encode_type(output, format, value.type()); maybe_encode_num_blocks(output, (num_mapped_dims > 0), value.cells().size / dense_subspace_size); std::vector address(num_mapped_dims); std::vector a_refs(num_mapped_dims); @@ -281,7 +255,7 @@ void new_encode(const Value &value, nbostream &output) { size_t subspace; while (view->next_result(a_refs, subspace)) { encode_mapped_labels(output, num_mapped_dims, address); - if (meta.cell_type == CellType::FLOAT) { + if (value.type().cell_type() == CellType::FLOAT) { auto iter = value.cells().typify().begin(); iter += (subspace * dense_subspace_size); for (size_t i = 0; i < dense_subspace_size; ++i) { @@ -302,10 +276,9 @@ std::unique_ptr new_decode(nbostream &input, const ValueBuilderFactory &f ValueType type = decode_type(input, format); size_t num_mapped_dims = type.count_mapped_dimensions(); size_t dense_subspace_size = type.dense_subspace_size(); - TypeMeta meta(type); const size_t num_blocks = maybe_decode_num_blocks(input, (num_mapped_dims > 0), format); DecodeState state{type, dense_subspace_size, num_blocks, num_mapped_dims}; - return typify_invoke<1,TypifyCellType,ContentDecoder>(meta.cell_type, input, state, factory); + return typify_invoke<1,TypifyCellType,ContentDecoder>(type.cell_type(), input, state, factory); } //----------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/eval/value_type.cpp b/eval/src/vespa/eval/eval/value_type.cpp index 30a36fcba1d..3685d2715d8 100644 --- a/eval/src/vespa/eval/eval/value_type.cpp +++ b/eval/src/vespa/eval/eval/value_type.cpp @@ -173,6 +173,18 @@ ValueType::is_dense() const return true; } +size_t +ValueType::count_indexed_dimensions() const +{ + size_t cnt = 0; + for (const auto &dim : dimensions()) { + if (dim.is_indexed()) { + ++cnt; + } + } + return cnt; +} + size_t ValueType::count_mapped_dimensions() const { @@ -208,6 +220,17 @@ ValueType::nontrivial_indexed_dimensions() const { return result; } +std::vector +ValueType::indexed_dimensions() const { + std::vector result; + for (const auto &dim: dimensions()) { + if (dim.is_indexed()) { + result.push_back(dim); + } + } + return result; +} + std::vector ValueType::mapped_dimensions() const { std::vector result; diff --git a/eval/src/vespa/eval/eval/value_type.h b/eval/src/vespa/eval/eval/value_type.h index 4199b3a3381..5fac4d5d27d 100644 --- a/eval/src/vespa/eval/eval/value_type.h +++ b/eval/src/vespa/eval/eval/value_type.h @@ -60,10 +60,12 @@ public: bool is_tensor() const { return (_type == Type::TENSOR); } bool is_sparse() const; bool is_dense() const; + size_t count_indexed_dimensions() const; size_t count_mapped_dimensions() const; size_t dense_subspace_size() const; const std::vector &dimensions() const { return _dimensions; } std::vector nontrivial_indexed_dimensions() const; + std::vector indexed_dimensions() const; std::vector mapped_dimensions() const; size_t dimension_index(const vespalib::string &name) const; std::vector dimension_names() const; -- cgit v1.2.3 From 79a2099622bb2ba7f4f03edb6b6314ff5eea8bf7 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Wed, 23 Sep 2020 17:35:30 +0200 Subject: Revert "Balder/group operations to tls and commit in batches" --- .../vespa/searchcore/proton/server/feedhandler.cpp | 32 +---------- .../vespa/searchcore/proton/server/feedhandler.h | 5 -- .../searchcore/proton/server/i_operation_storer.h | 4 -- .../proton/server/lid_space_compaction_job.cpp | 2 +- .../proton/server/lid_space_compaction_job.h | 6 +- .../src/vespa/searchlib/config/translogserver.def | 2 +- .../src/vespa/searchlib/transactionlog/domain.cpp | 66 ++-------------------- .../src/vespa/searchlib/transactionlog/domain.h | 5 -- .../vespa/searchlib/transactionlog/domainpart.cpp | 12 +--- .../searchlib/transactionlog/translogserver.cpp | 7 +-- 10 files changed, 16 insertions(+), 125 deletions(-) diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 209a35ce4a2..37dfddf0c2c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -402,8 +402,6 @@ FeedHandler::FeedHandler(IThreadingService &writeService, _tlsReplayProgress(), _serialNum(0), _prunedSerialNum(0), - _numPendingCommit(0), - _numCommitsCompleted(0), _delayedPrune(false), _feedLock(), _feedState(make_shared(getDocTypeName())), @@ -496,40 +494,12 @@ FeedHandler::getTransactionLogReplayDone() const { return _tlsMgr.getReplayDone(); } -void -FeedHandler::onCommitDone(uint64_t numPendingAtStart) { - assert(numPendingAtStart <= _numPendingCommit); - _numPendingCommit -= numPendingAtStart; - if (_numPendingCommit > 0) { - enqueCommitTask(); - } -} - -void FeedHandler::enqueCommitTask() { - _writeService.master().execute(makeLambdaTask([this]() { initiateCommit(); })); -} - -void -FeedHandler::initiateCommit() { - auto commitResult = _tlsWriter->startCommit(std::make_shared( - _writeService.master(), - makeLambdaTask([this, numPendingAtStart=_numPendingCommit]() { - onCommitDone(numPendingAtStart); - }))); - if (_activeFeedView) { - _activeFeedView->forceCommit(_serialNum, std::make_shared>(std::move(commitResult))); - } -} - void FeedHandler::appendOperation(const FeedOperation &op, TlsWriter::DoneCallback onDone) { if (!op.getSerialNum()) { const_cast(op).setSerialNum(incSerialNum()); } _tlsWriter->appendOperation(op, std::move(onDone)); - if (++_numPendingCommit == 1) { - enqueCommitTask(); - } } FeedHandler::CommitResult @@ -540,7 +510,7 @@ FeedHandler::startCommit(DoneCallback onDone) { void FeedHandler::storeOperationSync(const FeedOperation &op) { vespalib::Gate gate; - appendAndCommitOperation(op, make_shared(gate)); + appendOperation(op, make_shared(gate)); gate.await(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index 97629bfc018..4807c596130 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -76,8 +76,6 @@ private: // the serial num of the last message in the transaction log SerialNum _serialNum; SerialNum _prunedSerialNum; - size_t _numPendingCommit; - size_t _numCommitsCompleted; bool _delayedPrune; mutable std::shared_mutex _feedLock; FeedStateSP _feedState; @@ -127,9 +125,6 @@ private: FeedStateSP getFeedState() const; void changeFeedState(FeedStateSP newState); void doChangeFeedState(FeedStateSP newState); - void onCommitDone(uint64_t numPendingAtStart); - void initiateCommit(); - void enqueCommitTask(); public: FeedHandler(const FeedHandler &) = delete; FeedHandler & operator = (const FeedHandler &) = delete; diff --git a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h index c3b76a9db75..b276779c2ee 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h @@ -22,10 +22,6 @@ struct IOperationStorer */ virtual void appendOperation(const FeedOperation &op, DoneCallback onDone) = 0; [[nodiscard]] virtual CommitResult startCommit(DoneCallback onDone) = 0; - void appendAndCommitOperation(const FeedOperation &op, DoneCallback onDone) { - appendOperation(op, onDone); - (void) startCommit(std::move(onDone)); - } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp index 468850b4409..d423e095ad9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp @@ -82,7 +82,7 @@ LidSpaceCompactionJob::compactLidSpace(const LidUsageStats &stats) uint32_t wantedLidLimit = stats.getHighestUsedLid() + 1; CompactLidSpaceOperation op(_handler.getSubDbId(), wantedLidLimit); vespalib::Gate gate; - _opStorer.appendAndCommitOperation(op, std::make_shared(gate)); + _opStorer.appendOperation(op, std::make_shared(gate)); gate.await(); _handler.handleCompactLidSpace(op); EventLogger::lidSpaceCompactionComplete(_handler.getName(), wantedLidLimit); diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h index 35549f21471..37497eaa998 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h @@ -63,13 +63,13 @@ public: ~LidSpaceCompactionJob(); // Implements IDiskMemUsageListener - void notifyDiskMemUsage(DiskMemUsageState state) override; + virtual void notifyDiskMemUsage(DiskMemUsageState state) override; // Implements IClusterStateChangedNofifier - void notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) override; + virtual void notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) override; // Implements IMaintenanceJob - bool run() override; + virtual bool run() override; }; } // namespace proton diff --git a/searchlib/src/vespa/searchlib/config/translogserver.def b/searchlib/src/vespa/searchlib/config/translogserver.def index defce8c3421..540895b2404 100644 --- a/searchlib/src/vespa/searchlib/config/translogserver.def +++ b/searchlib/src/vespa/searchlib/config/translogserver.def @@ -15,7 +15,7 @@ basedir string default="tmp" restart ## Use fsync after each commit. ## If not the below interval is used. -usefsync bool default=false +usefsync bool default=false restart ##Number of threads available for visiting/subscription. maxthreads int default=4 restart diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index bd7feec0598..9e0f1a8a1aa 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -113,12 +113,7 @@ Domain::addPart(SerialNum partId, bool isLastPart) { } } -Domain::~Domain() { - MonitorGuard guard(_currentChunkMonitor); - guard.broadcast(); - commitChunk(grabCurrentChunk(guard), guard); - _singleCommitter->shutdown().sync(); -} +Domain::~Domain() { } DomainInfo Domain::getDomainInfo() const @@ -323,73 +318,22 @@ Domain::optionallyRotateFile(SerialNum serialNum) { return dp; } -void -Domain::append(const Packet & packet, Writer::DoneCallback onDone) { - vespalib::MonitorGuard guard(_currentChunkMonitor); - if (_lastSerial >= packet.range().from()) { - throw runtime_error(fmt("Incoming serial number(%" PRIu64 ") must be bigger than the last one (%" PRIu64 ").", - packet.range().from(), _lastSerial)); - } else { - _lastSerial = packet.range().to(); - } - _currentChunk->add(packet, std::move(onDone)); - commitIfFull(guard); -} - Domain::CommitResult Domain::startCommit(DoneCallback onDone) { - vespalib::MonitorGuard guard(_currentChunkMonitor); - if ( !_currentChunk->empty() ) { - auto completed = grabCurrentChunk(guard); - completed->setCommitDoneCallback(std::move(onDone)); - CommitResult result(completed->createCommitResult()); - commitChunk(std::move(completed), guard); - return result; - } + (void) onDone; return CommitResult(); } void -Domain::commitIfFull(const vespalib::MonitorGuard &guard) { - if (_currentChunk->sizeBytes() > _config.getChunkSizeLimit()) { - auto completed = std::move(_currentChunk); - _currentChunk = std::make_unique(_config.getChunkSizeLimit(), completed->stealCallbacks()); - commitChunk(std::move(completed), guard); - } -} - -std::unique_ptr -Domain::grabCurrentChunk(const vespalib::MonitorGuard & guard) { - assert(guard.monitors(_currentChunkMonitor)); - auto chunk = std::move(_currentChunk); - _currentChunk = createCommitChunk(_config); - return chunk; -} - -void -Domain::commitChunk(std::unique_ptr chunk, const vespalib::MonitorGuard & chunkOrderGuard) { - assert(chunkOrderGuard.monitors(_currentChunkMonitor)); - _singleCommitter->execute( makeLambdaTask([this, chunk = std::move(chunk)]() mutable { - doCommit(std::move(chunk)); - })); -} - -void -Domain::doCommit(std::unique_ptr chunk) { - const Packet & packet = chunk->getPacket(); - if (packet.empty()) return; - +Domain::append(const Packet & packet, Writer::DoneCallback onDone) +{ + (void) onDone; vespalib::nbostream_longlivedbuf is(packet.getHandle().data(), packet.getHandle().size()); Packet::Entry entry; entry.deserialize(is); DomainPart::SP dp = optionallyRotateFile(entry.serial()); dp->commit(entry.serial(), packet); - if (_config.getFSyncOnCommit()) { - dp->sync(); - } cleanSessions(); - LOG(debug, "Releasing %zu acks and %zu entries and %zu bytes.", - chunk->getNumCallBacks(), chunk->getPacket().size(), chunk->sizeBytes()); } bool diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h index 041ec27cf23..7e77e6ef0ef 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h @@ -56,11 +56,6 @@ public: uint64_t size() const; Domain & setConfig(const DomainConfig & cfg); private: - void commitIfFull(const vespalib::MonitorGuard & guard); - - std::unique_ptr grabCurrentChunk(const vespalib::MonitorGuard & guard); - void commitChunk(std::unique_ptr chunk, const vespalib::MonitorGuard & chunkOrderGuard); - void doCommit(std::unique_ptr chunk); SerialNum begin(const vespalib::LockGuard & guard) const; SerialNum end(const vespalib::LockGuard & guard) const; size_t byteSize(const vespalib::LockGuard & guard) const; diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp index b7e02894e6b..8855183226d 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp @@ -252,7 +252,7 @@ DomainPart::buildPacketMapping(bool allowTruncate) DomainPart::DomainPart(const string & name, const string & baseDir, SerialNum s, Encoding encoding, uint8_t compressionLevel, const FileHeaderContext &fileHeaderContext, bool allowTruncate) - : _encoding(encoding), + : _encoding(encoding.getCrc(), Encoding::Compression::none), //TODO We do not yet support compression _compressionLevel(compressionLevel), _lock(), _fileLock(), @@ -396,19 +396,16 @@ DomainPart::commit(SerialNum firstSerial, const Packet &packet) if (_range.from() == 0) { _range.from(firstSerial); } - IChunk::UP chunk = IChunk::create(_encoding, _compressionLevel); for (size_t i(0); h.size() > 0; i++) { //LOG(spam, //"Pos(%d) Len(%d), Lim(%d), Remaining(%d)", //h.getPos(), h.getLength(), h.getLimit(), h.getRemaining()); + IChunk::UP chunk = IChunk::create(_encoding, _compressionLevel); Packet::Entry entry; entry.deserialize(h); if (_range.to() < entry.serial()) { chunk->add(entry); - if (_encoding.getCompression() == Encoding::Compression::none) { - write(*_transLog, *chunk); - chunk = IChunk::create(_encoding, _compressionLevel); - } + write(*_transLog, *chunk); _sz++; _range.to(entry.serial()); } else { @@ -416,9 +413,6 @@ DomainPart::commit(SerialNum firstSerial, const Packet &packet) entry.serial(), _range.to())); } } - if ( ! chunk->getEntries().empty()) { - write(*_transLog, *chunk); - } bool merged(false); LockGuard guard(_lock); diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index 0c0c9186e12..7be3dd708a5 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -572,11 +572,8 @@ TransLogServer::domainCommit(FRT_RPCRequest *req) Packet packet(params[1]._data._buf, params[1]._data._len); try { vespalib::Gate gate; - { - // Need to scope in order to drain out all the callbacks. - domain->append(packet, make_shared(gate)); - auto keep = domain->startCommit(make_shared()); - } + domain->append(packet, make_shared(gate)); + auto keep = domain->startCommit(make_shared()); gate.await(); ret.AddInt32(0); ret.AddString("ok"); -- cgit v1.2.3 From 57b71a601f169d1444038227189f9dd8608ccb63 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Wed, 23 Sep 2020 17:37:46 +0200 Subject: Revert "Revert "Balder/group operations to tls and commit in batches"" --- .../vespa/searchcore/proton/server/feedhandler.cpp | 32 ++++++++++- .../vespa/searchcore/proton/server/feedhandler.h | 5 ++ .../searchcore/proton/server/i_operation_storer.h | 4 ++ .../proton/server/lid_space_compaction_job.cpp | 2 +- .../proton/server/lid_space_compaction_job.h | 6 +- .../src/vespa/searchlib/config/translogserver.def | 2 +- .../src/vespa/searchlib/transactionlog/domain.cpp | 66 ++++++++++++++++++++-- .../src/vespa/searchlib/transactionlog/domain.h | 5 ++ .../vespa/searchlib/transactionlog/domainpart.cpp | 12 +++- .../searchlib/transactionlog/translogserver.cpp | 7 ++- 10 files changed, 125 insertions(+), 16 deletions(-) diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 37dfddf0c2c..209a35ce4a2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -402,6 +402,8 @@ FeedHandler::FeedHandler(IThreadingService &writeService, _tlsReplayProgress(), _serialNum(0), _prunedSerialNum(0), + _numPendingCommit(0), + _numCommitsCompleted(0), _delayedPrune(false), _feedLock(), _feedState(make_shared(getDocTypeName())), @@ -494,12 +496,40 @@ FeedHandler::getTransactionLogReplayDone() const { return _tlsMgr.getReplayDone(); } +void +FeedHandler::onCommitDone(uint64_t numPendingAtStart) { + assert(numPendingAtStart <= _numPendingCommit); + _numPendingCommit -= numPendingAtStart; + if (_numPendingCommit > 0) { + enqueCommitTask(); + } +} + +void FeedHandler::enqueCommitTask() { + _writeService.master().execute(makeLambdaTask([this]() { initiateCommit(); })); +} + +void +FeedHandler::initiateCommit() { + auto commitResult = _tlsWriter->startCommit(std::make_shared( + _writeService.master(), + makeLambdaTask([this, numPendingAtStart=_numPendingCommit]() { + onCommitDone(numPendingAtStart); + }))); + if (_activeFeedView) { + _activeFeedView->forceCommit(_serialNum, std::make_shared>(std::move(commitResult))); + } +} + void FeedHandler::appendOperation(const FeedOperation &op, TlsWriter::DoneCallback onDone) { if (!op.getSerialNum()) { const_cast(op).setSerialNum(incSerialNum()); } _tlsWriter->appendOperation(op, std::move(onDone)); + if (++_numPendingCommit == 1) { + enqueCommitTask(); + } } FeedHandler::CommitResult @@ -510,7 +540,7 @@ FeedHandler::startCommit(DoneCallback onDone) { void FeedHandler::storeOperationSync(const FeedOperation &op) { vespalib::Gate gate; - appendOperation(op, make_shared(gate)); + appendAndCommitOperation(op, make_shared(gate)); gate.await(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index 4807c596130..97629bfc018 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -76,6 +76,8 @@ private: // the serial num of the last message in the transaction log SerialNum _serialNum; SerialNum _prunedSerialNum; + size_t _numPendingCommit; + size_t _numCommitsCompleted; bool _delayedPrune; mutable std::shared_mutex _feedLock; FeedStateSP _feedState; @@ -125,6 +127,9 @@ private: FeedStateSP getFeedState() const; void changeFeedState(FeedStateSP newState); void doChangeFeedState(FeedStateSP newState); + void onCommitDone(uint64_t numPendingAtStart); + void initiateCommit(); + void enqueCommitTask(); public: FeedHandler(const FeedHandler &) = delete; FeedHandler & operator = (const FeedHandler &) = delete; diff --git a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h index b276779c2ee..c3b76a9db75 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h @@ -22,6 +22,10 @@ struct IOperationStorer */ virtual void appendOperation(const FeedOperation &op, DoneCallback onDone) = 0; [[nodiscard]] virtual CommitResult startCommit(DoneCallback onDone) = 0; + void appendAndCommitOperation(const FeedOperation &op, DoneCallback onDone) { + appendOperation(op, onDone); + (void) startCommit(std::move(onDone)); + } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp index d423e095ad9..468850b4409 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp @@ -82,7 +82,7 @@ LidSpaceCompactionJob::compactLidSpace(const LidUsageStats &stats) uint32_t wantedLidLimit = stats.getHighestUsedLid() + 1; CompactLidSpaceOperation op(_handler.getSubDbId(), wantedLidLimit); vespalib::Gate gate; - _opStorer.appendOperation(op, std::make_shared(gate)); + _opStorer.appendAndCommitOperation(op, std::make_shared(gate)); gate.await(); _handler.handleCompactLidSpace(op); EventLogger::lidSpaceCompactionComplete(_handler.getName(), wantedLidLimit); diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h index 37497eaa998..35549f21471 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h @@ -63,13 +63,13 @@ public: ~LidSpaceCompactionJob(); // Implements IDiskMemUsageListener - virtual void notifyDiskMemUsage(DiskMemUsageState state) override; + void notifyDiskMemUsage(DiskMemUsageState state) override; // Implements IClusterStateChangedNofifier - virtual void notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) override; + void notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) override; // Implements IMaintenanceJob - virtual bool run() override; + bool run() override; }; } // namespace proton diff --git a/searchlib/src/vespa/searchlib/config/translogserver.def b/searchlib/src/vespa/searchlib/config/translogserver.def index 540895b2404..defce8c3421 100644 --- a/searchlib/src/vespa/searchlib/config/translogserver.def +++ b/searchlib/src/vespa/searchlib/config/translogserver.def @@ -15,7 +15,7 @@ basedir string default="tmp" restart ## Use fsync after each commit. ## If not the below interval is used. -usefsync bool default=false restart +usefsync bool default=false ##Number of threads available for visiting/subscription. maxthreads int default=4 restart diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index 9e0f1a8a1aa..bd7feec0598 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -113,7 +113,12 @@ Domain::addPart(SerialNum partId, bool isLastPart) { } } -Domain::~Domain() { } +Domain::~Domain() { + MonitorGuard guard(_currentChunkMonitor); + guard.broadcast(); + commitChunk(grabCurrentChunk(guard), guard); + _singleCommitter->shutdown().sync(); +} DomainInfo Domain::getDomainInfo() const @@ -318,22 +323,73 @@ Domain::optionallyRotateFile(SerialNum serialNum) { return dp; } +void +Domain::append(const Packet & packet, Writer::DoneCallback onDone) { + vespalib::MonitorGuard guard(_currentChunkMonitor); + if (_lastSerial >= packet.range().from()) { + throw runtime_error(fmt("Incoming serial number(%" PRIu64 ") must be bigger than the last one (%" PRIu64 ").", + packet.range().from(), _lastSerial)); + } else { + _lastSerial = packet.range().to(); + } + _currentChunk->add(packet, std::move(onDone)); + commitIfFull(guard); +} + Domain::CommitResult Domain::startCommit(DoneCallback onDone) { - (void) onDone; + vespalib::MonitorGuard guard(_currentChunkMonitor); + if ( !_currentChunk->empty() ) { + auto completed = grabCurrentChunk(guard); + completed->setCommitDoneCallback(std::move(onDone)); + CommitResult result(completed->createCommitResult()); + commitChunk(std::move(completed), guard); + return result; + } return CommitResult(); } void -Domain::append(const Packet & packet, Writer::DoneCallback onDone) -{ - (void) onDone; +Domain::commitIfFull(const vespalib::MonitorGuard &guard) { + if (_currentChunk->sizeBytes() > _config.getChunkSizeLimit()) { + auto completed = std::move(_currentChunk); + _currentChunk = std::make_unique(_config.getChunkSizeLimit(), completed->stealCallbacks()); + commitChunk(std::move(completed), guard); + } +} + +std::unique_ptr +Domain::grabCurrentChunk(const vespalib::MonitorGuard & guard) { + assert(guard.monitors(_currentChunkMonitor)); + auto chunk = std::move(_currentChunk); + _currentChunk = createCommitChunk(_config); + return chunk; +} + +void +Domain::commitChunk(std::unique_ptr chunk, const vespalib::MonitorGuard & chunkOrderGuard) { + assert(chunkOrderGuard.monitors(_currentChunkMonitor)); + _singleCommitter->execute( makeLambdaTask([this, chunk = std::move(chunk)]() mutable { + doCommit(std::move(chunk)); + })); +} + +void +Domain::doCommit(std::unique_ptr chunk) { + const Packet & packet = chunk->getPacket(); + if (packet.empty()) return; + vespalib::nbostream_longlivedbuf is(packet.getHandle().data(), packet.getHandle().size()); Packet::Entry entry; entry.deserialize(is); DomainPart::SP dp = optionallyRotateFile(entry.serial()); dp->commit(entry.serial(), packet); + if (_config.getFSyncOnCommit()) { + dp->sync(); + } cleanSessions(); + LOG(debug, "Releasing %zu acks and %zu entries and %zu bytes.", + chunk->getNumCallBacks(), chunk->getPacket().size(), chunk->sizeBytes()); } bool diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h index 7e77e6ef0ef..041ec27cf23 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h @@ -56,6 +56,11 @@ public: uint64_t size() const; Domain & setConfig(const DomainConfig & cfg); private: + void commitIfFull(const vespalib::MonitorGuard & guard); + + std::unique_ptr grabCurrentChunk(const vespalib::MonitorGuard & guard); + void commitChunk(std::unique_ptr chunk, const vespalib::MonitorGuard & chunkOrderGuard); + void doCommit(std::unique_ptr chunk); SerialNum begin(const vespalib::LockGuard & guard) const; SerialNum end(const vespalib::LockGuard & guard) const; size_t byteSize(const vespalib::LockGuard & guard) const; diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp index 8855183226d..b7e02894e6b 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp @@ -252,7 +252,7 @@ DomainPart::buildPacketMapping(bool allowTruncate) DomainPart::DomainPart(const string & name, const string & baseDir, SerialNum s, Encoding encoding, uint8_t compressionLevel, const FileHeaderContext &fileHeaderContext, bool allowTruncate) - : _encoding(encoding.getCrc(), Encoding::Compression::none), //TODO We do not yet support compression + : _encoding(encoding), _compressionLevel(compressionLevel), _lock(), _fileLock(), @@ -396,16 +396,19 @@ DomainPart::commit(SerialNum firstSerial, const Packet &packet) if (_range.from() == 0) { _range.from(firstSerial); } + IChunk::UP chunk = IChunk::create(_encoding, _compressionLevel); for (size_t i(0); h.size() > 0; i++) { //LOG(spam, //"Pos(%d) Len(%d), Lim(%d), Remaining(%d)", //h.getPos(), h.getLength(), h.getLimit(), h.getRemaining()); - IChunk::UP chunk = IChunk::create(_encoding, _compressionLevel); Packet::Entry entry; entry.deserialize(h); if (_range.to() < entry.serial()) { chunk->add(entry); - write(*_transLog, *chunk); + if (_encoding.getCompression() == Encoding::Compression::none) { + write(*_transLog, *chunk); + chunk = IChunk::create(_encoding, _compressionLevel); + } _sz++; _range.to(entry.serial()); } else { @@ -413,6 +416,9 @@ DomainPart::commit(SerialNum firstSerial, const Packet &packet) entry.serial(), _range.to())); } } + if ( ! chunk->getEntries().empty()) { + write(*_transLog, *chunk); + } bool merged(false); LockGuard guard(_lock); diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index 7be3dd708a5..0c0c9186e12 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -572,8 +572,11 @@ TransLogServer::domainCommit(FRT_RPCRequest *req) Packet packet(params[1]._data._buf, params[1]._data._len); try { vespalib::Gate gate; - domain->append(packet, make_shared(gate)); - auto keep = domain->startCommit(make_shared()); + { + // Need to scope in order to drain out all the callbacks. + domain->append(packet, make_shared(gate)); + auto keep = domain->startCommit(make_shared()); + } gate.await(); ret.AddInt32(0); ret.AddString("ok"); -- cgit v1.2.3 From 959ae0f84ec4b62c6ac170451924a0a3bd4eaaed Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Wed, 23 Sep 2020 16:19:14 +0000 Subject: use typify_invoke for new_encode also --- eval/src/vespa/eval/eval/value_codec.cpp | 53 +++++++++++++++++--------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/eval/src/vespa/eval/eval/value_codec.cpp b/eval/src/vespa/eval/eval/value_codec.cpp index 88f7d2c9829..ddca54995d0 100644 --- a/eval/src/vespa/eval/eval/value_codec.cpp +++ b/eval/src/vespa/eval/eval/value_codec.cpp @@ -238,6 +238,33 @@ struct CreateTensorSpecFromValue { } // namespace +struct EncodeState { + size_t num_mapped_dims; + size_t subspace_size; +}; + +struct ContentEncoder { + template + static void invoke(const Value &value, const EncodeState &state, nbostream &output) { + std::vector address(state.num_mapped_dims); + std::vector a_refs(state.num_mapped_dims);; + for (size_t i = 0; i < state.num_mapped_dims; ++i) { + a_refs[i] = &address[i]; + } + auto view = value.index().create_view({}); + view->lookup({}); + size_t subspace; + while (view->next_result(a_refs, subspace)) { + encode_mapped_labels(output, state.num_mapped_dims, address); + auto iter = value.cells().typify().begin(); + iter += (subspace * state.subspace_size); + for (size_t i = 0; i < state.subspace_size; ++i) { + output << *iter++; + } + } + } +}; + void new_encode(const Value &value, nbostream &output) { size_t num_mapped_dims = value.type().count_mapped_dimensions(); size_t dense_subspace_size = value.type().dense_subspace_size(); @@ -245,30 +272,8 @@ void new_encode(const Value &value, nbostream &output) { output.putInt1_4Bytes(format.tag); encode_type(output, format, value.type()); maybe_encode_num_blocks(output, (num_mapped_dims > 0), value.cells().size / dense_subspace_size); - std::vector address(num_mapped_dims); - std::vector a_refs(num_mapped_dims); - for (size_t i = 0; i < num_mapped_dims; ++i) { - a_refs[i] = &address[i]; - } - auto view = value.index().create_view({}); - view->lookup({}); - size_t subspace; - while (view->next_result(a_refs, subspace)) { - encode_mapped_labels(output, num_mapped_dims, address); - if (value.type().cell_type() == CellType::FLOAT) { - auto iter = value.cells().typify().begin(); - iter += (subspace * dense_subspace_size); - for (size_t i = 0; i < dense_subspace_size; ++i) { - output << (float) *iter++; - } - } else { - auto iter = value.cells().typify().begin(); - iter += (subspace * dense_subspace_size); - for (size_t i = 0; i < dense_subspace_size; ++i) { - output << *iter++; - } - } - } + EncodeState state{num_mapped_dims, dense_subspace_size}; + typify_invoke<1,TypifyCellType,ContentEncoder>(value.type().cell_type(), value, state, output); } std::unique_ptr new_decode(nbostream &input, const ValueBuilderFactory &factory) { -- cgit v1.2.3 From 447ef46eff9c2247e6db8d559f2dc62d3d0d499e Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Wed, 23 Sep 2020 16:22:30 +0000 Subject: cosmetic test fixes --- .../tests/eval/value_codec/value_codec_test.cpp | 28 ++++++++++------------ 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/eval/src/tests/eval/value_codec/value_codec_test.cpp b/eval/src/tests/eval/value_codec/value_codec_test.cpp index 43bbebfc094..12e1af0d7d5 100644 --- a/eval/src/tests/eval/value_codec/value_codec_test.cpp +++ b/eval/src/tests/eval/value_codec/value_codec_test.cpp @@ -12,7 +12,15 @@ using namespace vespalib; using namespace vespalib::eval; using namespace vespalib::eval::test; -using vespalib::make_string_short::fmt; +struct Factory { + SimpleValueBuilderFactory simple; + std::unique_ptr from_spec(const TensorSpec &spec) { + return value_from_spec(spec, simple); + } + std::unique_ptr decode(nbostream &input) { + return new_decode(input, simple); + } +} simple_factory; std::vector layouts = { {}, @@ -29,15 +37,6 @@ std::vector layouts = { float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) }; -struct Factory { - SimpleValueBuilderFactory simple; - std::unique_ptr from_spec(const TensorSpec &spec) { - return value_from_spec(spec, simple); - } - std::unique_ptr decode(nbostream &input) { - return new_decode(input, simple); - } -} simple_factory; TEST(ValueCodecTest, simple_values_can_be_converted_from_and_to_tensor_spec) { for (const auto &layout: layouts) { @@ -48,8 +47,7 @@ TEST(ValueCodecTest, simple_values_can_be_converted_from_and_to_tensor_spec) { } } - -TEST(ValueCodecTest, require_that_simple_tensors_can_be_built_using_tensor_spec) { +TEST(ValueCodecTest, simple_values_can_be_built_using_tensor_spec) { TensorSpec spec("tensor(w{},x[2],y{},z[2])"); spec.add({{"w", "xxx"}, {"x", 0}, {"y", "xxx"}, {"z", 0}}, 1.0) .add({{"w", "xxx"}, {"x", 0}, {"y", "yyy"}, {"z", 1}}, 2.0) @@ -163,7 +161,7 @@ struct SparseTensorExample : TensorExample { } }; -TEST(ValueCodecTest, require_that_sparse_tensors_can_be_encoded_and_decoded) { +TEST(ValueCodecTest, sparse_tensors_can_be_encoded_and_decoded) { SparseTensorExample f1; f1.verify_encode_decode(); } @@ -213,7 +211,7 @@ struct DenseTensorExample : TensorExample { } }; -TEST(ValueCodecTest, require_that_dense_tensors_can_be_encoded_and_decoded) { +TEST(ValueCodecTest, dense_tensors_can_be_encoded_and_decoded) { DenseTensorExample f1; f1.verify_encode_decode(); } @@ -271,7 +269,7 @@ struct MixedTensorExample : TensorExample { } }; -TEST(ValueCodecTest, require_that_mixed_tensors_can_be_encoded_and_decoded) { +TEST(ValueCodecTest, mixed_tensors_can_be_encoded_and_decoded) { MixedTensorExample f1; f1.verify_encode_decode(); } -- cgit v1.2.3 From 9a11a8d39ee7138e81fdc3d3b0adfd58cbeea84a Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Wed, 23 Sep 2020 16:36:35 +0000 Subject: rename new API new_encode -> encode_value new_decode -> decode_value --- eval/src/tests/eval/value_codec/value_codec_test.cpp | 6 +++--- .../tests/tensor/tensor_serialization/tensor_serialization_test.cpp | 4 ++-- eval/src/vespa/eval/eval/value_codec.cpp | 4 ++-- eval/src/vespa/eval/eval/value_codec.h | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/eval/src/tests/eval/value_codec/value_codec_test.cpp b/eval/src/tests/eval/value_codec/value_codec_test.cpp index 12e1af0d7d5..7968e5cb107 100644 --- a/eval/src/tests/eval/value_codec/value_codec_test.cpp +++ b/eval/src/tests/eval/value_codec/value_codec_test.cpp @@ -18,7 +18,7 @@ struct Factory { return value_from_spec(spec, simple); } std::unique_ptr decode(nbostream &input) { - return new_decode(input, simple); + return decode_value(input, simple); } } simple_factory; @@ -104,8 +104,8 @@ struct TensorExample { encode_with_float(expect_float); nbostream data_double; nbostream data_float; - new_encode(*make_tensor(false), data_double); - new_encode(*make_tensor(true), data_float); + encode_value(*make_tensor(false), data_double); + encode_value(*make_tensor(true), data_float); EXPECT_EQ(Memory(data_double.peek(), data_double.size()), Memory(expect_default.peek(), expect_default.size())); EXPECT_EQ(Memory(data_float.peek(), data_float.size()), diff --git a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp index c00b0e5d35b..4a6698b021e 100644 --- a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp +++ b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp @@ -53,9 +53,9 @@ TensorSpec verify_new_value_serialized(const ExpBuffer &exp, const TensorSpec &s auto new_value = vespalib::eval::value_from_spec(spec, factory); auto new_value_spec = vespalib::eval::spec_from_value(*new_value); nbostream actual; - vespalib::eval::new_encode(*new_value, actual); + vespalib::eval::encode_value(*new_value, actual); ASSERT_EQUAL(exp, actual); - auto new_decoded = vespalib::eval::new_decode(actual, factory); + auto new_decoded = vespalib::eval::decode_value(actual, factory); auto new_decoded_spec = vespalib::eval::spec_from_value(*new_decoded); EXPECT_EQUAL(0u, actual.size()); EXPECT_EQUAL(new_value_spec, new_decoded_spec); diff --git a/eval/src/vespa/eval/eval/value_codec.cpp b/eval/src/vespa/eval/eval/value_codec.cpp index ddca54995d0..e1ea0fa04c3 100644 --- a/eval/src/vespa/eval/eval/value_codec.cpp +++ b/eval/src/vespa/eval/eval/value_codec.cpp @@ -265,7 +265,7 @@ struct ContentEncoder { } }; -void new_encode(const Value &value, nbostream &output) { +void encode_value(const Value &value, nbostream &output) { size_t num_mapped_dims = value.type().count_mapped_dimensions(); size_t dense_subspace_size = value.type().dense_subspace_size(); Format format(value.type()); @@ -276,7 +276,7 @@ void new_encode(const Value &value, nbostream &output) { typify_invoke<1,TypifyCellType,ContentEncoder>(value.type().cell_type(), value, state, output); } -std::unique_ptr new_decode(nbostream &input, const ValueBuilderFactory &factory) { +std::unique_ptr decode_value(nbostream &input, const ValueBuilderFactory &factory) { Format format(input.getInt1_4Bytes()); ValueType type = decode_type(input, format); size_t num_mapped_dims = type.count_mapped_dimensions(); diff --git a/eval/src/vespa/eval/eval/value_codec.h b/eval/src/vespa/eval/eval/value_codec.h index 08b0ac73cc3..3644f952a6c 100644 --- a/eval/src/vespa/eval/eval/value_codec.h +++ b/eval/src/vespa/eval/eval/value_codec.h @@ -10,14 +10,14 @@ namespace vespalib { class nbostream; } namespace vespalib::eval { /** - * encode a value to binary format + * encode a value (which must support the new APIs) to binary format **/ -void new_encode(const Value &value, nbostream &output); +void encode_value(const Value &value, nbostream &output); /** * decode a value from binary format **/ -std::unique_ptr new_decode(nbostream &input, const ValueBuilderFactory &factory); +std::unique_ptr decode_value(nbostream &input, const ValueBuilderFactory &factory); /** * Make a value from a tensor spec using a value builder factory -- cgit v1.2.3 From f57624bf57555221c2a3c9a0e8217fa2136f172a Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Wed, 23 Sep 2020 17:05:18 +0000 Subject: Separate immediateCommit and allowEarlyAck --- .../documentmetastore/lid_reuse_delayer_config.cpp | 2 ++ .../documentmetastore/lid_reuse_delayer_config.h | 2 ++ .../proton/documentmetastore/lidreusedelayer.cpp | 1 + .../proton/documentmetastore/lidreusedelayer.h | 4 +++- .../proton/server/document_db_maintenance_config.h | 1 + .../proton/server/maintenance_jobs_injector.cpp | 2 +- .../searchcore/proton/server/storeonlyfeedview.cpp | 26 +++++++++++----------- .../searchcore/proton/server/storeonlyfeedview.h | 2 +- 8 files changed, 24 insertions(+), 16 deletions(-) diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp index b04bac5ef26..1f979d1566c 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp @@ -7,6 +7,7 @@ namespace proton::documentmetastore { LidReuseDelayerConfig::LidReuseDelayerConfig(const DocumentDBConfig & configSnapshot) : _visibilityDelay(configSnapshot.getMaintenanceConfigSP()->getVisibilityDelay()), + _allowEarlyAck(configSnapshot.getMaintenanceConfigSP()->allowEarlyAck()), _hasIndexedOrAttributeFields(configSnapshot.getSchemaSP()->getNumIndexFields() > 0 || configSnapshot.getSchemaSP()->getNumAttributeFields() > 0) { @@ -18,6 +19,7 @@ LidReuseDelayerConfig::LidReuseDelayerConfig() LidReuseDelayerConfig::LidReuseDelayerConfig(vespalib::duration visibilityDelay, bool hasIndexedOrAttributeFields_in) : _visibilityDelay(visibilityDelay), + _allowEarlyAck(visibilityDelay > 1ms), _hasIndexedOrAttributeFields(hasIndexedOrAttributeFields_in) { } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h index 82dab433a22..c81a2ff399f 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h @@ -15,6 +15,7 @@ class LidReuseDelayerConfig { private: vespalib::duration _visibilityDelay; + bool _allowEarlyAck; bool _hasIndexedOrAttributeFields; public: LidReuseDelayerConfig(); @@ -22,6 +23,7 @@ public: explicit LidReuseDelayerConfig(const DocumentDBConfig &configSnapshot); vespalib::duration visibilityDelay() const { return _visibilityDelay; } bool hasIndexedOrAttributeFields() const { return _hasIndexedOrAttributeFields; } + bool allowEarlyAck() const { return _allowEarlyAck; } }; } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp index 03dfd83a132..ed2202c830b 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp @@ -16,6 +16,7 @@ LidReuseDelayer::LidReuseDelayer(IThreadingService &writeService, IStore &docume : _writeService(writeService), _documentMetaStore(documentMetaStore), _immediateCommit(config.visibilityDelay() == vespalib::duration::zero()), + _allowEarlyAck(config.allowEarlyAck()), _config(config), _pendingLids() { diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h index 5f1de878b4a..ba407ab57f8 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h @@ -27,6 +27,7 @@ class LidReuseDelayer searchcorespi::index::IThreadingService &_writeService; IStore &_documentMetaStore; const bool _immediateCommit; + const bool _allowEarlyAck; LidReuseDelayerConfig _config; std::vector _pendingLids; // lids waiting for commit @@ -38,7 +39,8 @@ public: bool delayReuse(const std::vector &lids); std::vector getReuseLids(); - bool getImmediateCommit() const { return _immediateCommit; } + bool needImmediateCommit() const { return _immediateCommit; } + bool allowEarlyAck() const { return _allowEarlyAck; } const LidReuseDelayerConfig & getConfig() const { return _config; } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h index 4c0485baec6..5ed0ad7492c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h @@ -135,6 +135,7 @@ public: } vespalib::duration getVisibilityDelay() const { return _visibilityDelay; } bool hasVisibilityDelay() const { return _visibilityDelay > vespalib::duration::zero(); } + bool allowEarlyAck() const { return _visibilityDelay > 1ms; } const DocumentDBLidSpaceCompactionConfig &getLidSpaceCompactionConfig() const { return _lidSpaceCompaction; } diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp index d94bb2e3d03..d4b542a0af8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp @@ -105,7 +105,7 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller, AttributeUsageFilter &attributeUsageFilter) { controller.registerJobInMasterThread(std::make_unique(hbHandler, config.getHeartBeatConfig())); controller.registerJobInDefaultPool(std::make_unique(scPruner, config.getSessionCachePruneInterval())); - if (config.hasVisibilityDelay()) { + if (config.hasVisibilityDelay() && config.allowEarlyAck()) { controller.registerJobInMasterThread(std::make_unique(commit, config.getVisibilityDelay())); } const MaintenanceDocumentSubDB &mRemSubDB(controller.getRemSubDB()); diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index 217a3bb24d3..9927a2d2ce0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -210,11 +210,11 @@ moveMetaData(documentmetastore::IStore &meta_store, const DocumentId & doc_id, c } std::unique_ptr -createUncommitedLidTracker(bool needImmediateCommit) { - if (needImmediateCommit) { - return std::make_unique(); - } else { +createUncommitedLidTracker(bool allowEarlyAck) { + if (allowEarlyAck) { return std::make_unique(); + } else { + return std::make_unique(); } } @@ -229,7 +229,7 @@ StoreOnlyFeedView::StoreOnlyFeedView(const Context &ctx, const PersistentParams _docType(nullptr), _lidReuseDelayer(ctx._writeService, _documentMetaStoreContext->get(), ctx._lidReuseDelayerConfig), _pendingLidsForDocStore(), - _pendingLidsForCommit(createUncommitedLidTracker(_lidReuseDelayer.getImmediateCommit())), + _pendingLidsForCommit(createUncommitedLidTracker(_lidReuseDelayer.allowEarlyAck())), _schema(ctx._schema), _writeService(ctx._writeService), _params(params), @@ -275,7 +275,7 @@ StoreOnlyFeedView::internalForceCommit(SerialNum serialNum, OnForceCommitDoneTyp void StoreOnlyFeedView::considerEarlyAck(FeedToken & token) { - if ( ! needCommit() && token) { + if ( _lidReuseDelayer.allowEarlyAck() && token) { token.reset(); } } @@ -327,7 +327,7 @@ StoreOnlyFeedView::internalPut(FeedToken token, const PutOperation &putOp) bool docAlreadyExists = putOp.getValidPrevDbdId(_params._subDbId); if (putOp.getValidDbdId(_params._subDbId)) { - bool immediateCommit = needCommit(); + bool immediateCommit = needImmediateCommit(); const document::GlobalId &gid = docId.getGlobalId(); std::shared_ptr onWriteDone = createPutDoneContext(std::move(token), std::move(uncommitted), @@ -345,8 +345,8 @@ StoreOnlyFeedView::internalPut(FeedToken token, const PutOperation &putOp) } bool -StoreOnlyFeedView::needCommit() const { - return _lidReuseDelayer.getImmediateCommit(); +StoreOnlyFeedView::needImmediateCommit() const { + return _lidReuseDelayer.needImmediateCommit(); } void @@ -483,7 +483,7 @@ StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp) auto uncommitted = _pendingLidsForCommit->produce(updOp.getLid()); considerEarlyAck(token); - bool immediateCommit = needCommit(); + bool immediateCommit = needImmediateCommit(); auto onWriteDone = createUpdateDoneContext(std::move(token), std::move(uncommitted), updOp.getUpdate()); UpdateScope updateScope(*_schema, upd); updateAttributes(serialNum, lid, upd, immediateCommit, onWriteDone, updateScope); @@ -657,7 +657,7 @@ StoreOnlyFeedView::internalRemove(FeedToken token, IPendingLidTracker::Token unc std::move(pendingNotifyRemoveDone), (explicitReuseLid ? lid : 0u), std::move(moveDoneCtx)); removeSummary(serialNum, lid, onWriteDone); - bool immediateCommit = needCommit(); + bool immediateCommit = needImmediateCommit(); removeAttributes(serialNum, lid, immediateCommit, onWriteDone); removeIndexedFields(serialNum, lid, immediateCommit, onWriteDone); } @@ -770,7 +770,7 @@ StoreOnlyFeedView::handleDeleteBucket(const DeleteBucketOperation &delOp) void StoreOnlyFeedView::internalDeleteBucket(const DeleteBucketOperation &delOp) { - bool immediateCommit = needCommit(); + bool immediateCommit = needImmediateCommit(); size_t rm_count = removeDocuments(delOp, true, immediateCommit); LOG(debug, "internalDeleteBucket(): docType(%s), bucket(%s), lidsToRemove(%zu)", _params._docTypeName.toString().c_str(), delOp.getBucketId().toString().c_str(), rm_count); @@ -809,7 +809,7 @@ StoreOnlyFeedView::handleMove(const MoveOperation &moveOp, IDestructorCallback:: PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(moveOp, docId.getGlobalId(), docId); bool docAlreadyExists = moveOp.getValidPrevDbdId(_params._subDbId); if (moveOp.getValidDbdId(_params._subDbId)) { - bool immediateCommit = needCommit(); + bool immediateCommit = needImmediateCommit(); const document::GlobalId &gid = docId.getGlobalId(); std::shared_ptr onWriteDone = createPutDoneContext(FeedToken(), _pendingLidsForCommit->produce(moveOp.getLid()), diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h index 7d91ea86a22..4569e01f9fd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h @@ -161,7 +161,7 @@ private: void putSummary(SerialNum serialNum, Lid lid, DocumentSP doc, OnOperationDoneType onDone); void removeSummary(SerialNum serialNum, Lid lid, OnWriteDoneType onDone); void heartBeatSummary(SerialNum serialNum); - bool needCommit() const; + bool needImmediateCommit() const; bool useDocumentStore(SerialNum replaySerialNum) const { -- cgit v1.2.3 From 0944da353ad8912ed0cc04cf74808a753bde15c8 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Wed, 23 Sep 2020 17:07:13 +0000 Subject: fixes after review * move namespace end to correct place * remove ValueType::indexed_dimensions(), just loop over all dimensions when getting names to avoid making an extra vector * unit test ValueType::count_indexed_dimensions() --- eval/src/tests/eval/value_type/value_type_test.cpp | 9 ++++++-- eval/src/vespa/eval/eval/value_codec.cpp | 24 ++++++++++++---------- eval/src/vespa/eval/eval/value_type.cpp | 11 ---------- eval/src/vespa/eval/eval/value_type.h | 1 - 4 files changed, 20 insertions(+), 25 deletions(-) diff --git a/eval/src/tests/eval/value_type/value_type_test.cpp b/eval/src/tests/eval/value_type/value_type_test.cpp index 2b103a91b81..9f1519ee7c0 100644 --- a/eval/src/tests/eval/value_type/value_type_test.cpp +++ b/eval/src/tests/eval/value_type/value_type_test.cpp @@ -325,12 +325,17 @@ TEST("require that type-related predicate functions work as expected") { TEST_DO(verify_predicates(type("tensor(x[5],y{})"), false, false, true, false, false)); } -TEST("require that mapped dimensions can be counted") { +TEST("require that mapped and indexed dimensions can be counted") { EXPECT_EQUAL(type("double").count_mapped_dimensions(), 0u); + EXPECT_EQUAL(type("double").count_indexed_dimensions(), 0u); EXPECT_EQUAL(type("tensor(x[5],y[5])").count_mapped_dimensions(), 0u); + EXPECT_EQUAL(type("tensor(x[5],y[5])").count_indexed_dimensions(), 2u); EXPECT_EQUAL(type("tensor(x{},y[5])").count_mapped_dimensions(), 1u); - EXPECT_EQUAL(type("tensor(x[5],y{})").count_mapped_dimensions(), 1u); + EXPECT_EQUAL(type("tensor(x{},y[5])").count_indexed_dimensions(), 1u); + EXPECT_EQUAL(type("tensor(x[1],y{})").count_mapped_dimensions(), 1u); + EXPECT_EQUAL(type("tensor(x[1],y{})").count_indexed_dimensions(), 1u); EXPECT_EQUAL(type("tensor(x{},y{})").count_mapped_dimensions(), 2u); + EXPECT_EQUAL(type("tensor(x{},y{})").count_indexed_dimensions(), 0u); } TEST("require that dense subspace size calculation works as expected") { diff --git a/eval/src/vespa/eval/eval/value_codec.cpp b/eval/src/vespa/eval/eval/value_codec.cpp index e1ea0fa04c3..1755ce164ff 100644 --- a/eval/src/vespa/eval/eval/value_codec.cpp +++ b/eval/src/vespa/eval/eval/value_codec.cpp @@ -58,18 +58,20 @@ void maybe_encode_cell_type(nbostream &output, const Format &format, CellType ce void encode_type(nbostream &output, const Format &format, const ValueType &type) { maybe_encode_cell_type(output, format, type.cell_type()); if (format.has_sparse) { - const auto & dims = type.mapped_dimensions(); - output.putInt1_4Bytes(dims.size()); - for (const auto & dim : dims) { - output.writeSmallString(dim.name); + output.putInt1_4Bytes(type.count_mapped_dimensions()); + for (const auto & dim : type.dimensions()) { + if (dim.is_mapped()) { + output.writeSmallString(dim.name); + } } } if (format.has_dense) { - const auto & dims = type.indexed_dimensions(); - output.putInt1_4Bytes(dims.size()); - for (const auto & dim : dims) { - output.writeSmallString(dim.name); - output.putInt1_4Bytes(dim.size); + output.putInt1_4Bytes(type.count_indexed_dimensions()); + for (const auto & dim : type.dimensions()) { + if (dim.is_indexed()) { + output.writeSmallString(dim.name); + output.putInt1_4Bytes(dim.size); + } } } } @@ -236,8 +238,6 @@ struct CreateTensorSpecFromValue { } }; -} // namespace - struct EncodeState { size_t num_mapped_dims; size_t subspace_size; @@ -264,6 +264,8 @@ struct ContentEncoder { } } }; + +} // namespace void encode_value(const Value &value, nbostream &output) { size_t num_mapped_dims = value.type().count_mapped_dimensions(); diff --git a/eval/src/vespa/eval/eval/value_type.cpp b/eval/src/vespa/eval/eval/value_type.cpp index 3685d2715d8..4b776fc4bcd 100644 --- a/eval/src/vespa/eval/eval/value_type.cpp +++ b/eval/src/vespa/eval/eval/value_type.cpp @@ -220,17 +220,6 @@ ValueType::nontrivial_indexed_dimensions() const { return result; } -std::vector -ValueType::indexed_dimensions() const { - std::vector result; - for (const auto &dim: dimensions()) { - if (dim.is_indexed()) { - result.push_back(dim); - } - } - return result; -} - std::vector ValueType::mapped_dimensions() const { std::vector result; diff --git a/eval/src/vespa/eval/eval/value_type.h b/eval/src/vespa/eval/eval/value_type.h index 5fac4d5d27d..a0934de2704 100644 --- a/eval/src/vespa/eval/eval/value_type.h +++ b/eval/src/vespa/eval/eval/value_type.h @@ -65,7 +65,6 @@ public: size_t dense_subspace_size() const; const std::vector &dimensions() const { return _dimensions; } std::vector nontrivial_indexed_dimensions() const; - std::vector indexed_dimensions() const; std::vector mapped_dimensions() const; size_t dimension_index(const vespalib::string &name) const; std::vector dimension_names() const; -- cgit v1.2.3 From 3576e8402e028b75af6cc17f7c861aba6d45b1fc Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Wed, 23 Sep 2020 19:42:57 +0000 Subject: Early exit if closed --- .../src/main/java/com/yahoo/search/cluster/ClusterMonitor.java | 2 ++ .../src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java | 1 + 2 files changed, 3 insertions(+) diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java index 15cf4995b77..27d8bb27ee8 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java @@ -62,6 +62,8 @@ public class ClusterMonitor { /** Returns the configuration of this cluster monitor */ public MonitorConfiguration getConfiguration() { return configuration; } + public boolean isClosed() { return closed.get(); } + /** * Adds a new node for monitoring. * The object representing the node must diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java index 5e04f1d7a3e..5992d47855f 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java @@ -86,6 +86,7 @@ public class RpcPing implements Pinger, Client.ResponseReceiver { @Override public void receive(ResponseOrError response) { + if (clusterMonitor.isClosed()) return; if (node.isLastReceivedPong(pingSequenceId)) { pongHandler.handle(toPong(response)); } else { -- cgit v1.2.3 From 08ace27cf15655e6b9b3df61a217611297d05e57 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Wed, 23 Sep 2020 21:04:16 +0000 Subject: Since only a simple single decimal floatingpoint number is needed avoid using string and floating point formatting. Use a simpler variant you get a 15 X speedup of this frequently called code. --- .../com/yahoo/config/provision/NodeResources.java | 32 ++++++++++++++++++---- .../yahoo/config/provision/NodeResourcesTest.java | 26 ++++++++++++++++++ 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index 9cae0a08360..e5c2bf658ab 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -1,6 +1,9 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; +import java.text.DecimalFormat; +import java.text.FieldPosition; +import java.text.NumberFormat; import java.util.Locale; import java.util.Objects; import java.util.Optional; @@ -215,13 +218,32 @@ public class NodeResources { return Objects.hash(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } + private static StringBuffer appendDouble(StringBuffer sb, double d) { + sb.append((long)d).append('.').append(Math.round(d*10)%10); + return sb; + } @Override public String toString() { - return String.format(Locale.ENGLISH, "[vcpu: %1$.1f, memory: %2$.1f Gb, disk %3$.1f Gb" + - (bandwidthGbps > 0 ? ", bandwidth: %4$.1f Gbps" : "") + - ( ! diskSpeed.isDefault() ? ", disk speed: " + diskSpeed : "") + - ( ! storageType.isDefault() ? ", storage type: " + storageType : "") + "]", - vcpu, memoryGb, diskGb, bandwidthGbps); + StringBuffer sb = new StringBuffer("[vcpu: "); + appendDouble(sb, vcpu); + sb.append(", memory: "); + appendDouble(sb, memoryGb); + sb.append(" Gb, disk "); + appendDouble(sb, diskGb); + sb.append(" Gb"); + if (bandwidthGbps > 0) { + sb.append(", bandwidth: "); + appendDouble(sb, bandwidthGbps); + sb.append(" Gbps"); + } + if ( !diskSpeed.isDefault()) { + sb.append(", disk speed: ").append(diskSpeed); + } + if ( !storageType.isDefault()) { + sb.append(", storage type: ").append(storageType); + } + sb.append(']'); + return sb.toString(); } /** Returns true if all the resources of this are the same or larger than the given resources */ diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java index 21e1afeed17..8d39691a3bb 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java @@ -16,6 +16,32 @@ public class NodeResourcesTest { new NodeResources(1., 10., 100., 0).toString()); assertEquals("[vcpu: 0.3, memory: 3.3 Gb, disk 33.3 Gb, bandwidth: 0.3 Gbps]", new NodeResources(1/3., 10/3., 100/3., 0.3).toString()); + assertEquals("[vcpu: 0.7, memory: 6.7 Gb, disk 66.7 Gb, bandwidth: 0.7 Gbps]", + new NodeResources(2/3., 20/3., 200/3., 0.67).toString()); + } + + private long runTest(NodeResources [] resouces, int num) { + long sum = 0; + for (int i = 0; i < num; i++) { + for (NodeResources ns :resouces) { + sum += ns.toString().length(); + } + } + return sum; + } + @Test + public void benchmark() { + NodeResources [] resouces = new NodeResources[100]; + for (int i = 0; i < resouces.length; i++) { + resouces[i] = new NodeResources(1/3., 10/3., 100/3., 0.3); + } + int NUM_ITER = 100; // Use at least 100000 for proper benchmarking + long warmup = runTest(resouces, NUM_ITER); + long start = System.nanoTime(); + long benchmark = runTest(resouces, NUM_ITER); + long duration = System.nanoTime() - start; + System.out.println("NodeResources.toString() took " + duration/1000000 + " ms"); + assertEquals(warmup, benchmark); } } -- cgit v1.2.3 From d17b160b5c1f6cd3932a54b195ac0027c3ee3d24 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Wed, 23 Sep 2020 21:10:46 +0000 Subject: Update ABI --- container-search/abi-spec.json | 1 + 1 file changed, 1 insertion(+) diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 26efb6c6312..4f69f5a5b54 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -1987,6 +1987,7 @@ "public void (com.yahoo.search.cluster.NodeManager, boolean)", "public void start()", "public com.yahoo.search.cluster.MonitorConfiguration getConfiguration()", + "public boolean isClosed()", "public void add(java.lang.Object, boolean)", "public com.yahoo.search.cluster.BaseNodeMonitor getNodeMonitor(java.lang.Object)", "public synchronized void failed(java.lang.Object, com.yahoo.search.result.ErrorMessage)", -- cgit v1.2.3 From 9de031e5218a26aa0ee01ee313a97ea9195ac9cd Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Wed, 23 Sep 2020 21:48:13 +0000 Subject: Round prior to printing og digits. --- .../src/main/java/com/yahoo/config/provision/NodeResources.java | 3 ++- .../src/test/java/com/yahoo/config/provision/NodeResourcesTest.java | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index e5c2bf658ab..1c53ff18222 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -219,7 +219,8 @@ public class NodeResources { } private static StringBuffer appendDouble(StringBuffer sb, double d) { - sb.append((long)d).append('.').append(Math.round(d*10)%10); + long x10 = Math.round(d*10); + sb.append(x10/10).append('.').append(x10%10); return sb; } @Override diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java index 8d39691a3bb..044afa72a5d 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java @@ -16,8 +16,8 @@ public class NodeResourcesTest { new NodeResources(1., 10., 100., 0).toString()); assertEquals("[vcpu: 0.3, memory: 3.3 Gb, disk 33.3 Gb, bandwidth: 0.3 Gbps]", new NodeResources(1/3., 10/3., 100/3., 0.3).toString()); - assertEquals("[vcpu: 0.7, memory: 6.7 Gb, disk 66.7 Gb, bandwidth: 0.7 Gbps]", - new NodeResources(2/3., 20/3., 200/3., 0.67).toString()); + assertEquals("[vcpu: 0.7, memory: 9.0 Gb, disk 66.7 Gb, bandwidth: 0.7 Gbps]", + new NodeResources(2/3., 8.97, 200/3., 0.67).toString()); } private long runTest(NodeResources [] resouces, int num) { -- cgit v1.2.3 From f43b36c544f2a0b40f5300a85f3ba543d6621613 Mon Sep 17 00:00:00 2001 From: Håvard Pettersen Date: Wed, 23 Sep 2020 16:22:48 +0000 Subject: instruction-level generic join also move some stuff out of simple_value --- .../tests/eval/simple_value/simple_value_test.cpp | 18 ++- .../tests/eval/value_codec/value_codec_test.cpp | 28 ++-- .../tensor_serialization_test.cpp | 2 +- eval/src/vespa/eval/eval/CMakeLists.txt | 2 + eval/src/vespa/eval/eval/interpreted_function.cpp | 23 +++ eval/src/vespa/eval/eval/interpreted_function.h | 19 +++ eval/src/vespa/eval/eval/simple_value.cpp | 163 +-------------------- eval/src/vespa/eval/eval/simple_value.h | 141 +----------------- eval/src/vespa/eval/eval/tensor_instructions.cpp | 148 +++++++++++++++++++ eval/src/vespa/eval/eval/tensor_instructions.h | 24 +++ eval/src/vespa/eval/eval/tensor_plans.cpp | 86 +++++++++++ eval/src/vespa/eval/eval/tensor_plans.h | 77 ++++++++++ eval/src/vespa/eval/eval/value.h | 60 ++++++++ 13 files changed, 472 insertions(+), 319 deletions(-) create mode 100644 eval/src/vespa/eval/eval/tensor_instructions.cpp create mode 100644 eval/src/vespa/eval/eval/tensor_instructions.h create mode 100644 eval/src/vespa/eval/eval/tensor_plans.cpp create mode 100644 eval/src/vespa/eval/eval/tensor_plans.h diff --git a/eval/src/tests/eval/simple_value/simple_value_test.cpp b/eval/src/tests/eval/simple_value/simple_value_test.cpp index 233c09f7e2b..6fa89989bb7 100644 --- a/eval/src/tests/eval/simple_value/simple_value_test.cpp +++ b/eval/src/tests/eval/simple_value/simple_value_test.cpp @@ -2,6 +2,9 @@ #include #include +#include +#include +#include #include #include #include @@ -63,16 +66,19 @@ TensorSpec simple_tensor_join(const TensorSpec &a, const TensorSpec &b, join_fun } TensorSpec simple_value_new_join(const TensorSpec &a, const TensorSpec &b, join_fun_t function) { - auto lhs = value_from_spec(a, SimpleValueBuilderFactory()); - auto rhs = value_from_spec(b, SimpleValueBuilderFactory()); - auto result = new_join(*lhs, *rhs, function, SimpleValueBuilderFactory()); - return spec_from_value(*result); + Stash stash; + const auto &factory = SimpleValueBuilderFactory::get(); + auto lhs = value_from_spec(a, factory); + auto rhs = value_from_spec(b, factory); + auto my_op = tensor_instruction::make_join(lhs->type(), rhs->type(), function, factory, stash); + InterpretedFunction::EvalSingle single(my_op); + return spec_from_value(single.eval(std::vector({*lhs,*rhs}))); } TEST(SimpleValueTest, simple_values_can_be_converted_from_and_to_tensor_spec) { for (const auto &layout: layouts) { TensorSpec expect = spec(layout, N()); - std::unique_ptr value = value_from_spec(expect, SimpleValueBuilderFactory()); + std::unique_ptr value = value_from_spec(expect, SimpleValueBuilderFactory::get()); TensorSpec actual = spec_from_value(*value); EXPECT_EQ(actual, expect); } @@ -80,7 +86,7 @@ TEST(SimpleValueTest, simple_values_can_be_converted_from_and_to_tensor_spec) { TEST(SimpleValueTest, simple_value_can_be_built_and_inspected) { ValueType type = ValueType::from_spec("tensor(x{},y[2],z{})"); - SimpleValueBuilderFactory factory; + const auto &factory = SimpleValueBuilderFactory::get(); std::unique_ptr> builder = factory.create_value_builder(type); float seq = 0.0; for (vespalib::string x: {"a", "b", "c"}) { diff --git a/eval/src/tests/eval/value_codec/value_codec_test.cpp b/eval/src/tests/eval/value_codec/value_codec_test.cpp index 7968e5cb107..00f37e1f87a 100644 --- a/eval/src/tests/eval/value_codec/value_codec_test.cpp +++ b/eval/src/tests/eval/value_codec/value_codec_test.cpp @@ -12,15 +12,7 @@ using namespace vespalib; using namespace vespalib::eval; using namespace vespalib::eval::test; -struct Factory { - SimpleValueBuilderFactory simple; - std::unique_ptr from_spec(const TensorSpec &spec) { - return value_from_spec(spec, simple); - } - std::unique_ptr decode(nbostream &input) { - return decode_value(input, simple); - } -} simple_factory; +const ValueBuilderFactory &factory = SimpleValueBuilderFactory::get(); std::vector layouts = { {}, @@ -41,7 +33,7 @@ std::vector layouts = { TEST(ValueCodecTest, simple_values_can_be_converted_from_and_to_tensor_spec) { for (const auto &layout: layouts) { TensorSpec expect = spec(layout, N()); - std::unique_ptr value = simple_factory.from_spec(expect); + std::unique_ptr value = value_from_spec(expect, factory); TensorSpec actual = spec_from_value(*value); EXPECT_EQ(actual, expect); } @@ -53,7 +45,7 @@ TEST(ValueCodecTest, simple_values_can_be_built_using_tensor_spec) { .add({{"w", "xxx"}, {"x", 0}, {"y", "yyy"}, {"z", 1}}, 2.0) .add({{"w", "yyy"}, {"x", 1}, {"y", "xxx"}, {"z", 0}}, 3.0) .add({{"w", "yyy"}, {"x", 1}, {"y", "yyy"}, {"z", 1}}, 4.0); - Value::UP tensor = simple_factory.from_spec(spec); + Value::UP tensor = value_from_spec(spec, factory); TensorSpec full_spec("tensor(w{},x[2],y{},z[2])"); full_spec .add({{"w", "xxx"}, {"x", 0}, {"y", "xxx"}, {"z", 0}}, 1.0) @@ -72,7 +64,7 @@ TEST(ValueCodecTest, simple_values_can_be_built_using_tensor_spec) { .add({{"w", "yyy"}, {"x", 1}, {"y", "xxx"}, {"z", 1}}, 0.0) .add({{"w", "yyy"}, {"x", 1}, {"y", "yyy"}, {"z", 0}}, 0.0) .add({{"w", "yyy"}, {"x", 1}, {"y", "yyy"}, {"z", 1}}, 4.0); - Value::UP full_tensor = simple_factory.from_spec(full_spec); + Value::UP full_tensor = value_from_spec(full_spec, factory); EXPECT_EQUAL(full_spec, spec_from_value(*tensor)); EXPECT_EQUAL(full_spec, spec_from_value(*full_tensor)); }; @@ -110,9 +102,9 @@ struct TensorExample { Memory(expect_default.peek(), expect_default.size())); EXPECT_EQ(Memory(data_float.peek(), data_float.size()), Memory(expect_float.peek(), expect_float.size())); - EXPECT_EQ(spec_from_value(*simple_factory.decode(expect_default)), make_spec(false)); - EXPECT_EQ(spec_from_value(*simple_factory.decode(expect_double)), make_spec(false)); - EXPECT_EQ(spec_from_value(*simple_factory.decode(expect_float)), make_spec(true)); + EXPECT_EQ(spec_from_value(*decode_value(expect_default, factory)), make_spec(false)); + EXPECT_EQ(spec_from_value(*decode_value(expect_double, factory)), make_spec(false)); + EXPECT_EQ(spec_from_value(*decode_value(expect_float, factory)), make_spec(true)); } }; TensorExample::~TensorExample() = default; @@ -127,7 +119,7 @@ struct SparseTensorExample : TensorExample { .add({{"x","b"},{"y","a"}}, 3); } std::unique_ptr make_tensor(bool use_float) const override { - return simple_factory.from_spec(make_spec(use_float)); + return value_from_spec(make_spec(use_float), factory); } template void encode_inner(nbostream &dst) const { @@ -179,7 +171,7 @@ struct DenseTensorExample : TensorExample { .add({{"x",2},{"y",1}}, 6); } std::unique_ptr make_tensor(bool use_float) const override { - return simple_factory.from_spec(make_spec(use_float)); + return value_from_spec(make_spec(use_float), factory); } template void encode_inner(nbostream &dst) const { @@ -229,7 +221,7 @@ struct MixedTensorExample : TensorExample { .add({{"x","b"},{"y","a"},{"z",1}}, 6); } std::unique_ptr make_tensor(bool use_float) const override { - return simple_factory.from_spec(make_spec(use_float)); + return value_from_spec(make_spec(use_float), factory); } template void encode_inner(nbostream &dst) const { diff --git a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp index 4a6698b021e..e1009969b43 100644 --- a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp +++ b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp @@ -49,7 +49,7 @@ void verify_cells_only(const ExpBuffer &exp, const TensorSpec &spec) { } TensorSpec verify_new_value_serialized(const ExpBuffer &exp, const TensorSpec &spec) { - auto factory = vespalib::eval::SimpleValueBuilderFactory(); + const auto &factory = vespalib::eval::SimpleValueBuilderFactory::get(); auto new_value = vespalib::eval::value_from_spec(spec, factory); auto new_value_spec = vespalib::eval::spec_from_value(*new_value); nbostream actual; diff --git a/eval/src/vespa/eval/eval/CMakeLists.txt b/eval/src/vespa/eval/eval/CMakeLists.txt index 84cebe8cfd0..53ee6ed87c0 100644 --- a/eval/src/vespa/eval/eval/CMakeLists.txt +++ b/eval/src/vespa/eval/eval/CMakeLists.txt @@ -25,7 +25,9 @@ vespa_add_library(eval_eval OBJECT tensor.cpp tensor_engine.cpp tensor_function.cpp + tensor_instructions.cpp tensor_nodes.cpp + tensor_plans.cpp tensor_spec.cpp value.cpp value_codec.cpp diff --git a/eval/src/vespa/eval/eval/interpreted_function.cpp b/eval/src/vespa/eval/eval/interpreted_function.cpp index 121db4ffb6e..e9c82789991 100644 --- a/eval/src/vespa/eval/eval/interpreted_function.cpp +++ b/eval/src/vespa/eval/eval/interpreted_function.cpp @@ -7,6 +7,7 @@ #include "tensor_engine.h" #include "make_tensor_function.h" #include "compile_tensor_function.h" +#include "simple_tensor_engine.h" #include #include #include @@ -117,4 +118,26 @@ InterpretedFunction::detect_issues(const Function &function) return Function::Issues(std::move(checker.issues)); } +InterpretedFunction::EvalSingle::EvalSingle(Instruction op) + : _state(SimpleTensorEngine::ref()), + _op(op) +{ +} + +InterpretedFunction::EvalSingle::EvalSingle(const TensorEngine &engine, Instruction op) + : _state(engine), + _op(op) +{ +} + +const Value & +InterpretedFunction::EvalSingle::eval(const std::vector &stack) +{ + _state.stash.clear(); + _state.stack = stack; + _op.perform(_state); + assert(_state.stack.size() == 1); + return _state.stack.back(); +} + } diff --git a/eval/src/vespa/eval/eval/interpreted_function.h b/eval/src/vespa/eval/eval/interpreted_function.h index fb67fcb0b74..f94e3144cc5 100644 --- a/eval/src/vespa/eval/eval/interpreted_function.h +++ b/eval/src/vespa/eval/eval/interpreted_function.h @@ -102,6 +102,25 @@ public: const Value &eval(Context &ctx, const LazyParams ¶ms) const; double estimate_cost_us(const std::vector ¶ms, double budget = 5.0) const; static Function::Issues detect_issues(const Function &function); + + /** + * This inner class is used for testing and benchmarking. It runs + * a single interpreted instruction in isolation. Note that + * instructions manipulating the program counter or resolving + * parameters may not be run in this way. Also note that the stack + * must contain exactly one value after the instruction is + * executed. If no tensor engine is specified, SimpleTensorEngine + * is used (typically for instructions ignoring the engine). + **/ + class EvalSingle { + private: + State _state; + Instruction _op; + public: + EvalSingle(Instruction op); + EvalSingle(const TensorEngine &engine, Instruction op); + const Value &eval(const std::vector &stack); + }; }; } diff --git a/eval/src/vespa/eval/eval/simple_value.cpp b/eval/src/vespa/eval/eval/simple_value.cpp index 3bf5351f0d2..1ccf6f8cd25 100644 --- a/eval/src/vespa/eval/eval/simple_value.cpp +++ b/eval/src/vespa/eval/eval/simple_value.cpp @@ -97,79 +97,6 @@ public: } }; -// Contains various state needed to perform the sparse part (all -// mapped dimensions) of the join operation. Performs swapping of -// sparse indexes to ensure that we look up entries from the smallest -// index in the largest index. -struct SparseJoinState { - bool swapped; - const Value::Index &first_index; - const Value::Index &second_index; - const std::vector &second_view_dims; - std::vector full_address; - std::vector first_address; - std::vector address_overlap; - std::vector second_only_address; - size_t lhs_subspace; - size_t rhs_subspace; - size_t &first_subspace; - size_t &second_subspace; - - SparseJoinState(const SparseJoinPlan &plan, const Value::Index &lhs, const Value::Index &rhs) - : swapped(rhs.size() < lhs.size()), - first_index(swapped ? rhs : lhs), second_index(swapped ? lhs : rhs), - second_view_dims(swapped ? plan.lhs_overlap : plan.rhs_overlap), - full_address(plan.sources.size()), - first_address(), address_overlap(), second_only_address(), - lhs_subspace(), rhs_subspace(), - first_subspace(swapped ? rhs_subspace : lhs_subspace), - second_subspace(swapped ? lhs_subspace : rhs_subspace) - { - auto first_source = swapped ? SparseJoinPlan::Source::RHS : SparseJoinPlan::Source::LHS; - for (size_t i = 0; i < full_address.size(); ++i) { - if (plan.sources[i] == SparseJoinPlan::Source::BOTH) { - first_address.push_back(&full_address[i]); - address_overlap.push_back(&full_address[i]); - } else if (plan.sources[i] == first_source) { - first_address.push_back(&full_address[i]); - } else { - second_only_address.push_back(&full_address[i]); - } - } - } - ~SparseJoinState(); -}; -SparseJoinState::~SparseJoinState() = default; - -// Treats all values as mixed tensors. Needs output cell type as well -// as input cell types since output cell type cannot always be -// directly inferred. -struct GenericJoin { - template static std::unique_ptr - invoke(const Value &lhs, const Value &rhs, join_fun_t function, - const SparseJoinPlan &sparse_plan, const DenseJoinPlan &dense_plan, - const ValueType &res_type, const ValueBuilderFactory &factory) - { - Fun fun(function); - auto lhs_cells = lhs.cells().typify(); - auto rhs_cells = rhs.cells().typify(); - SparseJoinState state(sparse_plan, lhs.index(), rhs.index()); - auto builder = factory.create_value_builder(res_type, sparse_plan.sources.size(), dense_plan.out_size, state.first_index.size()); - auto outer = state.first_index.create_view({}); - auto inner = state.second_index.create_view(state.second_view_dims); - outer->lookup({}); - while (outer->next_result(state.first_address, state.first_subspace)) { - inner->lookup(state.address_overlap); - while (inner->next_result(state.second_only_address, state.second_subspace)) { - OCT *dst = builder->add_subspace(state.full_address).begin(); - auto join_cells = [&](size_t lhs_idx, size_t rhs_idx) { *dst++ = fun(lhs_cells[lhs_idx], rhs_cells[rhs_idx]); }; - dense_plan.execute(dense_plan.lhs_size * state.lhs_subspace, dense_plan.rhs_size * state.rhs_subspace, join_cells); - } - } - return builder->build(std::move(builder)); - } -}; - } // namespace //----------------------------------------------------------------------------- @@ -229,6 +156,9 @@ SimpleValueT::add_subspace(const std::vector &addr) //----------------------------------------------------------------------------- +SimpleValueBuilderFactory::SimpleValueBuilderFactory() = default; +SimpleValueBuilderFactory SimpleValueBuilderFactory::_factory; + std::unique_ptr SimpleValueBuilderFactory::create_value_builder_base(const ValueType &type, size_t num_mapped_dims_in, size_t subspace_size_in, size_t) const @@ -238,91 +168,4 @@ SimpleValueBuilderFactory::create_value_builder_base(const ValueType &type, //----------------------------------------------------------------------------- -DenseJoinPlan::DenseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type) - : lhs_size(1), rhs_size(1), out_size(1), loop_cnt(), lhs_stride(), rhs_stride() -{ - enum class Case { NONE, LHS, RHS, BOTH }; - Case prev_case = Case::NONE; - auto update_plan = [&](Case my_case, size_t my_size, size_t in_lhs, size_t in_rhs) { - if (my_case == prev_case) { - assert(!loop_cnt.empty()); - loop_cnt.back() *= my_size; - } else { - loop_cnt.push_back(my_size); - lhs_stride.push_back(in_lhs); - rhs_stride.push_back(in_rhs); - prev_case = my_case; - } - }; - auto visitor = overload - { - [&](visit_ranges_first, const auto &a) { update_plan(Case::LHS, a.size, 1, 0); }, - [&](visit_ranges_second, const auto &b) { update_plan(Case::RHS, b.size, 0, 1); }, - [&](visit_ranges_both, const auto &a, const auto &) { update_plan(Case::BOTH, a.size, 1, 1); } - }; - auto lhs_dims = lhs_type.nontrivial_indexed_dimensions(); - auto rhs_dims = rhs_type.nontrivial_indexed_dimensions(); - visit_ranges(visitor, lhs_dims.begin(), lhs_dims.end(), rhs_dims.begin(), rhs_dims.end(), - [](const auto &a, const auto &b){ return (a.name < b.name); }); - for (size_t i = loop_cnt.size(); i-- > 0; ) { - out_size *= loop_cnt[i]; - if (lhs_stride[i] != 0) { - lhs_stride[i] = lhs_size; - lhs_size *= loop_cnt[i]; - } - if (rhs_stride[i] != 0) { - rhs_stride[i] = rhs_size; - rhs_size *= loop_cnt[i]; - } - } -} - -DenseJoinPlan::~DenseJoinPlan() = default; - -//----------------------------------------------------------------------------- - -SparseJoinPlan::SparseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type) - : sources(), lhs_overlap(), rhs_overlap() -{ - size_t lhs_idx = 0; - size_t rhs_idx = 0; - auto visitor = overload - { - [&](visit_ranges_first, const auto &) { - sources.push_back(Source::LHS); - ++lhs_idx; - }, - [&](visit_ranges_second, const auto &) { - sources.push_back(Source::RHS); - ++rhs_idx; - }, - [&](visit_ranges_both, const auto &, const auto &) { - sources.push_back(Source::BOTH); - lhs_overlap.push_back(lhs_idx++); - rhs_overlap.push_back(rhs_idx++); - } - }; - auto lhs_dims = lhs_type.mapped_dimensions(); - auto rhs_dims = rhs_type.mapped_dimensions(); - visit_ranges(visitor, lhs_dims.begin(), lhs_dims.end(), rhs_dims.begin(), rhs_dims.end(), - [](const auto &a, const auto &b){ return (a.name < b.name); }); -} - -SparseJoinPlan::~SparseJoinPlan() = default; - -//----------------------------------------------------------------------------- - -using JoinTypify = TypifyValue; - -std::unique_ptr new_join(const Value &a, const Value &b, join_fun_t function, const ValueBuilderFactory &factory) { - auto res_type = ValueType::join(a.type(), b.type()); - assert(!res_type.is_error()); - SparseJoinPlan sparse_plan(a.type(), b.type()); - DenseJoinPlan dense_plan(a.type(), b.type()); - return typify_invoke<4,JoinTypify,GenericJoin>(a.type().cell_type(), b.type().cell_type(), res_type.cell_type(), function, - a, b, function, sparse_plan, dense_plan, res_type, factory); } - -//----------------------------------------------------------------------------- - -} // namespace diff --git a/eval/src/vespa/eval/eval/simple_value.h b/eval/src/vespa/eval/eval/simple_value.h index 14ba5df835e..af943b2904d 100644 --- a/eval/src/vespa/eval/eval/simple_value.h +++ b/eval/src/vespa/eval/eval/simple_value.h @@ -15,66 +15,6 @@ class TensorSpec; using TypedCells = ::vespalib::tensor::TypedCells; -/** - * Tagging interface used as return type from factories before - * downcasting to actual builder with specialized cell type. - **/ -struct ValueBuilderBase { - virtual ~ValueBuilderBase() {} -}; - -/** - * Interface used to build a value one dense subspace at a - * time. Enables decoupling of what the value should contain from how - * to store the value. - **/ -template -struct ValueBuilder : ValueBuilderBase { - // add a dense subspace for the given address (label for all - // mapped dimensions in canonical order). Note that previously - // returned subspaces will be invalidated when new subspaces are - // added. Also note that adding the same subspace multiple times - // is not allowed. - virtual ArrayRef add_subspace(const std::vector &addr) = 0; - - // Given the ownership of the builder itself, produce the newly - // created value. This means that builders can only be used once, - // it also means values can build themselves. - virtual std::unique_ptr build(std::unique_ptr self) = 0; -}; - -/** - * Factory able to create appropriate value builders. We do not really - * care about the full mathematical type here, but it needs to be - * passed since it is exposed in the value api. The expected number of - * subspaces is also passed since it enables the builder to pre-size - * internal structures appropriately. Note that since we are not able - * to have virtual templated functions we need to cast the created - * builder. With interoperability between all values. - **/ -struct ValueBuilderFactory { - template - std::unique_ptr> create_value_builder(const ValueType &type, - size_t num_mapped_dims_in, size_t subspace_size_in, size_t expected_subspaces) const - { - assert(check_cell_type(type.cell_type())); - auto base = create_value_builder_base(type, num_mapped_dims_in, subspace_size_in, expected_subspaces); - ValueBuilder *builder = dynamic_cast*>(base.get()); - assert(builder); - base.release(); - return std::unique_ptr>(builder); - } - template - std::unique_ptr> create_value_builder(const ValueType &type) const - { - return create_value_builder(type, type.count_mapped_dimensions(), type.dense_subspace_size(), 1); - } - virtual ~ValueBuilderFactory() {} -protected: - virtual std::unique_ptr create_value_builder_base(const ValueType &type, - size_t num_mapped_dims_in, size_t subspace_size_in, size_t expected_subspaces) const = 0; -}; - /** * A simple implementation of a generic value that can also be used to * build new values. This class focuses on simplicity over speed and @@ -124,83 +64,16 @@ public: }; /** - * ValueBuilderFactory implementation for SimpleValue. + * ValueBuilderFactory implementation for SimpleValue. **/ -struct SimpleValueBuilderFactory : ValueBuilderFactory { - ~SimpleValueBuilderFactory() override {} -protected: +class SimpleValueBuilderFactory : public ValueBuilderFactory { +private: + SimpleValueBuilderFactory(); + static SimpleValueBuilderFactory _factory; std::unique_ptr create_value_builder_base(const ValueType &type, size_t num_mapped_dims_in, size_t subspace_size_in, size_t expected_subspaces) const override; +public: + static const SimpleValueBuilderFactory &get() { return _factory; } }; -/** - * Plan for how to traverse two partially overlapping dense subspaces - * in parallel, identifying all matching cell index combinations, in - * the exact order the joined cells will be stored in the result. The - * plan can be made up-front during tensor function compilation. - **/ -struct DenseJoinPlan { - size_t lhs_size; - size_t rhs_size; - size_t out_size; - std::vector loop_cnt; - std::vector lhs_stride; - std::vector rhs_stride; - DenseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type); - ~DenseJoinPlan(); - template void execute(size_t lhs, size_t rhs, F &&f) const { - switch(loops_left(0)) { - case 0: return execute_few(0, lhs, rhs, std::forward(f)); - case 1: return execute_few(0, lhs, rhs, std::forward(f)); - case 2: return execute_few(0, lhs, rhs, std::forward(f)); - case 3: return execute_few(0, lhs, rhs, std::forward(f)); - default: return execute_many(0, lhs, rhs, std::forward(f)); - } - } -private: - size_t loops_left(size_t idx) const { return (loop_cnt.size() - idx); } - template void execute_few(size_t idx, size_t lhs, size_t rhs, F &&f) const { - if constexpr (N == 0) { - f(lhs, rhs); - } else { - for (size_t i = 0; i < loop_cnt[idx]; ++i, lhs += lhs_stride[idx], rhs += rhs_stride[idx]) { - execute_few(idx + 1, lhs, rhs, std::forward(f)); - } - } - } - template void execute_many(size_t idx, size_t lhs, size_t rhs, F &&f) const { - for (size_t i = 0; i < loop_cnt[idx]; ++i, lhs += lhs_stride[idx], rhs += rhs_stride[idx]) { - if (loops_left(idx + 1) == 3) { - execute_few(idx + 1, lhs, rhs, std::forward(f)); - } else { - execute_many(idx + 1, lhs, rhs, std::forward(f)); - } - } - } -}; - -/** - * Plan for how to join the sparse part (all mapped dimensions) - * between two values. The plan can be made up-front during tensor - * function compilation. - **/ -struct SparseJoinPlan { - enum class Source { LHS, RHS, BOTH }; - std::vector sources; - std::vector lhs_overlap; - std::vector rhs_overlap; - SparseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type); - ~SparseJoinPlan(); -}; - -/** - * Generic join operation treating both values as mixed - * tensors. Packaging will change, and while the baseline join will - * not have information about low-level value class implementations, - * it will have up-front knowledge about types, specifically - * dimensional overlap and result type. - **/ -using join_fun_t = double (*)(double, double); -std::unique_ptr new_join(const Value &a, const Value &b, join_fun_t function, const ValueBuilderFactory &factory); - } diff --git a/eval/src/vespa/eval/eval/tensor_instructions.cpp b/eval/src/vespa/eval/eval/tensor_instructions.cpp new file mode 100644 index 00000000000..d839bfdad50 --- /dev/null +++ b/eval/src/vespa/eval/eval/tensor_instructions.cpp @@ -0,0 +1,148 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "tensor_instructions.h" +#include "tensor_plans.h" +#include "inline_operation.h" +#include +#include + +namespace vespalib::eval::tensor_instruction { + +using State = InterpretedFunction::State; +using Instruction = InterpretedFunction::Instruction; + +//----------------------------------------------------------------------------- + +namespace { + +//----------------------------------------------------------------------------- + +template uint64_t wrap_param(const IN &value_in) { + const T &value = value_in; + static_assert(sizeof(uint64_t) == sizeof(&value)); + return (uint64_t)&value; +} + +template const T &unwrap_param(uint64_t param) { + return *((const T *)param); +} + +//----------------------------------------------------------------------------- + +struct JoinParam { + ValueType res_type; + SparseJoinPlan sparse_plan; + DenseJoinPlan dense_plan; + join_fun_t function; + const ValueBuilderFactory &factory; + JoinParam(const ValueType &lhs_type, const ValueType &rhs_type, + join_fun_t function_in, const ValueBuilderFactory &factory_in) + : res_type(ValueType::join(lhs_type, rhs_type)), + sparse_plan(lhs_type, rhs_type), + dense_plan(lhs_type, rhs_type), + function(function_in), + factory(factory_in) + { + assert(!res_type.is_error()); + } + ~JoinParam(); +}; +JoinParam::~JoinParam() = default; + +//----------------------------------------------------------------------------- + +// Contains various state needed to perform the sparse part (all +// mapped dimensions) of the join operation. Performs swapping of +// sparse indexes to ensure that we look up entries from the smallest +// index in the largest index. +struct SparseJoinState { + bool swapped; + const Value::Index &first_index; + const Value::Index &second_index; + const std::vector &second_view_dims; + std::vector full_address; + std::vector first_address; + std::vector address_overlap; + std::vector second_only_address; + size_t lhs_subspace; + size_t rhs_subspace; + size_t &first_subspace; + size_t &second_subspace; + + SparseJoinState(const SparseJoinPlan &plan, const Value::Index &lhs, const Value::Index &rhs) + : swapped(rhs.size() < lhs.size()), + first_index(swapped ? rhs : lhs), second_index(swapped ? lhs : rhs), + second_view_dims(swapped ? plan.lhs_overlap : plan.rhs_overlap), + full_address(plan.sources.size()), + first_address(), address_overlap(), second_only_address(), + lhs_subspace(), rhs_subspace(), + first_subspace(swapped ? rhs_subspace : lhs_subspace), + second_subspace(swapped ? lhs_subspace : rhs_subspace) + { + auto first_source = swapped ? SparseJoinPlan::Source::RHS : SparseJoinPlan::Source::LHS; + for (size_t i = 0; i < full_address.size(); ++i) { + if (plan.sources[i] == SparseJoinPlan::Source::BOTH) { + first_address.push_back(&full_address[i]); + address_overlap.push_back(&full_address[i]); + } else if (plan.sources[i] == first_source) { + first_address.push_back(&full_address[i]); + } else { + second_only_address.push_back(&full_address[i]); + } + } + } + ~SparseJoinState(); +}; +SparseJoinState::~SparseJoinState() = default; + +template +void my_generic_join(State &state, uint64_t param_in) { + const auto ¶m = unwrap_param(param_in); + Fun fun(param.function); + const Value &lhs = state.peek(1); + const Value &rhs = state.peek(0); + auto lhs_cells = lhs.cells().typify(); + auto rhs_cells = rhs.cells().typify(); + SparseJoinState sparse(param.sparse_plan, lhs.index(), rhs.index()); + auto builder = param.factory.create_value_builder(param.res_type, param.sparse_plan.sources.size(), param.dense_plan.out_size, sparse.first_index.size()); + auto outer = sparse.first_index.create_view({}); + auto inner = sparse.second_index.create_view(sparse.second_view_dims); + outer->lookup({}); + while (outer->next_result(sparse.first_address, sparse.first_subspace)) { + inner->lookup(sparse.address_overlap); + while (inner->next_result(sparse.second_only_address, sparse.second_subspace)) { + OCT *dst = builder->add_subspace(sparse.full_address).begin(); + auto join_cells = [&](size_t lhs_idx, size_t rhs_idx) { *dst++ = fun(lhs_cells[lhs_idx], rhs_cells[rhs_idx]); }; + param.dense_plan.execute(param.dense_plan.lhs_size * sparse.lhs_subspace, param.dense_plan.rhs_size * sparse.rhs_subspace, join_cells); + } + } + auto &result = state.stash.create>(builder->build(std::move(builder))); + const Value &result_ref = *(result.get()); + state.pop_pop_push(result_ref); +}; + +struct SelectGenericJoin { + template static auto invoke() { + return my_generic_join; + } +}; + +//----------------------------------------------------------------------------- + +} // namespace + +//----------------------------------------------------------------------------- + +using JoinTypify = TypifyValue; + +Instruction make_join(const ValueType &lhs_type, const ValueType &rhs_type, join_fun_t function, + const ValueBuilderFactory &factory, Stash &stash) +{ + auto ¶m = stash.create(lhs_type, rhs_type, function, factory); + auto fun = typify_invoke<4,JoinTypify,SelectGenericJoin>(lhs_type.cell_type(), rhs_type.cell_type(), param.res_type.cell_type(), function); + return Instruction(fun, wrap_param(param)); +} + +//----------------------------------------------------------------------------- + +} diff --git a/eval/src/vespa/eval/eval/tensor_instructions.h b/eval/src/vespa/eval/eval/tensor_instructions.h new file mode 100644 index 00000000000..f5c5e88d07c --- /dev/null +++ b/eval/src/vespa/eval/eval/tensor_instructions.h @@ -0,0 +1,24 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "interpreted_function.h" + +namespace vespalib { + +class Stash; + +namespace eval { + +struct JoinPlan; + +namespace tensor_instruction { + +using join_fun_t = double (*)(double, double); + +InterpretedFunction::Instruction make_join(const ValueType &lhs_type, const ValueType &rhs_type, join_fun_t function, + const ValueBuilderFactory &factory, Stash &stash); + +} +} +} diff --git a/eval/src/vespa/eval/eval/tensor_plans.cpp b/eval/src/vespa/eval/eval/tensor_plans.cpp new file mode 100644 index 00000000000..203ca72c92a --- /dev/null +++ b/eval/src/vespa/eval/eval/tensor_plans.cpp @@ -0,0 +1,86 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "tensor_plans.h" +#include +#include +#include + +namespace vespalib::eval { + +//----------------------------------------------------------------------------- + +DenseJoinPlan::DenseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type) + : lhs_size(1), rhs_size(1), out_size(1), loop_cnt(), lhs_stride(), rhs_stride() +{ + enum class Case { NONE, LHS, RHS, BOTH }; + Case prev_case = Case::NONE; + auto update_plan = [&](Case my_case, size_t my_size, size_t in_lhs, size_t in_rhs) { + if (my_case == prev_case) { + assert(!loop_cnt.empty()); + loop_cnt.back() *= my_size; + } else { + loop_cnt.push_back(my_size); + lhs_stride.push_back(in_lhs); + rhs_stride.push_back(in_rhs); + prev_case = my_case; + } + }; + auto visitor = overload + { + [&](visit_ranges_first, const auto &a) { update_plan(Case::LHS, a.size, 1, 0); }, + [&](visit_ranges_second, const auto &b) { update_plan(Case::RHS, b.size, 0, 1); }, + [&](visit_ranges_both, const auto &a, const auto &) { update_plan(Case::BOTH, a.size, 1, 1); } + }; + auto lhs_dims = lhs_type.nontrivial_indexed_dimensions(); + auto rhs_dims = rhs_type.nontrivial_indexed_dimensions(); + visit_ranges(visitor, lhs_dims.begin(), lhs_dims.end(), rhs_dims.begin(), rhs_dims.end(), + [](const auto &a, const auto &b){ return (a.name < b.name); }); + for (size_t i = loop_cnt.size(); i-- > 0; ) { + out_size *= loop_cnt[i]; + if (lhs_stride[i] != 0) { + lhs_stride[i] = lhs_size; + lhs_size *= loop_cnt[i]; + } + if (rhs_stride[i] != 0) { + rhs_stride[i] = rhs_size; + rhs_size *= loop_cnt[i]; + } + } +} + +DenseJoinPlan::~DenseJoinPlan() = default; + +//----------------------------------------------------------------------------- + +SparseJoinPlan::SparseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type) + : sources(), lhs_overlap(), rhs_overlap() +{ + size_t lhs_idx = 0; + size_t rhs_idx = 0; + auto visitor = overload + { + [&](visit_ranges_first, const auto &) { + sources.push_back(Source::LHS); + ++lhs_idx; + }, + [&](visit_ranges_second, const auto &) { + sources.push_back(Source::RHS); + ++rhs_idx; + }, + [&](visit_ranges_both, const auto &, const auto &) { + sources.push_back(Source::BOTH); + lhs_overlap.push_back(lhs_idx++); + rhs_overlap.push_back(rhs_idx++); + } + }; + auto lhs_dims = lhs_type.mapped_dimensions(); + auto rhs_dims = rhs_type.mapped_dimensions(); + visit_ranges(visitor, lhs_dims.begin(), lhs_dims.end(), rhs_dims.begin(), rhs_dims.end(), + [](const auto &a, const auto &b){ return (a.name < b.name); }); +} + +SparseJoinPlan::~SparseJoinPlan() = default; + +//----------------------------------------------------------------------------- + +} diff --git a/eval/src/vespa/eval/eval/tensor_plans.h b/eval/src/vespa/eval/eval/tensor_plans.h new file mode 100644 index 00000000000..9a924a8ba48 --- /dev/null +++ b/eval/src/vespa/eval/eval/tensor_plans.h @@ -0,0 +1,77 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "value_type.h" +#include +#include + +namespace vespalib::eval { + +class ValueBuilderFactory; + +//----------------------------------------------------------------------------- + +/** + * Plan for how to traverse two partially overlapping dense subspaces + * in parallel, identifying all matching cell index combinations, in + * the exact order the joined cells will be stored in the result. The + * plan can be made up-front during tensor function compilation. + **/ +struct DenseJoinPlan { + size_t lhs_size; + size_t rhs_size; + size_t out_size; + std::vector loop_cnt; + std::vector lhs_stride; + std::vector rhs_stride; + DenseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type); + ~DenseJoinPlan(); + template void execute(size_t lhs, size_t rhs, F &&f) const { + switch(loops_left(0)) { + case 0: return execute_few(0, lhs, rhs, std::forward(f)); + case 1: return execute_few(0, lhs, rhs, std::forward(f)); + case 2: return execute_few(0, lhs, rhs, std::forward(f)); + case 3: return execute_few(0, lhs, rhs, std::forward(f)); + default: return execute_many(0, lhs, rhs, std::forward(f)); + } + } +private: + size_t loops_left(size_t idx) const { return (loop_cnt.size() - idx); } + template void execute_few(size_t idx, size_t lhs, size_t rhs, F &&f) const { + if constexpr (N == 0) { + f(lhs, rhs); + } else { + for (size_t i = 0; i < loop_cnt[idx]; ++i, lhs += lhs_stride[idx], rhs += rhs_stride[idx]) { + execute_few(idx + 1, lhs, rhs, std::forward(f)); + } + } + } + template void execute_many(size_t idx, size_t lhs, size_t rhs, F &&f) const { + for (size_t i = 0; i < loop_cnt[idx]; ++i, lhs += lhs_stride[idx], rhs += rhs_stride[idx]) { + if (loops_left(idx + 1) == 3) { + execute_few(idx + 1, lhs, rhs, std::forward(f)); + } else { + execute_many(idx + 1, lhs, rhs, std::forward(f)); + } + } + } +}; + +/** + * Plan for how to join the sparse part (all mapped dimensions) + * between two values. The plan can be made up-front during tensor + * function compilation. + **/ +struct SparseJoinPlan { + enum class Source { LHS, RHS, BOTH }; + std::vector sources; + std::vector lhs_overlap; + std::vector rhs_overlap; + SparseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type); + ~SparseJoinPlan(); +}; + +//----------------------------------------------------------------------------- + +} diff --git a/eval/src/vespa/eval/eval/value.h b/eval/src/vespa/eval/eval/value.h index 86c62d58eca..20923fcd621 100644 --- a/eval/src/vespa/eval/eval/value.h +++ b/eval/src/vespa/eval/eval/value.h @@ -99,6 +99,66 @@ public: static const ValueType &double_type() { return _type; } }; +/** + * Tagging interface used as return type from factories before + * downcasting to actual builder with specialized cell type. + **/ +struct ValueBuilderBase { + virtual ~ValueBuilderBase() {} +}; + +/** + * Interface used to build a value one dense subspace at a + * time. Enables decoupling of what the value should contain from how + * to store the value. + **/ +template +struct ValueBuilder : ValueBuilderBase { + // add a dense subspace for the given address (label for all + // mapped dimensions in canonical order). Note that previously + // returned subspaces will be invalidated when new subspaces are + // added. Also note that adding the same subspace multiple times + // is not allowed. + virtual ArrayRef add_subspace(const std::vector &addr) = 0; + + // Given the ownership of the builder itself, produce the newly + // created value. This means that builders can only be used once, + // it also means values can build themselves. + virtual std::unique_ptr build(std::unique_ptr self) = 0; +}; + +/** + * Factory able to create appropriate value builders. We do not really + * care about the full mathematical type here, but it needs to be + * passed since it is exposed in the value api. The expected number of + * subspaces is also passed since it enables the builder to pre-size + * internal structures appropriately. Note that since we are not able + * to have virtual templated functions we need to cast the created + * builder. With interoperability between all values. + **/ +struct ValueBuilderFactory { + template + std::unique_ptr> create_value_builder(const ValueType &type, + size_t num_mapped_dims_in, size_t subspace_size_in, size_t expected_subspaces) const + { + assert(check_cell_type(type.cell_type())); + auto base = create_value_builder_base(type, num_mapped_dims_in, subspace_size_in, expected_subspaces); + ValueBuilder *builder = dynamic_cast*>(base.get()); + assert(builder); + base.release(); + return std::unique_ptr>(builder); + } + template + std::unique_ptr> create_value_builder(const ValueType &type) const + { + return create_value_builder(type, type.count_mapped_dimensions(), type.dense_subspace_size(), 1); + } + virtual ~ValueBuilderFactory() {} +protected: + virtual std::unique_ptr create_value_builder_base(const ValueType &type, + size_t num_mapped_dims_in, size_t subspace_size_in, size_t expected_subspaces) const = 0; +}; + } VESPA_CAN_SKIP_DESTRUCTION(::vespalib::eval::DoubleValue); -- cgit v1.2.3 From 1ce8c46c6071e52c1367490e810c1211945499e8 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 24 Sep 2020 10:38:12 +0200 Subject: Update javadoc for DocumentAccess and add convencience for outside containers --- .../java/com/yahoo/documentapi/DocumentAccess.java | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java index 7a9818ba4fd..1aa5c4c0df0 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java @@ -5,6 +5,7 @@ import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentTypeManagerConfigurer; import com.yahoo.document.select.parser.ParseException; import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; /** *

This is the starting point of the document api. This api provides @@ -55,13 +56,29 @@ public abstract class DocumentAccess { * while attempting to create such an object, this method will throw an * exception. * - * @deprecated Inject a DocumentManagerConfig and create a MessageBusDocumentAccess from this instead. + * @deprecated DocumentAccess may be injected in containers — otherwise use {@link #createForNonContainer()}. * * @return a running document access object with all default configuration */ @Deprecated(since = "7") public static DocumentAccess createDefault() { - return new com.yahoo.documentapi.messagebus.MessageBusDocumentAccess(); + return new MessageBusDocumentAccess(); + } + + + /** + * This is a convenience method to return a document access object when running + * outside of a Vespa application container, with all default parameter values. + * The client that calls this method is also responsible for shutting the object + * down when done. If an error occurred while attempting to create such an object, + * this method will throw an exception. + * This document access requires new config subscriptions to be set up, which should + * be avoided in application containers, but is suitable for, e.g., CLIs. + * + * @return a running document access object with all default configuration + */ + public static DocumentAccess createForNonContainer() { + return new MessageBusDocumentAccess(); } /** -- cgit v1.2.3 From 83b48e9ebf2060780fd9675e6019fd0083fa2d95 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 24 Sep 2020 11:21:31 +0200 Subject: Count prod deployments as relevant (to not abort) when same targets --- .../yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index d39e0c134fd..709064c8715 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -107,7 +107,9 @@ public class DeploymentTrigger { Map> newJobsToRun = jobs.deploymentStatus(application.get()).jobsToRun(); for (Run run : jobs.active(application.get().id().instance(instanceName))) { if ( ! run.id().type().environment().isManuallyDeployed() - && ! newJobsToRun.getOrDefault(run.id().job(), List.of()).contains(run.versions())) + && newJobsToRun.getOrDefault(run.id().job(), List.of()).stream() + .noneMatch(versions -> versions.targetsMatch(run.versions()) + && versions.sourcesMatchIfPresent(run.versions()))) jobs.abort(run.id()); } } -- cgit v1.2.3 From 8c6369d5c14cc3c7e7504ffdeb14f07566870364 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 24 Sep 2020 11:21:42 +0200 Subject: Increase readability --- .../java/com/yahoo/vespa/hosted/controller/deployment/Versions.java | 4 ++-- .../java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java | 2 +- .../vespa/hosted/provision/persistence/CuratorDatabaseClient.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java index 7a0349d7737..a05b3425a01 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java @@ -58,9 +58,9 @@ public class Versions { /** Returns whether source versions are present and match those of the given job other versions. */ public boolean sourcesMatchIfPresent(Versions versions) { - return (sourcePlatform.filter(version -> ! version.equals(targetPlatform)).isEmpty() || + return (sourcePlatform.map(targetPlatform::equals).orElse(true) || sourcePlatform.equals(versions.sourcePlatform())) && - (sourceApplication.filter(version -> ! version.equals(targetApplication)).isEmpty() || + (sourceApplication.map(targetApplication::equals).orElse(true) || sourceApplication.equals(versions.sourceApplication())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index e0f2f0718ef..e5ee06f81dd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -74,7 +74,7 @@ public class JobRunner extends ControllerMaintainer { /** Advances each of the ready steps for the given run, or marks it as finished, and stashes it. Public for testing. */ public void advance(Run run) { if ( ! run.hasFailed() - && run.start().isBefore(controller().clock().instant().minus(jobTimeout))) { + && controller().clock().instant().isAfter(run.start().plus(jobTimeout))) { jobs.abort(run.id()); advance(jobs.run(run.id()).get()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 504854108b6..1f0a0310378 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -290,7 +290,7 @@ public class CuratorDatabaseClient { } /** - * Returns a particular node, or empty if this noe is not in any of the given states. + * Returns a particular node, or empty if this node is not in any of the given states. * If no states are given this returns the node if it is present in any state. */ public Optional readNode(CuratorDatabase.Session session, String hostname, Node.State ... states) { -- cgit v1.2.3 From 0f507022b657294e37f8c54456602d5c2be73fc0 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 24 Sep 2020 11:41:34 +0200 Subject: Update abi spec --- documentapi/abi-spec.json | 1 + 1 file changed, 1 insertion(+) diff --git a/documentapi/abi-spec.json b/documentapi/abi-spec.json index 6f5e6d66e2a..7a0a5fa0dd7 100644 --- a/documentapi/abi-spec.json +++ b/documentapi/abi-spec.json @@ -73,6 +73,7 @@ ], "methods": [ "public static com.yahoo.documentapi.DocumentAccess createDefault()", + "public static com.yahoo.documentapi.DocumentAccess createForNonContainer()", "protected void (com.yahoo.documentapi.DocumentAccessParams)", "public abstract com.yahoo.documentapi.SyncSession createSyncSession(com.yahoo.documentapi.SyncParameters)", "public abstract com.yahoo.documentapi.AsyncSession createAsyncSession(com.yahoo.documentapi.AsyncParameters)", -- cgit v1.2.3 From 1cd364b2e2d0681a4533ea7307ea3e7c6762fa87 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Thu, 24 Sep 2020 12:15:54 +0200 Subject: Update pattern to new type of error message for invalid client certificate (#14521) --- .../com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java index 9ced933105d..fc9a6fc03be 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java @@ -67,8 +67,8 @@ class SslHandshakeFailedListener implements SslHandshakeListener { // Note: this pattern will match certificates with too late notBefore as well "PKIX path validation failed: java.security.cert.CertPathValidatorException: validity check failed"), INVALID_CLIENT_CERT( - Metrics.SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT, - "PKIX path (building|validation) failed: .+"); + Metrics.SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT, // Includes mismatch of client certificate and private key + "(PKIX path (building|validation) failed: .+)|(Invalid CertificateVerify signature)"); private final String metricName; private final Predicate messageMatcher; -- cgit v1.2.3 From 985e7c38cd848c5f4f3adcdc454bbe62ba817323 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Thu, 24 Sep 2020 10:32:46 +0200 Subject: Add node object cache --- .../provision/persistence/NodeSerializer.java | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 0ce81e49bda..0882bf7ad3a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -1,7 +1,11 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.google.common.collect.ImmutableSet; +import com.google.common.hash.Hashing; +import com.google.common.util.concurrent.UncheckedExecutionException; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; @@ -32,11 +36,13 @@ import com.yahoo.vespa.hosted.provision.node.Reports; import com.yahoo.vespa.hosted.provision.node.Status; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ExecutionException; import java.util.function.UnaryOperator; /** @@ -107,6 +113,12 @@ public class NodeSerializer { // Network port fields private static final String networkPortsKey = "networkPorts"; + // A cache of deserialized Node objects. The cache is keyed on the hash of serialized node data. + // + // Deserializing a Node from slime is expensive, and happens frequently. Node instances that have already been + // deserialized are returned from this cache instead of being deserialized again. + private final Cache cache = CacheBuilder.newBuilder().maximumSize(1000).build(); + // ---------------- Serialization ---------------------------------------------------- public NodeSerializer(NodeFlavors flavors) { @@ -196,7 +208,15 @@ public class NodeSerializer { // ---------------- Deserialization -------------------------------------------------- public Node fromJson(Node.State state, byte[] data) { - return nodeFromSlime(state, SlimeUtils.jsonToSlime(data).get()); + var key = Hashing.sipHash24().newHasher() + .putString(state.name(), StandardCharsets.UTF_8) + .putBytes(data).hash() + .asLong(); + try { + return cache.get(key, () -> nodeFromSlime(state, SlimeUtils.jsonToSlime(data).get())); + } catch (ExecutionException e) { + throw new UncheckedExecutionException(e); + } } private Node nodeFromSlime(Node.State state, Inspector object) { -- cgit v1.2.3 From a4c4d9075fa1a6bd9d23cac3c72ac57071493fcf Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Thu, 24 Sep 2020 12:46:23 +0200 Subject: Store vespa version in ZooKeeper as well Use version in ZooKeeper if configserver-distribute-application-package is true, fallback to version in file. Will always write to ZooKeeper so this can be the only source in the future. This means that if configserver-distribute-application-package is true and version data is found in ZooKeeper, redeployment of appplication packages will only be done on one config server --- .../vespa/config/server/version/VersionState.java | 47 +++++++++++++++++++--- .../config/server/ConfigServerBootstrapTest.java | 19 +++++---- .../config/server/version/VersionStateTest.java | 41 ++++++++++++++++--- 3 files changed, 89 insertions(+), 18 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/version/VersionState.java b/configserver/src/main/java/com/yahoo/vespa/config/server/version/VersionState.java index 641aa31a6b7..a591ca07e5e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/version/VersionState.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/version/VersionState.java @@ -5,45 +5,78 @@ import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import java.util.Optional; /** - * Contains version information for this configserver. + * + * Contains version information for this configserver. Stored both in file system and in ZooKeeper (uses + * data in ZooKeeper if distributeApplicationPackage and data found in ZooKeeper) * * @author Ulf Lilleengen */ public class VersionState { + static final Path versionPath = Path.fromString("/config/v2/vespa_version"); + private final File versionFile; + private final Curator curator; + private final BooleanFlag distributeApplicationPackage; @Inject - public VersionState(ConfigserverConfig config) { - this(new File(Defaults.getDefaults().underVespaHome(config.configServerDBDir()), "vespa_version")); + public VersionState(ConfigserverConfig config, Curator curator, FlagSource flagsource) { + this(new File(Defaults.getDefaults().underVespaHome(config.configServerDBDir()), "vespa_version"), curator, flagsource); } - public VersionState(File versionFile) { + public VersionState(File versionFile, Curator curator, FlagSource flagSource) { this.versionFile = versionFile; + this.curator = curator; + this.distributeApplicationPackage = Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.bindTo(flagSource); } public boolean isUpgraded() { + System.out.println("current version: " + currentVersion() + ", stored version: " + storedVersion()); return currentVersion().compareTo(storedVersion()) > 0; } public void saveNewVersion() { + saveNewVersion(currentVersion().toFullString()); + } + + public void saveNewVersion(String vespaVersion) { + curator.set(versionPath, Utf8.toBytes(vespaVersion)); try (FileWriter writer = new FileWriter(versionFile)) { - writer.write(currentVersion().toFullString()); + writer.write(vespaVersion); } catch (IOException e) { throw new RuntimeException(e); } } public Version storedVersion() { + if (distributeApplicationPackage.value()) { + Optional version = curator.getData(versionPath); + if(version.isPresent()) { + System.out.println("Found version in zk "); + try { + return Version.fromString(Utf8.toString(version.get())); + } catch (Exception e) { + return new Version(0, 0, 0); // Use an old value to signal we don't know + } + } + } try (FileReader reader = new FileReader(versionFile)) { + System.out.println("Found version in file "); return Version.fromString(IOUtils.readAll(reader)); } catch (Exception e) { return new Version(0, 0, 0); // Use an old value to signal we don't know @@ -54,6 +87,10 @@ public class VersionState { return new Version(VespaVersion.major, VespaVersion.minor, VespaVersion.micro); } + File versionFile() { + return versionFile; + } + @Override public String toString() { return String.format("Current version:%s, stored version:%s", currentVersion(), storedVersion()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index bda17faec12..d7009c95ee2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -25,6 +25,7 @@ import com.yahoo.vespa.config.server.rpc.RpcServer; import com.yahoo.vespa.config.server.version.VersionState; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.InMemoryFlagSource; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -56,6 +57,8 @@ import static org.junit.Assert.assertTrue; */ public class ConfigServerBootstrapTest { + private final MockCurator curator = new MockCurator(); + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -66,8 +69,7 @@ public class ConfigServerBootstrapTest { DeployTester tester = new DeployTester(List.of(createHostedModelFactory()), configserverConfig, provisioner); tester.deployApp("src/test/apps/hosted/"); - File versionFile = temporaryFolder.newFile(); - VersionState versionState = new VersionState(versionFile); + VersionState versionState = createVersionState(); assertTrue(versionState.isUpgraded()); RpcServer rpcServer = createRpcServer(configserverConfig); @@ -99,8 +101,7 @@ public class ConfigServerBootstrapTest { DeployTester tester = new DeployTester(List.of(createHostedModelFactory()), configserverConfig, provisioner); tester.deployApp("src/test/apps/hosted/"); - File versionFile = temporaryFolder.newFile(); - VersionState versionState = new VersionState(versionFile); + VersionState versionState = createVersionState(); assertTrue(versionState.isUpgraded()); RpcServer rpcServer = createRpcServer(configserverConfig); @@ -124,8 +125,7 @@ public class ConfigServerBootstrapTest { DeployTester tester = new DeployTester(List.of(createHostedModelFactory()), configserverConfig); tester.deployApp("src/test/apps/hosted/"); - File versionFile = temporaryFolder.newFile(); - VersionState versionState = new VersionState(versionFile); + VersionState versionState = createVersionState(); assertTrue(versionState.isUpgraded()); // Manipulate application package so that it will fail deployment when config server starts @@ -168,8 +168,7 @@ public class ConfigServerBootstrapTest { tester.deployApp("src/test/apps/app/", vespaVersion, Instant.now()); ApplicationId applicationId = tester.applicationId(); - File versionFile = temporaryFolder.newFile(); - VersionState versionState = new VersionState(versionFile); + VersionState versionState = createVersionState(); assertTrue(versionState.isUpgraded()); // Ugly hack, but I see no other way of doing it: @@ -242,6 +241,10 @@ public class ConfigServerBootstrapTest { stateMonitor); } + private VersionState createVersionState() throws IOException { + return new VersionState(temporaryFolder.newFile(), curator, new InMemoryFlagSource()); + } + public static class MockRpcServer extends com.yahoo.vespa.config.server.rpc.MockRpcServer { volatile boolean isRunning = false; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/version/VersionStateTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/version/VersionStateTest.java index 8a1a7fbea20..1340935108b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/version/VersionStateTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/version/VersionStateTest.java @@ -4,12 +4,16 @@ package com.yahoo.vespa.config.server.version; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.io.IOUtils; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; +import java.nio.file.Files; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertFalse; @@ -20,28 +24,49 @@ import static org.junit.Assert.assertTrue; * @author Ulf Lilleengen */ public class VersionStateTest { + InMemoryFlagSource flagSource = new InMemoryFlagSource(); @Rule public TemporaryFolder tempDir = new TemporaryFolder(); + private final MockCurator curator = new MockCurator(); @Test public void upgrade() throws IOException { + upgrade(true); + upgrade(false); + } + + public void upgrade(boolean distributeApplicationPackage) throws IOException { + flagSource.withBooleanFlag(Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.id(), distributeApplicationPackage); Version unknownVersion = new Version(0, 0, 0); - File versionFile = tempDir.newFile(); - VersionState state = new VersionState(versionFile); + + VersionState state = createVersionState(); assertThat(state.storedVersion(), is(unknownVersion)); assertTrue(state.isUpgraded()); state.saveNewVersion(); assertFalse(state.isUpgraded()); - IOUtils.writeFile(versionFile, "badversion", false); + state.saveNewVersion("badversion"); assertThat(state.storedVersion(), is(unknownVersion)); assertTrue(state.isUpgraded()); - IOUtils.writeFile(versionFile, "5.0.0", false); + state.saveNewVersion("5.0.0"); assertThat(state.storedVersion(), is(new Version(5, 0, 0))); assertTrue(state.isUpgraded()); + // Remove zk node, should find version in ZooKeeper + curator.delete(VersionState.versionPath); + assertThat(state.storedVersion(), is(new Version(5, 0, 0))); + assertTrue(state.isUpgraded()); + + // Save new version, remove version in file, should find version in ZooKeeper + state.saveNewVersion("6.0.0"); + if (distributeApplicationPackage) { + Files.delete(state.versionFile().toPath()); + assertThat(state.storedVersion(), is(new Version(6, 0, 0))); + assertTrue(state.isUpgraded()); + } + state.saveNewVersion(); assertThat(state.currentVersion(), is(state.storedVersion())); assertFalse(state.isUpgraded()); @@ -50,7 +75,9 @@ public class VersionStateTest { @Test public void serverdbfile() throws IOException { File dbDir = tempDir.newFolder(); - VersionState state = new VersionState(new ConfigserverConfig(new ConfigserverConfig.Builder().configServerDBDir(dbDir.getAbsolutePath()))); + VersionState state = new VersionState(new ConfigserverConfig.Builder().configServerDBDir(dbDir.getAbsolutePath()).build(), + curator, + new InMemoryFlagSource()); state.saveNewVersion(); File versionFile = new File(dbDir, "vespa_version"); assertTrue(versionFile.exists()); @@ -58,4 +85,8 @@ public class VersionStateTest { assertThat(stored, is(state.currentVersion())); } + private VersionState createVersionState() throws IOException { + return new VersionState(tempDir.newFile(), curator, flagSource); + } + } -- cgit v1.2.3 From 87063d6525aace2042378d94bc1616a35c69f761 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Thu, 24 Sep 2020 12:56:04 +0200 Subject: Report node object cache metrics --- .../provision/maintenance/MetricsReporter.java | 9 ++++++ .../persistence/CuratorDatabaseClient.java | 4 +++ .../provision/persistence/NodeSerializer.java | 33 ++++++++++++++++++++++ .../provision/maintenance/MetricsReporterTest.java | 4 +++ 4 files changed, 50 insertions(+) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index e0d7dc5f19e..d15bc9ede90 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.History; +import com.yahoo.vespa.hosted.provision.persistence.NodeSerializer; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ServiceMonitor; @@ -68,9 +69,17 @@ public class MetricsReporter extends NodeRepositoryMaintainer { updateMaintenanceMetrics(); updateDockerMetrics(nodes); updateTenantUsageMetrics(nodes); + updateNodeSerializerCacheMetrics(); return true; } + private void updateNodeSerializerCacheMetrics() { + NodeSerializer.CacheStats cacheStats = nodeRepository().database().nodeSerializerCacheStats(); + metric.set("cache.nodeObject.hitRate", cacheStats.hitRate(), null); + metric.set("cache.nodeObject.evictionCount", cacheStats.evictionCount(), null); + metric.set("cache.nodeObject.size", cacheStats.size(), null); + } + private void updateMaintenanceMetrics() { metric.set("hostedVespa.pendingRedeployments", pendingRedeploymentsSupplier.get(), null); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 504854108b6..ed3c345a6bd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -588,6 +588,10 @@ public class CuratorDatabaseClient { .collect(Collectors.toList()); } + public NodeSerializer.CacheStats nodeSerializerCacheStats() { + return nodeSerializer.cacheStats(); + } + private Optional read(Path path, Function mapper) { return db.getData(path).filter(data -> data.length > 0).map(mapper); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 0882bf7ad3a..ff2c1bbcf86 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -136,6 +136,12 @@ public class NodeSerializer { } } + /** Returns cache statistics for this serializer */ + public CacheStats cacheStats() { + com.google.common.cache.CacheStats cacheStats = cache.stats(); + return new CacheStats(cacheStats.hitRate(), cacheStats.evictionCount(), cache.size()); + } + private void toSlime(Node node, Cursor object) { object.setString(hostnameKey, node.hostname()); toSlime(node.ipConfig().primary(), object.setArray(ipAddressesKey), IP.Config::require); @@ -476,4 +482,31 @@ public class NodeSerializer { throw new IllegalArgumentException("Serialized form of '" + type + "' not defined"); } + /** Cache statistics for this serializer's object cache */ + public static class CacheStats { + + private final double hitRate; + private final long evictionCount; + private final long size; + + public CacheStats(double hitRate, long evictionCount, long size) { + this.hitRate = hitRate; + this.evictionCount = evictionCount; + this.size = size; + } + + public double hitRate() { + return hitRate; + } + + public long evictionCount() { + return evictionCount; + } + + public long size() { + return size; + } + + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 6897a293525..1c5149702e7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -123,6 +123,10 @@ public class MetricsReporterTest { expectedMetrics.put("suspendedSeconds", 123L); expectedMetrics.put("numberOfServices", 0L); + expectedMetrics.put("cache.nodeObject.hitRate", 1.0D); + expectedMetrics.put("cache.nodeObject.evictionCount", 0L); + expectedMetrics.put("cache.nodeObject.size", 2L); + ManualClock clock = new ManualClock(Instant.ofEpochSecond(124)); Orchestrator orchestrator = mock(Orchestrator.class); -- cgit v1.2.3 From 7611e6c6ae1562679545ed6cf590d68dee5fc474 Mon Sep 17 00:00:00 2001 From: Tor Brede Vekterli Date: Thu, 24 Sep 2020 11:14:06 +0000 Subject: Add StatBucket storage protocol (de-)serialization support Adds both protobuf schema and type ID handling to protocol codec. Old protocol versions are not expected to handle this message, as StatBucket will run over the Document protocol instead of the Storage protocol in such cases. --- .../src/tests/mbusprot/storageprotocoltest.cpp | 19 ++++++++++++++ .../src/vespa/storageapi/mbusprot/CMakeLists.txt | 1 + .../storageapi/mbusprot/protobuf/inspect.proto | 19 ++++++++++++++ .../vespa/storageapi/mbusprot/protobuf_includes.h | 1 + .../storageapi/mbusprot/protocolserialization.cpp | 11 ++++++++ .../storageapi/mbusprot/protocolserialization.h | 6 +++++ .../mbusprot/protocolserialization4_2.cpp | 18 ++++++++++++++ .../storageapi/mbusprot/protocolserialization4_2.h | 4 +++ .../storageapi/mbusprot/protocolserialization7.cpp | 29 ++++++++++++++++++++++ .../storageapi/mbusprot/protocolserialization7.h | 6 +++++ storageapi/src/vespa/storageapi/message/stat.h | 12 ++++----- 11 files changed, 120 insertions(+), 6 deletions(-) create mode 100644 storageapi/src/vespa/storageapi/mbusprot/protobuf/inspect.proto diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp index 37159ab0011..636f9b1f701 100644 --- a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp +++ b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -567,6 +568,24 @@ TEST_P(StorageProtocolTest, remove_location) { } } +TEST_P(StorageProtocolTest, stat_bucket) { + if (GetParam().getMajor() < 7) { + return; // Only available for protobuf-backed protocol version. + } + auto cmd = std::make_shared(_bucket, "id.group == 'mygroup'"); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ("id.group == 'mygroup'", cmd2->getDocumentSelection()); + EXPECT_EQ(_bucket, cmd2->getBucket()); + + auto reply = std::make_shared(*cmd2, "neat bucket info goes here"); + reply->remapBucketId(_dummy_remap_bucket); + auto reply2 = copyReply(reply); + EXPECT_EQ(reply2->getResults(), "neat bucket info goes here"); + EXPECT_TRUE(reply2->hasBeenRemapped()); + EXPECT_EQ(_dummy_remap_bucket, reply2->getBucketId()); + EXPECT_EQ(_bucket_id, reply2->getOriginalBucketId()); +} + TEST_P(StorageProtocolTest, create_visitor) { std::vector buckets; buckets.push_back(document::BucketId(16, 1)); diff --git a/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt b/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt index b749844775d..ce5294a7470 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt +++ b/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt @@ -4,6 +4,7 @@ find_package(Protobuf REQUIRED) PROTOBUF_GENERATE_CPP(storageapi_PROTOBUF_SRCS storageapi_PROTOBUF_HDRS protobuf/common.proto protobuf/feed.proto + protobuf/inspect.proto protobuf/visiting.proto protobuf/maintenance.proto) diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/inspect.proto b/storageapi/src/vespa/storageapi/mbusprot/protobuf/inspect.proto new file mode 100644 index 00000000000..1bf833e4bf6 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protobuf/inspect.proto @@ -0,0 +1,19 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +syntax = "proto3"; + +option cc_enable_arenas = true; + +package storage.mbusprot.protobuf; + +import "common.proto"; + +message StatBucketRequest { + Bucket bucket = 1; + bytes document_selection = 2; +} + +message StatBucketResponse { + BucketId remapped_bucket_id = 1; + bytes results = 2; +} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h b/storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h index ed7fbc0bdf9..289e5dc355c 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h +++ b/storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h @@ -9,6 +9,7 @@ #endif #include "feed.pb.h" +#include "inspect.pb.h" #include "visiting.pb.h" #include "maintenance.pb.h" diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp index 917b60c50c3..a53bd415c8e 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include @@ -120,6 +121,12 @@ ProtocolSerialization::encode(const api::StorageMessage& msg) const case api::MessageType::VISITOR_DESTROY_REPLY_ID: onEncode(buf, static_cast(msg)); break; + case api::MessageType::STATBUCKET_ID: + onEncode(buf, static_cast(msg)); + break; + case api::MessageType::STATBUCKET_REPLY_ID: + onEncode(buf, static_cast(msg)); + break; case api::MessageType::REMOVELOCATION_ID: onEncode(buf, static_cast(msg)); break; @@ -191,6 +198,8 @@ ProtocolSerialization::decodeCommand(mbus::BlobRef data) const cmd = onDecodeCreateVisitorCommand(buf); break; case api::MessageType::VISITOR_DESTROY_ID: cmd = onDecodeDestroyVisitorCommand(buf); break; + case api::MessageType::STATBUCKET_ID: + cmd = onDecodeStatBucketCommand(buf); break; case api::MessageType::REMOVELOCATION_ID: cmd = onDecodeRemoveLocationCommand(buf); break; case api::MessageType::SETBUCKETSTATE_ID: @@ -253,6 +262,8 @@ ProtocolSerialization::decodeReply(mbus::BlobRef data, const api::StorageCommand reply = onDecodeCreateVisitorReply(cmd, buf); break; case api::MessageType::VISITOR_DESTROY_REPLY_ID: reply = onDecodeDestroyVisitorReply(cmd, buf); break; + case api::MessageType::STATBUCKET_REPLY_ID: + reply = onDecodeStatBucketReply(cmd, buf); break; case api::MessageType::REMOVELOCATION_REPLY_ID: reply = onDecodeRemoveLocationReply(cmd, buf); break; case api::MessageType::SETBUCKETSTATE_REPLY_ID: diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h index a57627b9ba9..569ff99c11f 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h @@ -43,6 +43,8 @@ class NotifyBucketChangeCommand; class NotifyBucketChangeReply; class SplitBucketCommand; class SplitBucketReply; +class StatBucketCommand; +class StatBucketReply; class JoinBucketsCommand; class JoinBucketsReply; class SetBucketStateCommand; @@ -111,6 +113,8 @@ protected: virtual void onEncode(GBBuf&, const api::DestroyVisitorReply&) const = 0; virtual void onEncode(GBBuf&, const api::RemoveLocationCommand&) const = 0; virtual void onEncode(GBBuf&, const api::RemoveLocationReply&) const = 0; + virtual void onEncode(GBBuf&, const api::StatBucketCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::StatBucketReply&) const = 0; virtual SCmd::UP onDecodePutCommand(BBuf&) const = 0; virtual SRep::UP onDecodePutReply(const SCmd&, BBuf&) const = 0; @@ -148,6 +152,8 @@ protected: virtual SRep::UP onDecodeDestroyVisitorReply(const SCmd&, BBuf&) const = 0; virtual SCmd::UP onDecodeRemoveLocationCommand(BBuf&) const = 0; virtual SRep::UP onDecodeRemoveLocationReply(const SCmd&, BBuf&) const = 0; + virtual SCmd::UP onDecodeStatBucketCommand(BBuf&) const = 0; + virtual SRep::UP onDecodeStatBucketReply(const SCmd&, BBuf&) const = 0; }; } diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp index 8deb689d3a8..13fba8b8508 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp @@ -488,6 +488,24 @@ ProtocolSerialization4_2::onDecodeRemoveLocationReply(const SCmd& cmd, BBuf& buf return msg; } +void ProtocolSerialization4_2::onEncode(GBBuf&, const api::StatBucketCommand&) const { + throw vespalib::IllegalStateException("StatBucketCommand not expected for legacy protocol version", VESPA_STRLOC); +} + +api::StorageCommand::UP +ProtocolSerialization4_2::onDecodeStatBucketCommand(BBuf&) const { + throw vespalib::IllegalStateException("StatBucketCommand not expected for legacy protocol version", VESPA_STRLOC); +} + +void ProtocolSerialization4_2::onEncode(GBBuf&, const api::StatBucketReply&) const { + throw vespalib::IllegalStateException("StatBucketReply not expected for legacy protocol version", VESPA_STRLOC); +} + +api::StorageReply::UP +ProtocolSerialization4_2::onDecodeStatBucketReply(const SCmd&, BBuf&) const { + throw vespalib::IllegalStateException("StatBucketReply not expected for legacy protocol version", VESPA_STRLOC); +} + // Utility functions for serialization void diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h index e4ab36dc989..3ae1770f3c2 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h @@ -26,6 +26,8 @@ protected: void onEncode(GBBuf&, const api::DestroyVisitorReply&) const override; void onEncode(GBBuf&, const api::RemoveLocationCommand&) const override; void onEncode(GBBuf&, const api::RemoveLocationReply&) const override; + void onEncode(GBBuf&, const api::StatBucketCommand&) const override; + void onEncode(GBBuf&, const api::StatBucketReply&) const override; // Not supported on 4.2, but implemented here for simplicity. void onEncode(GBBuf&, const api::SetBucketStateCommand&) const override; @@ -56,6 +58,8 @@ protected: SRep::UP onDecodeDestroyVisitorReply(const SCmd&, BBuf&) const override; SCmd::UP onDecodeRemoveLocationCommand(BBuf&) const override; SRep::UP onDecodeRemoveLocationReply(const SCmd&, BBuf&) const override; + SCmd::UP onDecodeStatBucketCommand(BBuf&) const override; + SRep::UP onDecodeStatBucketReply(const SCmd&, BBuf&) const override; virtual void onDecodeBucketInfoCommand(BBuf&, api::BucketInfoCommand&) const; virtual void onDecodeBucketInfoReply(BBuf&, api::BucketInfoReply&) const = 0; diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp index 8ea946eede4..166925382bd 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp @@ -10,6 +10,7 @@ #include #include #include +#include namespace storage::mbusprot { @@ -1313,4 +1314,32 @@ api::StorageReply::UP ProtocolSerialization7::onDecodeDestroyVisitorReply(const }); } +// ----------------------------------------------------------------- +// StatBucket +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::StatBucketCommand& msg) const { + encode_bucket_request(buf, msg, [&](auto& req) { + req.set_document_selection(msg.getDocumentSelection().data(), msg.getDocumentSelection().size()); + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::StatBucketReply& msg) const { + encode_bucket_response(buf, msg, [&](auto& res) { + res.set_results(msg.getResults().data(), msg.getResults().size()); + }); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeStatBucketCommand(BBuf& buf) const { + return decode_bucket_request(buf, [&](auto& req, auto& bucket) { + return std::make_unique(bucket, req.document_selection()); + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeStatBucketReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_response(buf, [&](auto& res) { + return std::make_unique(static_cast(cmd), res.results()); + }); +} + } // storage::mbusprot diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h index bb7f0308efa..e1d08691bc1 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h @@ -129,6 +129,12 @@ public: SCmd::UP onDecodeRemoveLocationCommand(BBuf&) const override; SRep::UP onDecodeRemoveLocationReply(const SCmd&, BBuf&) const override; + // StatBucket + void onEncode(GBBuf&, const api::StatBucketCommand&) const override; + void onEncode(GBBuf&, const api::StatBucketReply&) const override; + SCmd::UP onDecodeStatBucketCommand(BBuf&) const override; + SRep::UP onDecodeStatBucketReply(const SCmd&, BBuf&) const override; + private: template std::unique_ptr decode_request(document::ByteBuffer& in_buf, Func&& f) const; diff --git a/storageapi/src/vespa/storageapi/message/stat.h b/storageapi/src/vespa/storageapi/message/stat.h index 9020d16622a..a7d9a30ca6e 100644 --- a/storageapi/src/vespa/storageapi/message/stat.h +++ b/storageapi/src/vespa/storageapi/message/stat.h @@ -23,7 +23,7 @@ private: public: StatBucketCommand(const document::Bucket &bucket, vespalib::stringref documentSelection); - ~StatBucketCommand(); + ~StatBucketCommand() override; const vespalib::string& getDocumentSelection() const { return _docSelection; } void print(std::ostream& out, bool verbose, const std::string& indent) const override; @@ -33,8 +33,8 @@ public: class StatBucketReply : public BucketReply { vespalib::string _results; public: - StatBucketReply(const StatBucketCommand&, vespalib::stringref results = ""); - const vespalib::string& getResults() { return _results; } + explicit StatBucketReply(const StatBucketCommand&, vespalib::stringref results = ""); + const vespalib::string& getResults() const noexcept { return _results; } void print(std::ostream& out, bool verbose, const std::string& indent) const override; DECLARE_STORAGEREPLY(StatBucketReply, onStatBucketReply) }; @@ -51,7 +51,7 @@ public: */ class GetBucketListCommand : public BucketCommand { public: - GetBucketListCommand(const document::Bucket &bucket); + explicit GetBucketListCommand(const document::Bucket &bucket); void print(std::ostream& out, bool verbose, const std::string& indent) const override; DECLARE_STORAGECOMMAND(GetBucketListCommand, onGetBucketList); }; @@ -78,8 +78,8 @@ private: std::vector _buckets; public: - GetBucketListReply(const GetBucketListCommand&); - ~GetBucketListReply(); + explicit GetBucketListReply(const GetBucketListCommand&); + ~GetBucketListReply() override; std::vector& getBuckets() { return _buckets; } const std::vector& getBuckets() const { return _buckets; } void print(std::ostream& out, bool verbose, const std::string& indent) const override; -- cgit v1.2.3 From f8f6fa6b4ceed96f2fcfb98e7fcce0bfd7bdf0c6 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Thu, 24 Sep 2020 13:36:47 +0200 Subject: No need to shutdown/close more than once --- .../main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java index dfd81341e5d..095bde76c39 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java @@ -182,9 +182,6 @@ class RpcConfigSourceClient implements ConfigSourceClient, Runnable { shutdownSourceConnections(); delayedResponsesFuture.cancel(true); delayedResponsesScheduler.shutdownNow(); - nextConfigFuture.cancel(true); - nextConfigScheduler.shutdownNow(); - requester.close(); supervisor.transport().shutdown().join(); } @@ -195,6 +192,7 @@ class RpcConfigSourceClient implements ConfigSourceClient, Runnable { public void shutdownSourceConnections() { activeSubscribers.values().forEach(Subscriber::cancel); activeSubscribers.clear(); + nextConfigFuture.cancel(true); nextConfigScheduler.shutdownNow(); requester.close(); } -- cgit v1.2.3 From 19db8f60fa4a840573ece24a3a8826b5f769741c Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Thu, 24 Sep 2020 13:40:12 +0200 Subject: Rename vespa-spi-feed-bm to vespa-feed-bm. --- searchcore/CMakeLists.txt | 2 +- searchcore/src/apps/vespa-feed-bm/.gitignore | 1 + searchcore/src/apps/vespa-feed-bm/CMakeLists.txt | 27 + .../src/apps/vespa-feed-bm/vespa_feed_bm.cpp | 1172 ++++++++++++++++++++ searchcore/src/apps/vespa-spi-feed-bm/.gitignore | 1 - .../src/apps/vespa-spi-feed-bm/CMakeLists.txt | 27 - .../apps/vespa-spi-feed-bm/vespa_spi_feed_bm.cpp | 1172 -------------------- 7 files changed, 1201 insertions(+), 1201 deletions(-) create mode 100644 searchcore/src/apps/vespa-feed-bm/.gitignore create mode 100644 searchcore/src/apps/vespa-feed-bm/CMakeLists.txt create mode 100644 searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp delete mode 100644 searchcore/src/apps/vespa-spi-feed-bm/.gitignore delete mode 100644 searchcore/src/apps/vespa-spi-feed-bm/CMakeLists.txt delete mode 100644 searchcore/src/apps/vespa-spi-feed-bm/vespa_spi_feed_bm.cpp diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index f29712e7a5f..f98a3c87a2e 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -46,9 +46,9 @@ vespa_define_module( src/apps/tests src/apps/verify_ranksetup src/apps/vespa-dump-feed + src/apps/vespa-feed-bm src/apps/vespa-gen-testdocs src/apps/vespa-proton-cmd - src/apps/vespa-spi-feed-bm src/apps/vespa-transactionlog-inspect TESTS diff --git a/searchcore/src/apps/vespa-feed-bm/.gitignore b/searchcore/src/apps/vespa-feed-bm/.gitignore new file mode 100644 index 00000000000..0dc27e95ea8 --- /dev/null +++ b/searchcore/src/apps/vespa-feed-bm/.gitignore @@ -0,0 +1 @@ +vespa-feed-bm diff --git a/searchcore/src/apps/vespa-feed-bm/CMakeLists.txt b/searchcore/src/apps/vespa-feed-bm/CMakeLists.txt new file mode 100644 index 00000000000..9fa47c77b03 --- /dev/null +++ b/searchcore/src/apps/vespa-feed-bm/CMakeLists.txt @@ -0,0 +1,27 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_vespa_feed_bm_app + SOURCES + vespa_feed_bm.cpp + OUTPUT_NAME vespa-feed-bm + DEPENDS + searchcore_server + searchcore_initializer + searchcore_reprocessing + searchcore_index + searchcore_persistenceengine + searchcore_docsummary + searchcore_feedoperation + searchcore_matching + searchcore_attribute + searchcore_documentmetastore + searchcore_bucketdb + searchcore_flushengine + searchcore_pcommon + searchcore_grouping + searchcore_proton_metrics + searchcore_fconfig + storageserver_storageapp + messagebus_messagebus-test + messagebus + searchlib_searchlib_uca +) diff --git a/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp b/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp new file mode 100644 index 00000000000..85b310ccbad --- /dev/null +++ b/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp @@ -0,0 +1,1172 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +LOG_SETUP("vespa-feed-bm"); + +using namespace config; +using namespace proton; +using namespace cloud::config::filedistribution; +using namespace vespa::config::search::core; +using namespace vespa::config::search::summary; +using namespace vespa::config::search; +using namespace std::chrono_literals; +using vespa::config::content::core::BucketspacesConfig; +using vespa::config::content::core::BucketspacesConfigBuilder; +using vespa::config::content::StorDistributionConfigBuilder; +using vespa::config::content::StorFilestorConfigBuilder; +using vespa::config::content::PersistenceConfigBuilder; +using vespa::config::content::core::StorBouncerConfigBuilder; +using vespa::config::content::core::StorCommunicationmanagerConfigBuilder; +using vespa::config::content::core::StorBucketInitConfigBuilder; +using vespa::config::content::core::StorOpsloggerConfigBuilder; +using vespa::config::content::core::StorPrioritymappingConfigBuilder; +using vespa::config::content::LoadTypeConfigBuilder; +using vespa::config::content::UpgradingConfigBuilder; +using vespa::config::content::core::StorServerConfigBuilder; +using vespa::config::content::core::StorStatusConfigBuilder; +using vespa::config::content::core::StorVisitorConfigBuilder; +using metrics::MetricsmanagerConfigBuilder; +using cloud::config::SlobroksConfigBuilder; +using messagebus::MessagebusConfigBuilder; + +using config::ConfigContext; +using config::ConfigUri; +using config::ConfigSet; +using config::IConfigContext; +using document::AssignValueUpdate; +using document::BucketId; +using document::BucketSpace; +using document::Document; +using document::DocumentId; +using document::DocumentType; +using document::DocumentTypeRepo; +using document::DocumentTypeRepoFactory; +using document::DocumenttypesConfig; +using document::DocumenttypesConfigBuilder; +using document::DocumentUpdate; +using document::Field; +using document::FieldUpdate; +using document::IntFieldValue; +using document::test::makeBucketSpace; +using documentapi::LoadTypeSet; +using search::TuneFileDocumentDB; +using search::index::DummyFileHeaderContext; +using search::index::Schema; +using search::index::SchemaBuilder; +using search::transactionlog::TransLogServer; +using storage::rpc::MessageCodecProvider; +using storage::rpc::SharedRpcResources; +using storage::rpc::StorageApiRpcService; +using storage::spi::Bucket; +using storage::spi::PartitionId; +using storage::spi::PersistenceProvider; +using storage::spi::Priority; +using storage::spi::Timestamp; +using storage::spi::Trace; +using vespalib::makeLambdaTask; + +using DocumentDBMap = std::map>; + +namespace { + +storage::spi::LoadType default_load_type(0, "default"); + +vespalib::string base_dir = "testdb"; + +std::shared_ptr make_document_type() { + using Struct = document::config_builder::Struct; + using DataType = document::DataType; + document::config_builder::DocumenttypesConfigBuilderHelper builder; + builder.document(42, "test", Struct("test.header").addField("int", DataType::T_INT), Struct("test.body")); + return std::make_shared(builder.config()); +} + +std::shared_ptr make_attributes_config() { + AttributesConfigBuilder builder; + AttributesConfig::Attribute attribute; + attribute.name = "int"; + attribute.datatype = AttributesConfig::Attribute::Datatype::INT32; + builder.attribute.emplace_back(attribute); + return std::make_shared(builder); +} + +std::shared_ptr make_document_db_config(std::shared_ptr document_types, std::shared_ptr repo, const DocTypeName& doc_type_name) +{ + auto indexschema = std::make_shared(); + auto attributes = make_attributes_config(); + auto summary = std::make_shared(); + std::shared_ptr schema(new Schema()); + SchemaBuilder::build(*indexschema, *schema); + SchemaBuilder::build(*attributes, *schema); + SchemaBuilder::build(*summary, *schema); + return std::make_shared( + 1, + std::make_shared(), + std::make_shared(), + std::make_shared(), + indexschema, + attributes, + summary, + std::make_shared(), + std::make_shared(), + document_types, + repo, + std::make_shared(), + std::make_shared(), + schema, + std::make_shared(), + search::LogDocumentStore::Config(), + "client", + doc_type_name.getName()); +} + +class MyPersistenceEngineOwner : public IPersistenceEngineOwner +{ + void setClusterState(BucketSpace, const storage::spi::ClusterState &) override { } +}; + +struct MyResourceWriteFilter : public IResourceWriteFilter +{ + bool acceptWriteOperation() const override { return true; } + State getAcceptState() const override { return IResourceWriteFilter::State(); } +}; + +class MyPendingTracker { + uint32_t _pending; + uint32_t _limit; + std::mutex _mutex; + std::condition_variable _cond; + +public: + MyPendingTracker(uint32_t limit) + : _pending(0u), + _limit(limit), + _mutex(), + _cond() + { + } + + ~MyPendingTracker() + { + drain(); + } + + void release() { + std::unique_lock guard(_mutex); + --_pending; + if (_pending < _limit) { + _cond.notify_all(); + } + //LOG(info, "release, pending is now %u", _pending); + } + void retain() { + std::unique_lock guard(_mutex); + while (_pending >= _limit) { + _cond.wait(guard); + } + ++_pending; + //LOG(info, "retain, pending is now %u", _pending); + } + + void drain() { + std::unique_lock guard(_mutex); + while (_pending > 0) { + _cond.wait(guard); + } + } +}; + +class MyOperationComplete : public storage::spi::OperationComplete +{ + MyPendingTracker& _tracker; +public: + MyOperationComplete(MyPendingTracker &tracker); + ~MyOperationComplete(); + void onComplete(std::unique_ptr result) override; + void addResultHandler(const storage::spi::ResultHandler* resultHandler) override; +}; + +MyOperationComplete::MyOperationComplete(MyPendingTracker& tracker) + : _tracker(tracker) +{ + _tracker.retain(); +} + +MyOperationComplete::~MyOperationComplete() +{ + _tracker.release(); +} + +void +MyOperationComplete::onComplete(std::unique_ptr result) +{ + (void) result; +} + +void +MyOperationComplete::addResultHandler(const storage::spi::ResultHandler * resultHandler) +{ + (void) resultHandler; +} + +class BMRange +{ + uint32_t _start; + uint32_t _end; +public: + BMRange(uint32_t start_in, uint32_t end_in) + : _start(start_in), + _end(end_in) + { + } + uint32_t get_start() const { return _start; } + uint32_t get_end() const { return _end; } +}; + +class BMParams { + uint32_t _documents; + uint32_t _threads; + uint32_t _put_passes; + uint32_t _update_passes; + uint32_t _remove_passes; + uint32_t _rpc_network_threads; + bool _enable_service_layer; + uint32_t get_start(uint32_t thread_id) const { + return (_documents / _threads) * thread_id + std::min(thread_id, _documents % _threads); + } +public: + BMParams() + : _documents(160000), + _threads(32), + _put_passes(2), + _update_passes(1), + _remove_passes(2), + _rpc_network_threads(1), + _enable_service_layer(false) + { + } + BMRange get_range(uint32_t thread_id) const { + return BMRange(get_start(thread_id), get_start(thread_id + 1)); + } + uint32_t get_documents() const { return _documents; } + uint32_t get_threads() const { return _threads; } + uint32_t get_put_passes() const { return _put_passes; } + uint32_t get_update_passes() const { return _update_passes; } + uint32_t get_remove_passes() const { return _remove_passes; } + uint32_t get_rpc_network_threads() const { return _rpc_network_threads; } + bool get_enable_service_layer() const { return _enable_service_layer; } + void set_documents(uint32_t documents_in) { _documents = documents_in; } + void set_threads(uint32_t threads_in) { _threads = threads_in; } + void set_put_passes(uint32_t put_passes_in) { _put_passes = put_passes_in; } + void set_update_passes(uint32_t update_passes_in) { _update_passes = update_passes_in; } + void set_remove_passes(uint32_t remove_passes_in) { _remove_passes = remove_passes_in; } + void set_rpc_network_threads(uint32_t threads_in) { _rpc_network_threads = threads_in; } + void set_enable_service_layer(bool enable_service_layer_in) { _enable_service_layer = enable_service_layer_in; } + bool check() const; +}; + +bool +BMParams::check() const +{ + if (_threads < 1) { + std::cerr << "Too few threads: " << _threads << std::endl; + return false; + } + if (_threads > 256) { + std::cerr << "Too many threads: " << _threads << std::endl; + return false; + } + if (_documents < _threads) { + std::cerr << "Too few documents: " << _documents << std::endl; + return false; + } + if (_put_passes < 1) { + std::cerr << "Put passes too low: " << _put_passes << std::endl; + return false; + } + if (_rpc_network_threads < 1) { + std::cerr << "Too few rpc network threads: " << _rpc_network_threads << std::endl; + return false; + } + return true; +} + + +class MyServiceLayerProcess : public storage::ServiceLayerProcess { + PersistenceProvider& _provider; + +public: + MyServiceLayerProcess(const config::ConfigUri & configUri, + PersistenceProvider &provider); + ~MyServiceLayerProcess() override { shutdown(); } + + void shutdown() override; + void setupProvider() override; + PersistenceProvider& getProvider() override; +}; + +MyServiceLayerProcess::MyServiceLayerProcess(const config::ConfigUri & configUri, + PersistenceProvider &provider) + : ServiceLayerProcess(configUri), + _provider(provider) +{ +} + +void +MyServiceLayerProcess::shutdown() +{ + ServiceLayerProcess::shutdown(); +} + +void +MyServiceLayerProcess::setupProvider() +{ +} + +PersistenceProvider& +MyServiceLayerProcess::getProvider() +{ + return _provider; +} + +struct MyStorageConfig +{ + vespalib::string config_id; + DocumenttypesConfigBuilder documenttypes; + PersistenceConfigBuilder persistence; + StorDistributionConfigBuilder stor_distribution; + StorFilestorConfigBuilder stor_filestor; + StorBouncerConfigBuilder stor_bouncer; + StorCommunicationmanagerConfigBuilder stor_communicationmanager; + StorBucketInitConfigBuilder stor_bucket_init; + StorOpsloggerConfigBuilder stor_opslogger; + StorPrioritymappingConfigBuilder stor_prioritymapping; + UpgradingConfigBuilder upgrading; + StorServerConfigBuilder stor_server; + StorStatusConfigBuilder stor_status; + StorVisitorConfigBuilder stor_visitor; + BucketspacesConfigBuilder bucketspaces; + LoadTypeConfigBuilder load_type; + MetricsmanagerConfigBuilder metricsmanager; + SlobroksConfigBuilder slobroks; + MessagebusConfigBuilder messagebus; + + MyStorageConfig(const vespalib::string& config_id_in, const DocumenttypesConfig& documenttypes_in, int slobrok_port, int status_port, uint32_t rpc_network_threads) + : config_id(config_id_in), + documenttypes(documenttypes_in), + persistence(), + stor_distribution(), + stor_filestor(), + stor_bouncer(), + stor_communicationmanager(), + stor_bucket_init(), + stor_opslogger(), + stor_prioritymapping(), + upgrading(), + stor_server(), + stor_status(), + stor_visitor(), + bucketspaces(), + load_type(), + metricsmanager(), + slobroks(), + messagebus() + { + { + auto &dc = stor_distribution; + { + StorDistributionConfigBuilder::Group group; + { + StorDistributionConfigBuilder::Group::Nodes node; + node.index = 0; + group.nodes.push_back(std::move(node)); + } + group.index = "invalid"; + group.name = "invalid"; + group.capacity = 1.0; + group.partitions = ""; + dc.group.push_back(std::move(group)); + } + dc.redundancy = 1; + dc.readyCopies = 1; + } + stor_server.rootFolder = "storage"; + { + SlobroksConfigBuilder::Slobrok slobrok; + slobrok.connectionspec = vespalib::make_string("tcp/localhost:%d", slobrok_port); + slobroks.slobrok.push_back(std::move(slobrok)); + } + stor_communicationmanager.useDirectStorageapiRpc = true; + stor_communicationmanager.rpc.numNetworkThreads = rpc_network_threads; + stor_status.httpport = status_port; + } + + ~MyStorageConfig(); + + void add_builders(ConfigSet &set) { + set.addBuilder(config_id, &documenttypes); + set.addBuilder(config_id, &persistence); + set.addBuilder(config_id, &stor_distribution); + set.addBuilder(config_id, &stor_filestor); + set.addBuilder(config_id, &stor_bouncer); + set.addBuilder(config_id, &stor_communicationmanager); + set.addBuilder(config_id, &stor_bucket_init); + set.addBuilder(config_id, &stor_opslogger); + set.addBuilder(config_id, &stor_prioritymapping); + set.addBuilder(config_id, &upgrading); + set.addBuilder(config_id, &stor_server); + set.addBuilder(config_id, &stor_status); + set.addBuilder(config_id, &stor_visitor); + set.addBuilder(config_id, &bucketspaces); + set.addBuilder(config_id, &load_type); + set.addBuilder(config_id, &metricsmanager); + set.addBuilder(config_id, &slobroks); + set.addBuilder(config_id, &messagebus); + } +}; + +MyStorageConfig::~MyStorageConfig() = default; + +struct MyRpcClientConfig { + vespalib::string config_id; + SlobroksConfigBuilder slobroks; + + MyRpcClientConfig(const vespalib::string &config_id_in, int slobrok_port) + : config_id(config_id_in), + slobroks() + { + { + SlobroksConfigBuilder::Slobrok slobrok; + slobrok.connectionspec = vespalib::make_string("tcp/localhost:%d", slobrok_port); + slobroks.slobrok.push_back(std::move(slobrok)); + } + } + ~MyRpcClientConfig(); + + void add_builders(ConfigSet &set) { + set.addBuilder(config_id, &slobroks); + } +}; + +MyRpcClientConfig::~MyRpcClientConfig() = default; + +class MyMessageDispatcher : public storage::MessageDispatcher +{ + std::mutex _mutex; + vespalib::hash_map _pending; +public: + MyMessageDispatcher() + : storage::MessageDispatcher(), + _mutex(), + _pending() + { + } + ~MyMessageDispatcher() override; + void dispatch_sync(std::shared_ptr msg) override { + release(msg->getMsgId()); + } + void dispatch_async(std::shared_ptr msg) override { + release(msg->getMsgId()); + } + void retain(uint64_t msg_id, MyPendingTracker &tracker) { + tracker.retain(); + std::lock_guard lock(_mutex); + _pending.insert(std::make_pair(msg_id, &tracker)); + } + void release(uint64_t msg_id) { + MyPendingTracker *tracker = nullptr; + { + std::lock_guard lock(_mutex); + auto itr = _pending.find(msg_id); + assert(itr != _pending.end()); + tracker = itr->second; + _pending.erase(itr); + } + tracker->release(); + } +}; + +MyMessageDispatcher::~MyMessageDispatcher() +{ + std::lock_guard lock(_mutex); + assert(_pending.empty()); +} + +FRT_RPCRequest *make_set_cluster_state_request() { + storage::lib::ClusterStateBundle bundle(storage::lib::ClusterState("version:2 distributor:1 storage:1")); + storage::rpc::SlimeClusterStateBundleCodec codec; + auto encoded_bundle = codec.encode(bundle); + auto *req = new FRT_RPCRequest(); + auto* params = req->GetParams(); + params->AddInt8(static_cast(encoded_bundle._compression_type)); + params->AddInt32(encoded_bundle._uncompressed_length); + const auto buf_len = encoded_bundle._buffer->getDataLen(); + params->AddData(encoded_bundle._buffer->stealBuffer(), buf_len); + req->SetMethodName("setdistributionstates"); + return req; +} + +void set_cluster_up(SharedRpcResources &shared_rpc_resources, storage::api::StorageMessageAddress &storage_address) { + auto req = make_set_cluster_state_request(); + auto target_resolver = std::make_unique(shared_rpc_resources.slobrok_mirror(), shared_rpc_resources.target_factory()); + auto target = target_resolver->resolve_rpc_target(storage_address); + target->_target->get()->InvokeSync(req, 10.0); // 10 seconds timeout + assert(!req->IsError()); + req->SubRef(); +} + +} + +struct PersistenceProviderFixture { + std::shared_ptr _document_types; + std::shared_ptr _repo; + DocTypeName _doc_type_name; + const DocumentType* _document_type; + const Field& _field; + std::shared_ptr _document_db_config; + vespalib::string _base_dir; + DummyFileHeaderContext _file_header_context; + int _tls_listen_port; + int _slobrok_port; + int _status_port; + int _rpc_client_port; + TransLogServer _tls; + vespalib::string _tls_spec; + matching::QueryLimiter _query_limiter; + vespalib::Clock _clock; + DummyWireService _metrics_wire_service; + MemoryConfigStores _config_stores; + vespalib::ThreadStackExecutor _summary_executor; + DummyDBOwner _document_db_owner; + BucketSpace _bucket_space; + std::shared_ptr _document_db; + MyPersistenceEngineOwner _persistence_owner; + MyResourceWriteFilter _write_filter; + std::shared_ptr _persistence_engine; + storage::spi::Context _context; + uint32_t _bucket_bits; + MyStorageConfig _service_layer_config; + MyRpcClientConfig _rpc_client_config; + ConfigSet _config_set; + std::shared_ptr _config_context; + storage::api::StorageMessageAddress _storage_address; + std::unique_ptr _slobrok; + std::unique_ptr _service_layer; + std::unique_ptr _message_codec_provider; + std::unique_ptr _rpc_client_shared_rpc_resources; + std::unique_ptr _rpc_message_dispatcher; + std::unique_ptr _rpc_client; + + PersistenceProviderFixture(const BMParams& params); + ~PersistenceProviderFixture(); + void create_document_db(); + uint32_t num_buckets() const { return (1u << _bucket_bits); } + BucketId make_bucket_id(uint32_t i) const { return BucketId(_bucket_bits, i & (num_buckets() - 1)); } + Bucket make_bucket(uint32_t i) const { return Bucket(document::Bucket(_bucket_space, BucketId(_bucket_bits, i & (num_buckets() - 1))), PartitionId(0)); } + DocumentId make_document_id(uint32_t i) const; + std::unique_ptr make_document(uint32_t i) const; + std::unique_ptr make_document_update(uint32_t i) const; + void create_buckets(); + void start_service_layer(); + void shutdown_service_layer(); + void send_rpc(std::shared_ptr cmd, MyPendingTracker& pending_tracker); +}; + +PersistenceProviderFixture::PersistenceProviderFixture(const BMParams& params) + : _document_types(make_document_type()), + _repo(DocumentTypeRepoFactory::make(*_document_types)), + _doc_type_name("test"), + _document_type(_repo->getDocumentType(_doc_type_name.getName())), + _field(_document_type->getField("int")), + _document_db_config(make_document_db_config(_document_types, _repo, _doc_type_name)), + _base_dir(base_dir), + _file_header_context(), + _tls_listen_port(9017), + _slobrok_port(9018), + _status_port(9019), + _rpc_client_port(9020), + _tls("tls", _tls_listen_port, _base_dir, _file_header_context), + _tls_spec(vespalib::make_string("tcp/localhost:%d", _tls_listen_port)), + _query_limiter(), + _clock(), + _metrics_wire_service(), + _config_stores(), + _summary_executor(8, 128 * 1024), + _document_db_owner(), + _bucket_space(makeBucketSpace(_doc_type_name.getName())), + _document_db(), + _persistence_owner(), + _write_filter(), + _persistence_engine(), + _context(default_load_type, Priority(0), Trace::TraceLevel(0)), + _bucket_bits(16), + _service_layer_config("bm-servicelayer", *_document_types, _slobrok_port, _status_port, params.get_rpc_network_threads()), + _rpc_client_config("bm-rpc-client", _slobrok_port), + _config_set(), + _config_context(std::make_shared(_config_set)), + _storage_address("storage", storage::lib::NodeType::STORAGE, 0), + _slobrok(), + _service_layer(), + _message_codec_provider(), + _rpc_client_shared_rpc_resources(), + _rpc_message_dispatcher(), + _rpc_client() +{ + create_document_db(); + _persistence_engine = std::make_unique(_persistence_owner, _write_filter, -1, false); + auto proxy = std::make_shared(_document_db); + _persistence_engine->putHandler(_persistence_engine->getWLock(), _bucket_space, _doc_type_name, proxy); + _service_layer_config.add_builders(_config_set); + _rpc_client_config.add_builders(_config_set); +} + +PersistenceProviderFixture::~PersistenceProviderFixture() +{ + if (_persistence_engine) { + _persistence_engine->destroyIterators(); + _persistence_engine->removeHandler(_persistence_engine->getWLock(), _bucket_space, _doc_type_name); + } + if (_document_db) { + _document_db->close(); + } +} + +void +PersistenceProviderFixture::create_document_db() +{ + vespalib::mkdir(_base_dir, false); + vespalib::mkdir(_base_dir + "/" + _doc_type_name.getName(), false); + vespalib::string input_cfg = _base_dir + "/" + _doc_type_name.getName() + "/baseconfig"; + { + FileConfigManager fileCfg(input_cfg, "", _doc_type_name.getName()); + fileCfg.saveConfig(*_document_db_config, 1); + } + config::DirSpec spec(input_cfg + "/config-1"); + auto tuneFileDocDB = std::make_shared(); + DocumentDBConfigHelper mgr(spec, _doc_type_name.getName()); + auto bootstrap_config = std::make_shared(1, + _document_types, + _repo, + std::make_shared(), + std::make_shared(), + std::make_shared(), + tuneFileDocDB, HwInfo()); + mgr.forwardConfig(bootstrap_config); + mgr.nextGeneration(0ms); + _document_db = std::make_shared(_base_dir, + mgr.getConfig(), + _tls_spec, + _query_limiter, + _clock, + _doc_type_name, + _bucket_space, + *bootstrap_config->getProtonConfigSP(), + _document_db_owner, + _summary_executor, + _summary_executor, + _tls, + _metrics_wire_service, + _file_header_context, + _config_stores.getConfigStore(_doc_type_name.toString()), + std::make_shared(16, 128 * 1024), + HwInfo()); + _document_db->start(); + _document_db->waitForOnlineState(); +} + +DocumentId +PersistenceProviderFixture::make_document_id(uint32_t i) const +{ + DocumentId id(vespalib::make_string("id::test:n=%u:%u", i & (num_buckets() - 1), i)); + return id; +} + +std::unique_ptr +PersistenceProviderFixture::make_document(uint32_t i) const +{ + auto id = make_document_id(i); + auto document = std::make_unique(*_document_type, id); + document->setRepo(*_repo); + document->setFieldValue(_field, std::make_unique(i)); + return document; +} + +std::unique_ptr +PersistenceProviderFixture::make_document_update(uint32_t i) const +{ + auto id = make_document_id(i); + auto document_update = std::make_unique(*_repo, *_document_type, id); + document_update->addUpdate(FieldUpdate(_field).addUpdate(AssignValueUpdate(IntFieldValue(15)))); + return document_update; +} + +void +PersistenceProviderFixture::create_buckets() +{ + auto &provider = *_persistence_engine; + for (unsigned int i = 0; i < num_buckets(); ++i) { + provider.createBucket(make_bucket(i), _context); + } +} + +void +PersistenceProviderFixture::start_service_layer() +{ + LOG(info, "start slobrok"); + _slobrok = std::make_unique(_slobrok_port); + LOG(info, "start service layer"); + config::ConfigUri config_uri("bm-servicelayer", _config_context); + _service_layer = std::make_unique(config_uri, + *_persistence_engine); + _service_layer->setupConfig(100ms); + _service_layer->createNode(); + _service_layer->getNode().waitUntilInitialized(); + _message_codec_provider = std::make_unique(_repo, std::make_shared()); + LOG(info, "start rpc client shared resources"); + config::ConfigUri client_config_uri("bm-rpc-client", _config_context); + _rpc_client_shared_rpc_resources = std::make_unique(client_config_uri, _rpc_client_port, 100); + _rpc_client_shared_rpc_resources->start_server_and_register_slobrok("bm-rpc-client"); + _rpc_message_dispatcher = std::make_unique(); + _rpc_client = std::make_unique(*_rpc_message_dispatcher, *_rpc_client_shared_rpc_resources, *_message_codec_provider, StorageApiRpcService::Params()); + set_cluster_up(*_rpc_client_shared_rpc_resources, _storage_address); +} + +void +PersistenceProviderFixture::shutdown_service_layer() +{ + _rpc_client.reset(); + _rpc_message_dispatcher.reset(); + if (_rpc_client_shared_rpc_resources) { + LOG(info, "stop rpc client shared resources"); + _rpc_client_shared_rpc_resources->shutdown(); + _rpc_client_shared_rpc_resources.reset(); + } + if (_service_layer) { + LOG(info, "stop service layer"); + _service_layer->getNode().requestShutdown("controlled shutdown"); + _service_layer->shutdown(); + } + if (_slobrok) { + LOG(info, "stop slobrok"); + _slobrok.reset(); + } +} + +void +PersistenceProviderFixture::send_rpc(std::shared_ptr cmd, MyPendingTracker& pending_tracker) +{ + cmd->setSourceIndex(0); + cmd->setAddress(_storage_address); + _rpc_message_dispatcher->retain(cmd->getMsgId(), pending_tracker); + _rpc_client->send_rpc_v1_request(std::move(cmd)); +} + +vespalib::nbostream +make_put_feed(PersistenceProviderFixture &f, BMRange range) +{ + vespalib::nbostream serialized_feed; + LOG(debug, "make_put_feed([%u..%u))", range.get_start(), range.get_end()); + for (unsigned int i = range.get_start(); i < range.get_end(); ++i) { + serialized_feed << f.make_bucket_id(i); + auto document = f.make_document(i); + document->serialize(serialized_feed); + } + return serialized_feed; +} + +std::vector +make_feed(vespalib::ThreadStackExecutor &executor, const BMParams &bm_params, std::function func, const vespalib::string &label) +{ + LOG(info, "make_feed %s %u small documents", label.c_str(), bm_params.get_documents()); + std::vector serialized_feed_v; + auto start_time = std::chrono::steady_clock::now(); + serialized_feed_v.resize(bm_params.get_threads()); + for (uint32_t i = 0; i < bm_params.get_threads(); ++i) { + auto range = bm_params.get_range(i); + executor.execute(makeLambdaTask([&serialized_feed_v, i, range, &func]() + { serialized_feed_v[i] = func(range); })); + } + executor.sync(); + auto end_time = std::chrono::steady_clock::now(); + std::chrono::duration elapsed = end_time - start_time; + LOG(info, "%8.2f %s data elements/s", bm_params.get_documents() / elapsed.count(), label.c_str()); + return serialized_feed_v; +} + +void +put_async_task(PersistenceProviderFixture &f, BMRange range, const vespalib::nbostream &serialized_feed, int64_t time_bias) +{ + LOG(debug, "put_async_task([%u..%u))", range.get_start(), range.get_end()); + MyPendingTracker pending_tracker(100); + auto &provider = *f._persistence_engine; + auto &context = f._context; + auto &repo = *f._repo; + vespalib::nbostream is(serialized_feed.data(), serialized_feed.size()); + BucketId bucket_id; + auto bucket_space = f._bucket_space; + for (unsigned int i = range.get_start(); i < range.get_end(); ++i) { + is >> bucket_id; + Bucket bucket(document::Bucket(bucket_space, bucket_id), PartitionId(0)); + auto document = std::make_unique(repo, is); + if (f._rpc_client) { + auto cmd = std::make_unique(bucket.getBucket(), std::move(document), time_bias + i); + f.send_rpc(std::move(cmd), pending_tracker); + } else { + provider.putAsync(bucket, Timestamp(time_bias + i), std::move(document), context, std::make_unique(pending_tracker)); + } + } + assert(is.empty()); + pending_tracker.drain(); +} + +void +run_put_async_tasks(PersistenceProviderFixture &f, vespalib::ThreadStackExecutor &executor, int pass, int64_t& time_bias, const std::vector &serialized_feed_v, const BMParams& bm_params) +{ + LOG(info, "putAsync %u small documents, pass=%u", bm_params.get_documents(), pass); + auto start_time = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < bm_params.get_threads(); ++i) { + auto range = bm_params.get_range(i); + executor.execute(makeLambdaTask([&f, &serialized_feed = serialized_feed_v[i], range, time_bias]() + { put_async_task(f, range, serialized_feed, time_bias); })); + } + executor.sync(); + auto end_time = std::chrono::steady_clock::now(); + std::chrono::duration elapsed = end_time - start_time; + LOG(info, "%8.2f puts/s for pass=%u", bm_params.get_documents() / elapsed.count(), pass); + time_bias += bm_params.get_documents(); +} + +vespalib::nbostream +make_update_feed(PersistenceProviderFixture &f, BMRange range) +{ + vespalib::nbostream serialized_feed; + LOG(debug, "make_update_feed([%u..%u))", range.get_start(), range.get_end()); + for (unsigned int i = range.get_start(); i < range.get_end(); ++i) { + serialized_feed << f.make_bucket_id(i); + auto document_update = f.make_document_update(i); + document_update->serializeHEAD(serialized_feed); + } + return serialized_feed; +} + +void +update_async_task(PersistenceProviderFixture &f, BMRange range, const vespalib::nbostream &serialized_feed, int64_t time_bias) +{ + LOG(debug, "update_async_task([%u..%u))", range.get_start(), range.get_end()); + MyPendingTracker pending_tracker(100); + auto &provider = *f._persistence_engine; + auto &context = f._context; + auto &repo = *f._repo; + vespalib::nbostream is(serialized_feed.data(), serialized_feed.size()); + BucketId bucket_id; + auto bucket_space = f._bucket_space; + for (unsigned int i = range.get_start(); i < range.get_end(); ++i) { + is >> bucket_id; + Bucket bucket(document::Bucket(bucket_space, bucket_id), PartitionId(0)); + auto document_update = DocumentUpdate::createHEAD(repo, is); + if (f._rpc_client) { + auto cmd = std::make_unique(bucket.getBucket(), std::move(document_update), time_bias + i); + f.send_rpc(std::move(cmd), pending_tracker); + } else { + provider.updateAsync(bucket, Timestamp(time_bias + i), std::move(document_update), context, std::make_unique(pending_tracker)); + } + } + assert(is.empty()); + pending_tracker.drain(); +} + +void +run_update_async_tasks(PersistenceProviderFixture &f, vespalib::ThreadStackExecutor &executor, int pass, int64_t& time_bias, const std::vector &serialized_feed_v, const BMParams& bm_params) +{ + LOG(info, "updateAsync %u small documents, pass=%u", bm_params.get_documents(), pass); + auto start_time = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < bm_params.get_threads(); ++i) { + auto range = bm_params.get_range(i); + executor.execute(makeLambdaTask([&f, &serialized_feed = serialized_feed_v[i], range, time_bias]() + { update_async_task(f, range, serialized_feed, time_bias); })); + } + executor.sync(); + auto end_time = std::chrono::steady_clock::now(); + std::chrono::duration elapsed = end_time - start_time; + LOG(info, "%8.2f updates/s for pass=%u", bm_params.get_documents() / elapsed.count(), pass); + time_bias += bm_params.get_documents(); +} + +vespalib::nbostream +make_remove_feed(PersistenceProviderFixture &f, BMRange range) +{ + vespalib::nbostream serialized_feed; + LOG(debug, "make_update_feed([%u..%u))", range.get_start(), range.get_end()); + for (unsigned int i = range.get_start(); i < range.get_end(); ++i) { + serialized_feed << f.make_bucket_id(i); + auto document_id = f.make_document_id(i); + vespalib::string raw_id = document_id.toString(); + serialized_feed.write(raw_id.c_str(), raw_id.size() + 1); + } + return serialized_feed; +} + +void +remove_async_task(PersistenceProviderFixture &f, BMRange range, const vespalib::nbostream &serialized_feed, int64_t time_bias) +{ + LOG(debug, "remove_async_task([%u..%u))", range.get_start(), range.get_end()); + MyPendingTracker pending_tracker(100); + auto &provider = *f._persistence_engine; + auto &context = f._context; + vespalib::nbostream is(serialized_feed.data(), serialized_feed.size()); + BucketId bucket_id; + auto bucket_space = f._bucket_space; + for (unsigned int i = range.get_start(); i < range.get_end(); ++i) { + is >> bucket_id; + Bucket bucket(document::Bucket(bucket_space, bucket_id), PartitionId(0)); + DocumentId document_id(is); + if (f._rpc_client) { + auto cmd = std::make_unique(bucket.getBucket(), document_id, time_bias + i); + f.send_rpc(std::move(cmd), pending_tracker); + } else { + provider.removeAsync(bucket, Timestamp(time_bias + i), document_id, context, std::make_unique(pending_tracker)); + } + } + assert(is.empty()); + pending_tracker.drain(); +} + +void +run_remove_async_tasks(PersistenceProviderFixture &f, vespalib::ThreadStackExecutor &executor, int pass, int64_t& time_bias, const std::vector &serialized_feed_v, const BMParams &bm_params) +{ + LOG(info, "removeAsync %u small documents, pass=%u", bm_params.get_documents(), pass); + auto start_time = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < bm_params.get_threads(); ++i) { + auto range = bm_params.get_range(i); + executor.execute(makeLambdaTask([&f, &serialized_feed = serialized_feed_v[i], range, time_bias]() + { remove_async_task(f, range, serialized_feed, time_bias); })); + } + executor.sync(); + auto end_time = std::chrono::steady_clock::now(); + std::chrono::duration elapsed = end_time - start_time; + LOG(info, "%8.2f removes/s for pass=%u", bm_params.get_documents() / elapsed.count(), pass); + time_bias += bm_params.get_documents(); +} + +void benchmark_async_spi(const BMParams &bm_params) +{ + vespalib::rmdir(base_dir, true); + PersistenceProviderFixture f(bm_params); + auto &provider = *f._persistence_engine; + LOG(info, "start initialize"); + provider.initialize(); + LOG(info, "create %u buckets", f.num_buckets()); + f.create_buckets(); + if (bm_params.get_enable_service_layer()) { + f.start_service_layer(); + } + vespalib::ThreadStackExecutor executor(bm_params.get_threads(), 128 * 1024); + auto put_feed = make_feed(executor, bm_params, [&f](BMRange range) { return make_put_feed(f, range); }, "put"); + auto update_feed = make_feed(executor, bm_params, [&f](BMRange range) { return make_update_feed(f, range); }, "update"); + auto remove_feed = make_feed(executor, bm_params, [&f](BMRange range) { return make_remove_feed(f, range); }, "remove"); + int64_t time_bias = 1; + for (uint32_t pass = 0; pass < bm_params.get_put_passes(); ++pass) { + run_put_async_tasks(f, executor, pass, time_bias, put_feed, bm_params); + } + for (uint32_t pass = 0; pass < bm_params.get_update_passes(); ++pass) { + run_update_async_tasks(f, executor, pass, time_bias, update_feed, bm_params); + } + for (uint32_t pass = 0; pass < bm_params.get_remove_passes(); ++pass) { + run_remove_async_tasks(f, executor, pass, time_bias, remove_feed, bm_params); + } + f.shutdown_service_layer(); +} + +class App : public FastOS_Application +{ + BMParams _bm_params; +public: + App(); + ~App() override; + void usage(); + bool get_options(); + int Main() override; +}; + +App::App() + : _bm_params() +{ +} + +App::~App() = default; + +void +App::usage() +{ + std::cerr << + "vespa-feed-bm version 0.0\n" + "\n" + "USAGE:\n"; + std::cerr << + "vespa-feed-bm\n" + "[--threads threads]\n" + "[--documents documents]\n" + "[--put-passes put-passes]\n" + "[--update-passes update-passes]\n" + "[--remove-passes remove-passes]\n" + "[--rpc-network-threads threads]\n" + "[--enable-service-layer]" << std::endl; +} + +bool +App::get_options() +{ + int c; + const char *opt_argument = nullptr; + int long_opt_index = 0; + static struct option long_opts[] = { + { "threads", 1, nullptr, 0 }, + { "documents", 1, nullptr, 0 }, + { "put-passes", 1, nullptr, 0 }, + { "update-passes", 1, nullptr, 0 }, + { "remove-passes", 1, nullptr, 0 }, + { "rpc-network-threads", 1, nullptr, 0 }, + { "enable-service-layer", 0, nullptr, 0 } + }; + enum longopts_enum { + LONGOPT_THREADS, + LONGOPT_DOCUMENTS, + LONGOPT_PUT_PASSES, + LONGOPT_UPDATE_PASSES, + LONGOPT_REMOVE_PASSES, + LONGOPT_RPC_NETWORK_THREADS, + LONGOPT_ENABLE_SERVICE_LAYER + }; + int opt_index = 1; + resetOptIndex(opt_index); + while ((c = GetOptLong("", opt_argument, opt_index, long_opts, &long_opt_index)) != -1) { + switch (c) { + case 0: + switch(long_opt_index) { + case LONGOPT_THREADS: + _bm_params.set_threads(atoi(opt_argument)); + break; + case LONGOPT_DOCUMENTS: + _bm_params.set_documents(atoi(opt_argument)); + break; + case LONGOPT_PUT_PASSES: + _bm_params.set_put_passes(atoi(opt_argument)); + break; + case LONGOPT_UPDATE_PASSES: + _bm_params.set_update_passes(atoi(opt_argument)); + break; + case LONGOPT_REMOVE_PASSES: + _bm_params.set_remove_passes(atoi(opt_argument)); + break; + case LONGOPT_RPC_NETWORK_THREADS: + _bm_params.set_rpc_network_threads(atoi(opt_argument)); + break; + case LONGOPT_ENABLE_SERVICE_LAYER: + _bm_params.set_enable_service_layer(true); + break; + default: + return false; + } + break; + default: + return false; + } + } + return _bm_params.check(); +} + +int +App::Main() +{ + if (!get_options()) { + usage(); + return 1; + } + benchmark_async_spi(_bm_params); + return 0; +} + +int +main(int argc, char* argv[]) +{ + DummyFileHeaderContext::setCreator("vespa-feed-bm"); + App app; + auto exit_value = app.Entry(argc, argv); + vespalib::rmdir(base_dir, true); + return exit_value; +} diff --git a/searchcore/src/apps/vespa-spi-feed-bm/.gitignore b/searchcore/src/apps/vespa-spi-feed-bm/.gitignore deleted file mode 100644 index 02fff2fe280..00000000000 --- a/searchcore/src/apps/vespa-spi-feed-bm/.gitignore +++ /dev/null @@ -1 +0,0 @@ -vespa-spi-feed-bm diff --git a/searchcore/src/apps/vespa-spi-feed-bm/CMakeLists.txt b/searchcore/src/apps/vespa-spi-feed-bm/CMakeLists.txt deleted file mode 100644 index 5b562cd1d28..00000000000 --- a/searchcore/src/apps/vespa-spi-feed-bm/CMakeLists.txt +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_vespa_spi_feed_bm_app - SOURCES - vespa_spi_feed_bm.cpp - OUTPUT_NAME vespa-spi-feed-bm - DEPENDS - searchcore_server - searchcore_initializer - searchcore_reprocessing - searchcore_index - searchcore_persistenceengine - searchcore_docsummary - searchcore_feedoperation - searchcore_matching - searchcore_attribute - searchcore_documentmetastore - searchcore_bucketdb - searchcore_flushengine - searchcore_pcommon - searchcore_grouping - searchcore_proton_metrics - searchcore_fconfig - storageserver_storageapp - messagebus_messagebus-test - messagebus - searchlib_searchlib_uca -) diff --git a/searchcore/src/apps/vespa-spi-feed-bm/vespa_spi_feed_bm.cpp b/searchcore/src/apps/vespa-spi-feed-bm/vespa_spi_feed_bm.cpp deleted file mode 100644 index 06ad022a328..00000000000 --- a/searchcore/src/apps/vespa-spi-feed-bm/vespa_spi_feed_bm.cpp +++ /dev/null @@ -1,1172 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -LOG_SETUP("vespa-spi-feed-bm"); - -using namespace config; -using namespace proton; -using namespace cloud::config::filedistribution; -using namespace vespa::config::search::core; -using namespace vespa::config::search::summary; -using namespace vespa::config::search; -using namespace std::chrono_literals; -using vespa::config::content::core::BucketspacesConfig; -using vespa::config::content::core::BucketspacesConfigBuilder; -using vespa::config::content::StorDistributionConfigBuilder; -using vespa::config::content::StorFilestorConfigBuilder; -using vespa::config::content::PersistenceConfigBuilder; -using vespa::config::content::core::StorBouncerConfigBuilder; -using vespa::config::content::core::StorCommunicationmanagerConfigBuilder; -using vespa::config::content::core::StorBucketInitConfigBuilder; -using vespa::config::content::core::StorOpsloggerConfigBuilder; -using vespa::config::content::core::StorPrioritymappingConfigBuilder; -using vespa::config::content::LoadTypeConfigBuilder; -using vespa::config::content::UpgradingConfigBuilder; -using vespa::config::content::core::StorServerConfigBuilder; -using vespa::config::content::core::StorStatusConfigBuilder; -using vespa::config::content::core::StorVisitorConfigBuilder; -using metrics::MetricsmanagerConfigBuilder; -using cloud::config::SlobroksConfigBuilder; -using messagebus::MessagebusConfigBuilder; - -using config::ConfigContext; -using config::ConfigUri; -using config::ConfigSet; -using config::IConfigContext; -using document::AssignValueUpdate; -using document::BucketId; -using document::BucketSpace; -using document::Document; -using document::DocumentId; -using document::DocumentType; -using document::DocumentTypeRepo; -using document::DocumentTypeRepoFactory; -using document::DocumenttypesConfig; -using document::DocumenttypesConfigBuilder; -using document::DocumentUpdate; -using document::Field; -using document::FieldUpdate; -using document::IntFieldValue; -using document::test::makeBucketSpace; -using documentapi::LoadTypeSet; -using search::TuneFileDocumentDB; -using search::index::DummyFileHeaderContext; -using search::index::Schema; -using search::index::SchemaBuilder; -using search::transactionlog::TransLogServer; -using storage::rpc::MessageCodecProvider; -using storage::rpc::SharedRpcResources; -using storage::rpc::StorageApiRpcService; -using storage::spi::Bucket; -using storage::spi::PartitionId; -using storage::spi::PersistenceProvider; -using storage::spi::Priority; -using storage::spi::Timestamp; -using storage::spi::Trace; -using vespalib::makeLambdaTask; - -using DocumentDBMap = std::map>; - -namespace { - -storage::spi::LoadType default_load_type(0, "default"); - -vespalib::string base_dir = "testdb"; - -std::shared_ptr make_document_type() { - using Struct = document::config_builder::Struct; - using DataType = document::DataType; - document::config_builder::DocumenttypesConfigBuilderHelper builder; - builder.document(42, "test", Struct("test.header").addField("int", DataType::T_INT), Struct("test.body")); - return std::make_shared(builder.config()); -} - -std::shared_ptr make_attributes_config() { - AttributesConfigBuilder builder; - AttributesConfig::Attribute attribute; - attribute.name = "int"; - attribute.datatype = AttributesConfig::Attribute::Datatype::INT32; - builder.attribute.emplace_back(attribute); - return std::make_shared(builder); -} - -std::shared_ptr make_document_db_config(std::shared_ptr document_types, std::shared_ptr repo, const DocTypeName& doc_type_name) -{ - auto indexschema = std::make_shared(); - auto attributes = make_attributes_config(); - auto summary = std::make_shared(); - std::shared_ptr schema(new Schema()); - SchemaBuilder::build(*indexschema, *schema); - SchemaBuilder::build(*attributes, *schema); - SchemaBuilder::build(*summary, *schema); - return std::make_shared( - 1, - std::make_shared(), - std::make_shared(), - std::make_shared(), - indexschema, - attributes, - summary, - std::make_shared(), - std::make_shared(), - document_types, - repo, - std::make_shared(), - std::make_shared(), - schema, - std::make_shared(), - search::LogDocumentStore::Config(), - "client", - doc_type_name.getName()); -} - -class MyPersistenceEngineOwner : public IPersistenceEngineOwner -{ - void setClusterState(BucketSpace, const storage::spi::ClusterState &) override { } -}; - -struct MyResourceWriteFilter : public IResourceWriteFilter -{ - bool acceptWriteOperation() const override { return true; } - State getAcceptState() const override { return IResourceWriteFilter::State(); } -}; - -class MyPendingTracker { - uint32_t _pending; - uint32_t _limit; - std::mutex _mutex; - std::condition_variable _cond; - -public: - MyPendingTracker(uint32_t limit) - : _pending(0u), - _limit(limit), - _mutex(), - _cond() - { - } - - ~MyPendingTracker() - { - drain(); - } - - void release() { - std::unique_lock guard(_mutex); - --_pending; - if (_pending < _limit) { - _cond.notify_all(); - } - //LOG(info, "release, pending is now %u", _pending); - } - void retain() { - std::unique_lock guard(_mutex); - while (_pending >= _limit) { - _cond.wait(guard); - } - ++_pending; - //LOG(info, "retain, pending is now %u", _pending); - } - - void drain() { - std::unique_lock guard(_mutex); - while (_pending > 0) { - _cond.wait(guard); - } - } -}; - -class MyOperationComplete : public storage::spi::OperationComplete -{ - MyPendingTracker& _tracker; -public: - MyOperationComplete(MyPendingTracker &tracker); - ~MyOperationComplete(); - void onComplete(std::unique_ptr result) override; - void addResultHandler(const storage::spi::ResultHandler* resultHandler) override; -}; - -MyOperationComplete::MyOperationComplete(MyPendingTracker& tracker) - : _tracker(tracker) -{ - _tracker.retain(); -} - -MyOperationComplete::~MyOperationComplete() -{ - _tracker.release(); -} - -void -MyOperationComplete::onComplete(std::unique_ptr result) -{ - (void) result; -} - -void -MyOperationComplete::addResultHandler(const storage::spi::ResultHandler * resultHandler) -{ - (void) resultHandler; -} - -class BMRange -{ - uint32_t _start; - uint32_t _end; -public: - BMRange(uint32_t start_in, uint32_t end_in) - : _start(start_in), - _end(end_in) - { - } - uint32_t get_start() const { return _start; } - uint32_t get_end() const { return _end; } -}; - -class BMParams { - uint32_t _documents; - uint32_t _threads; - uint32_t _put_passes; - uint32_t _update_passes; - uint32_t _remove_passes; - uint32_t _rpc_network_threads; - bool _enable_service_layer; - uint32_t get_start(uint32_t thread_id) const { - return (_documents / _threads) * thread_id + std::min(thread_id, _documents % _threads); - } -public: - BMParams() - : _documents(160000), - _threads(32), - _put_passes(2), - _update_passes(1), - _remove_passes(2), - _rpc_network_threads(1), - _enable_service_layer(false) - { - } - BMRange get_range(uint32_t thread_id) const { - return BMRange(get_start(thread_id), get_start(thread_id + 1)); - } - uint32_t get_documents() const { return _documents; } - uint32_t get_threads() const { return _threads; } - uint32_t get_put_passes() const { return _put_passes; } - uint32_t get_update_passes() const { return _update_passes; } - uint32_t get_remove_passes() const { return _remove_passes; } - uint32_t get_rpc_network_threads() const { return _rpc_network_threads; } - bool get_enable_service_layer() const { return _enable_service_layer; } - void set_documents(uint32_t documents_in) { _documents = documents_in; } - void set_threads(uint32_t threads_in) { _threads = threads_in; } - void set_put_passes(uint32_t put_passes_in) { _put_passes = put_passes_in; } - void set_update_passes(uint32_t update_passes_in) { _update_passes = update_passes_in; } - void set_remove_passes(uint32_t remove_passes_in) { _remove_passes = remove_passes_in; } - void set_rpc_network_threads(uint32_t threads_in) { _rpc_network_threads = threads_in; } - void set_enable_service_layer(bool enable_service_layer_in) { _enable_service_layer = enable_service_layer_in; } - bool check() const; -}; - -bool -BMParams::check() const -{ - if (_threads < 1) { - std::cerr << "Too few threads: " << _threads << std::endl; - return false; - } - if (_threads > 256) { - std::cerr << "Too many threads: " << _threads << std::endl; - return false; - } - if (_documents < _threads) { - std::cerr << "Too few documents: " << _documents << std::endl; - return false; - } - if (_put_passes < 1) { - std::cerr << "Put passes too low: " << _put_passes << std::endl; - return false; - } - if (_rpc_network_threads < 1) { - std::cerr << "Too few rpc network threads: " << _rpc_network_threads << std::endl; - return false; - } - return true; -} - - -class MyServiceLayerProcess : public storage::ServiceLayerProcess { - PersistenceProvider& _provider; - -public: - MyServiceLayerProcess(const config::ConfigUri & configUri, - PersistenceProvider &provider); - ~MyServiceLayerProcess() override { shutdown(); } - - void shutdown() override; - void setupProvider() override; - PersistenceProvider& getProvider() override; -}; - -MyServiceLayerProcess::MyServiceLayerProcess(const config::ConfigUri & configUri, - PersistenceProvider &provider) - : ServiceLayerProcess(configUri), - _provider(provider) -{ -} - -void -MyServiceLayerProcess::shutdown() -{ - ServiceLayerProcess::shutdown(); -} - -void -MyServiceLayerProcess::setupProvider() -{ -} - -PersistenceProvider& -MyServiceLayerProcess::getProvider() -{ - return _provider; -} - -struct MyStorageConfig -{ - vespalib::string config_id; - DocumenttypesConfigBuilder documenttypes; - PersistenceConfigBuilder persistence; - StorDistributionConfigBuilder stor_distribution; - StorFilestorConfigBuilder stor_filestor; - StorBouncerConfigBuilder stor_bouncer; - StorCommunicationmanagerConfigBuilder stor_communicationmanager; - StorBucketInitConfigBuilder stor_bucket_init; - StorOpsloggerConfigBuilder stor_opslogger; - StorPrioritymappingConfigBuilder stor_prioritymapping; - UpgradingConfigBuilder upgrading; - StorServerConfigBuilder stor_server; - StorStatusConfigBuilder stor_status; - StorVisitorConfigBuilder stor_visitor; - BucketspacesConfigBuilder bucketspaces; - LoadTypeConfigBuilder load_type; - MetricsmanagerConfigBuilder metricsmanager; - SlobroksConfigBuilder slobroks; - MessagebusConfigBuilder messagebus; - - MyStorageConfig(const vespalib::string& config_id_in, const DocumenttypesConfig& documenttypes_in, int slobrok_port, int status_port, uint32_t rpc_network_threads) - : config_id(config_id_in), - documenttypes(documenttypes_in), - persistence(), - stor_distribution(), - stor_filestor(), - stor_bouncer(), - stor_communicationmanager(), - stor_bucket_init(), - stor_opslogger(), - stor_prioritymapping(), - upgrading(), - stor_server(), - stor_status(), - stor_visitor(), - bucketspaces(), - load_type(), - metricsmanager(), - slobroks(), - messagebus() - { - { - auto &dc = stor_distribution; - { - StorDistributionConfigBuilder::Group group; - { - StorDistributionConfigBuilder::Group::Nodes node; - node.index = 0; - group.nodes.push_back(std::move(node)); - } - group.index = "invalid"; - group.name = "invalid"; - group.capacity = 1.0; - group.partitions = ""; - dc.group.push_back(std::move(group)); - } - dc.redundancy = 1; - dc.readyCopies = 1; - } - stor_server.rootFolder = "storage"; - { - SlobroksConfigBuilder::Slobrok slobrok; - slobrok.connectionspec = vespalib::make_string("tcp/localhost:%d", slobrok_port); - slobroks.slobrok.push_back(std::move(slobrok)); - } - stor_communicationmanager.useDirectStorageapiRpc = true; - stor_communicationmanager.rpc.numNetworkThreads = rpc_network_threads; - stor_status.httpport = status_port; - } - - ~MyStorageConfig(); - - void add_builders(ConfigSet &set) { - set.addBuilder(config_id, &documenttypes); - set.addBuilder(config_id, &persistence); - set.addBuilder(config_id, &stor_distribution); - set.addBuilder(config_id, &stor_filestor); - set.addBuilder(config_id, &stor_bouncer); - set.addBuilder(config_id, &stor_communicationmanager); - set.addBuilder(config_id, &stor_bucket_init); - set.addBuilder(config_id, &stor_opslogger); - set.addBuilder(config_id, &stor_prioritymapping); - set.addBuilder(config_id, &upgrading); - set.addBuilder(config_id, &stor_server); - set.addBuilder(config_id, &stor_status); - set.addBuilder(config_id, &stor_visitor); - set.addBuilder(config_id, &bucketspaces); - set.addBuilder(config_id, &load_type); - set.addBuilder(config_id, &metricsmanager); - set.addBuilder(config_id, &slobroks); - set.addBuilder(config_id, &messagebus); - } -}; - -MyStorageConfig::~MyStorageConfig() = default; - -struct MyRpcClientConfig { - vespalib::string config_id; - SlobroksConfigBuilder slobroks; - - MyRpcClientConfig(const vespalib::string &config_id_in, int slobrok_port) - : config_id(config_id_in), - slobroks() - { - { - SlobroksConfigBuilder::Slobrok slobrok; - slobrok.connectionspec = vespalib::make_string("tcp/localhost:%d", slobrok_port); - slobroks.slobrok.push_back(std::move(slobrok)); - } - } - ~MyRpcClientConfig(); - - void add_builders(ConfigSet &set) { - set.addBuilder(config_id, &slobroks); - } -}; - -MyRpcClientConfig::~MyRpcClientConfig() = default; - -class MyMessageDispatcher : public storage::MessageDispatcher -{ - std::mutex _mutex; - vespalib::hash_map _pending; -public: - MyMessageDispatcher() - : storage::MessageDispatcher(), - _mutex(), - _pending() - { - } - ~MyMessageDispatcher() override; - void dispatch_sync(std::shared_ptr msg) override { - release(msg->getMsgId()); - } - void dispatch_async(std::shared_ptr msg) override { - release(msg->getMsgId()); - } - void retain(uint64_t msg_id, MyPendingTracker &tracker) { - tracker.retain(); - std::lock_guard lock(_mutex); - _pending.insert(std::make_pair(msg_id, &tracker)); - } - void release(uint64_t msg_id) { - MyPendingTracker *tracker = nullptr; - { - std::lock_guard lock(_mutex); - auto itr = _pending.find(msg_id); - assert(itr != _pending.end()); - tracker = itr->second; - _pending.erase(itr); - } - tracker->release(); - } -}; - -MyMessageDispatcher::~MyMessageDispatcher() -{ - std::lock_guard lock(_mutex); - assert(_pending.empty()); -} - -FRT_RPCRequest *make_set_cluster_state_request() { - storage::lib::ClusterStateBundle bundle(storage::lib::ClusterState("version:2 distributor:1 storage:1")); - storage::rpc::SlimeClusterStateBundleCodec codec; - auto encoded_bundle = codec.encode(bundle); - auto *req = new FRT_RPCRequest(); - auto* params = req->GetParams(); - params->AddInt8(static_cast(encoded_bundle._compression_type)); - params->AddInt32(encoded_bundle._uncompressed_length); - const auto buf_len = encoded_bundle._buffer->getDataLen(); - params->AddData(encoded_bundle._buffer->stealBuffer(), buf_len); - req->SetMethodName("setdistributionstates"); - return req; -} - -void set_cluster_up(SharedRpcResources &shared_rpc_resources, storage::api::StorageMessageAddress &storage_address) { - auto req = make_set_cluster_state_request(); - auto target_resolver = std::make_unique(shared_rpc_resources.slobrok_mirror(), shared_rpc_resources.target_factory()); - auto target = target_resolver->resolve_rpc_target(storage_address); - target->_target->get()->InvokeSync(req, 10.0); // 10 seconds timeout - assert(!req->IsError()); - req->SubRef(); -} - -} - -struct PersistenceProviderFixture { - std::shared_ptr _document_types; - std::shared_ptr _repo; - DocTypeName _doc_type_name; - const DocumentType* _document_type; - const Field& _field; - std::shared_ptr _document_db_config; - vespalib::string _base_dir; - DummyFileHeaderContext _file_header_context; - int _tls_listen_port; - int _slobrok_port; - int _status_port; - int _rpc_client_port; - TransLogServer _tls; - vespalib::string _tls_spec; - matching::QueryLimiter _query_limiter; - vespalib::Clock _clock; - DummyWireService _metrics_wire_service; - MemoryConfigStores _config_stores; - vespalib::ThreadStackExecutor _summary_executor; - DummyDBOwner _document_db_owner; - BucketSpace _bucket_space; - std::shared_ptr _document_db; - MyPersistenceEngineOwner _persistence_owner; - MyResourceWriteFilter _write_filter; - std::shared_ptr _persistence_engine; - storage::spi::Context _context; - uint32_t _bucket_bits; - MyStorageConfig _service_layer_config; - MyRpcClientConfig _rpc_client_config; - ConfigSet _config_set; - std::shared_ptr _config_context; - storage::api::StorageMessageAddress _storage_address; - std::unique_ptr _slobrok; - std::unique_ptr _service_layer; - std::unique_ptr _message_codec_provider; - std::unique_ptr _rpc_client_shared_rpc_resources; - std::unique_ptr _rpc_message_dispatcher; - std::unique_ptr _rpc_client; - - PersistenceProviderFixture(const BMParams& params); - ~PersistenceProviderFixture(); - void create_document_db(); - uint32_t num_buckets() const { return (1u << _bucket_bits); } - BucketId make_bucket_id(uint32_t i) const { return BucketId(_bucket_bits, i & (num_buckets() - 1)); } - Bucket make_bucket(uint32_t i) const { return Bucket(document::Bucket(_bucket_space, BucketId(_bucket_bits, i & (num_buckets() - 1))), PartitionId(0)); } - DocumentId make_document_id(uint32_t i) const; - std::unique_ptr make_document(uint32_t i) const; - std::unique_ptr make_document_update(uint32_t i) const; - void create_buckets(); - void start_service_layer(); - void shutdown_service_layer(); - void send_rpc(std::shared_ptr cmd, MyPendingTracker& pending_tracker); -}; - -PersistenceProviderFixture::PersistenceProviderFixture(const BMParams& params) - : _document_types(make_document_type()), - _repo(DocumentTypeRepoFactory::make(*_document_types)), - _doc_type_name("test"), - _document_type(_repo->getDocumentType(_doc_type_name.getName())), - _field(_document_type->getField("int")), - _document_db_config(make_document_db_config(_document_types, _repo, _doc_type_name)), - _base_dir(base_dir), - _file_header_context(), - _tls_listen_port(9017), - _slobrok_port(9018), - _status_port(9019), - _rpc_client_port(9020), - _tls("tls", _tls_listen_port, _base_dir, _file_header_context), - _tls_spec(vespalib::make_string("tcp/localhost:%d", _tls_listen_port)), - _query_limiter(), - _clock(), - _metrics_wire_service(), - _config_stores(), - _summary_executor(8, 128 * 1024), - _document_db_owner(), - _bucket_space(makeBucketSpace(_doc_type_name.getName())), - _document_db(), - _persistence_owner(), - _write_filter(), - _persistence_engine(), - _context(default_load_type, Priority(0), Trace::TraceLevel(0)), - _bucket_bits(16), - _service_layer_config("bm-servicelayer", *_document_types, _slobrok_port, _status_port, params.get_rpc_network_threads()), - _rpc_client_config("bm-rpc-client", _slobrok_port), - _config_set(), - _config_context(std::make_shared(_config_set)), - _storage_address("storage", storage::lib::NodeType::STORAGE, 0), - _slobrok(), - _service_layer(), - _message_codec_provider(), - _rpc_client_shared_rpc_resources(), - _rpc_message_dispatcher(), - _rpc_client() -{ - create_document_db(); - _persistence_engine = std::make_unique(_persistence_owner, _write_filter, -1, false); - auto proxy = std::make_shared(_document_db); - _persistence_engine->putHandler(_persistence_engine->getWLock(), _bucket_space, _doc_type_name, proxy); - _service_layer_config.add_builders(_config_set); - _rpc_client_config.add_builders(_config_set); -} - -PersistenceProviderFixture::~PersistenceProviderFixture() -{ - if (_persistence_engine) { - _persistence_engine->destroyIterators(); - _persistence_engine->removeHandler(_persistence_engine->getWLock(), _bucket_space, _doc_type_name); - } - if (_document_db) { - _document_db->close(); - } -} - -void -PersistenceProviderFixture::create_document_db() -{ - vespalib::mkdir(_base_dir, false); - vespalib::mkdir(_base_dir + "/" + _doc_type_name.getName(), false); - vespalib::string input_cfg = _base_dir + "/" + _doc_type_name.getName() + "/baseconfig"; - { - FileConfigManager fileCfg(input_cfg, "", _doc_type_name.getName()); - fileCfg.saveConfig(*_document_db_config, 1); - } - config::DirSpec spec(input_cfg + "/config-1"); - auto tuneFileDocDB = std::make_shared(); - DocumentDBConfigHelper mgr(spec, _doc_type_name.getName()); - auto bootstrap_config = std::make_shared(1, - _document_types, - _repo, - std::make_shared(), - std::make_shared(), - std::make_shared(), - tuneFileDocDB, HwInfo()); - mgr.forwardConfig(bootstrap_config); - mgr.nextGeneration(0ms); - _document_db = std::make_shared(_base_dir, - mgr.getConfig(), - _tls_spec, - _query_limiter, - _clock, - _doc_type_name, - _bucket_space, - *bootstrap_config->getProtonConfigSP(), - _document_db_owner, - _summary_executor, - _summary_executor, - _tls, - _metrics_wire_service, - _file_header_context, - _config_stores.getConfigStore(_doc_type_name.toString()), - std::make_shared(16, 128 * 1024), - HwInfo()); - _document_db->start(); - _document_db->waitForOnlineState(); -} - -DocumentId -PersistenceProviderFixture::make_document_id(uint32_t i) const -{ - DocumentId id(vespalib::make_string("id::test:n=%u:%u", i & (num_buckets() - 1), i)); - return id; -} - -std::unique_ptr -PersistenceProviderFixture::make_document(uint32_t i) const -{ - auto id = make_document_id(i); - auto document = std::make_unique(*_document_type, id); - document->setRepo(*_repo); - document->setFieldValue(_field, std::make_unique(i)); - return document; -} - -std::unique_ptr -PersistenceProviderFixture::make_document_update(uint32_t i) const -{ - auto id = make_document_id(i); - auto document_update = std::make_unique(*_repo, *_document_type, id); - document_update->addUpdate(FieldUpdate(_field).addUpdate(AssignValueUpdate(IntFieldValue(15)))); - return document_update; -} - -void -PersistenceProviderFixture::create_buckets() -{ - auto &provider = *_persistence_engine; - for (unsigned int i = 0; i < num_buckets(); ++i) { - provider.createBucket(make_bucket(i), _context); - } -} - -void -PersistenceProviderFixture::start_service_layer() -{ - LOG(info, "start slobrok"); - _slobrok = std::make_unique(_slobrok_port); - LOG(info, "start service layer"); - config::ConfigUri config_uri("bm-servicelayer", _config_context); - _service_layer = std::make_unique(config_uri, - *_persistence_engine); - _service_layer->setupConfig(100ms); - _service_layer->createNode(); - _service_layer->getNode().waitUntilInitialized(); - _message_codec_provider = std::make_unique(_repo, std::make_shared()); - LOG(info, "start rpc client shared resources"); - config::ConfigUri client_config_uri("bm-rpc-client", _config_context); - _rpc_client_shared_rpc_resources = std::make_unique(client_config_uri, _rpc_client_port, 100); - _rpc_client_shared_rpc_resources->start_server_and_register_slobrok("bm-rpc-client"); - _rpc_message_dispatcher = std::make_unique(); - _rpc_client = std::make_unique(*_rpc_message_dispatcher, *_rpc_client_shared_rpc_resources, *_message_codec_provider, StorageApiRpcService::Params()); - set_cluster_up(*_rpc_client_shared_rpc_resources, _storage_address); -} - -void -PersistenceProviderFixture::shutdown_service_layer() -{ - _rpc_client.reset(); - _rpc_message_dispatcher.reset(); - if (_rpc_client_shared_rpc_resources) { - LOG(info, "stop rpc client shared resources"); - _rpc_client_shared_rpc_resources->shutdown(); - _rpc_client_shared_rpc_resources.reset(); - } - if (_service_layer) { - LOG(info, "stop service layer"); - _service_layer->getNode().requestShutdown("controlled shutdown"); - _service_layer->shutdown(); - } - if (_slobrok) { - LOG(info, "stop slobrok"); - _slobrok.reset(); - } -} - -void -PersistenceProviderFixture::send_rpc(std::shared_ptr cmd, MyPendingTracker& pending_tracker) -{ - cmd->setSourceIndex(0); - cmd->setAddress(_storage_address); - _rpc_message_dispatcher->retain(cmd->getMsgId(), pending_tracker); - _rpc_client->send_rpc_v1_request(std::move(cmd)); -} - -vespalib::nbostream -make_put_feed(PersistenceProviderFixture &f, BMRange range) -{ - vespalib::nbostream serialized_feed; - LOG(debug, "make_put_feed([%u..%u))", range.get_start(), range.get_end()); - for (unsigned int i = range.get_start(); i < range.get_end(); ++i) { - serialized_feed << f.make_bucket_id(i); - auto document = f.make_document(i); - document->serialize(serialized_feed); - } - return serialized_feed; -} - -std::vector -make_feed(vespalib::ThreadStackExecutor &executor, const BMParams &bm_params, std::function func, const vespalib::string &label) -{ - LOG(info, "make_feed %s %u small documents", label.c_str(), bm_params.get_documents()); - std::vector serialized_feed_v; - auto start_time = std::chrono::steady_clock::now(); - serialized_feed_v.resize(bm_params.get_threads()); - for (uint32_t i = 0; i < bm_params.get_threads(); ++i) { - auto range = bm_params.get_range(i); - executor.execute(makeLambdaTask([&serialized_feed_v, i, range, &func]() - { serialized_feed_v[i] = func(range); })); - } - executor.sync(); - auto end_time = std::chrono::steady_clock::now(); - std::chrono::duration elapsed = end_time - start_time; - LOG(info, "%8.2f %s data elements/s", bm_params.get_documents() / elapsed.count(), label.c_str()); - return serialized_feed_v; -} - -void -put_async_task(PersistenceProviderFixture &f, BMRange range, const vespalib::nbostream &serialized_feed, int64_t time_bias) -{ - LOG(debug, "put_async_task([%u..%u))", range.get_start(), range.get_end()); - MyPendingTracker pending_tracker(100); - auto &provider = *f._persistence_engine; - auto &context = f._context; - auto &repo = *f._repo; - vespalib::nbostream is(serialized_feed.data(), serialized_feed.size()); - BucketId bucket_id; - auto bucket_space = f._bucket_space; - for (unsigned int i = range.get_start(); i < range.get_end(); ++i) { - is >> bucket_id; - Bucket bucket(document::Bucket(bucket_space, bucket_id), PartitionId(0)); - auto document = std::make_unique(repo, is); - if (f._rpc_client) { - auto cmd = std::make_unique(bucket.getBucket(), std::move(document), time_bias + i); - f.send_rpc(std::move(cmd), pending_tracker); - } else { - provider.putAsync(bucket, Timestamp(time_bias + i), std::move(document), context, std::make_unique(pending_tracker)); - } - } - assert(is.empty()); - pending_tracker.drain(); -} - -void -run_put_async_tasks(PersistenceProviderFixture &f, vespalib::ThreadStackExecutor &executor, int pass, int64_t& time_bias, const std::vector &serialized_feed_v, const BMParams& bm_params) -{ - LOG(info, "putAsync %u small documents, pass=%u", bm_params.get_documents(), pass); - auto start_time = std::chrono::steady_clock::now(); - for (uint32_t i = 0; i < bm_params.get_threads(); ++i) { - auto range = bm_params.get_range(i); - executor.execute(makeLambdaTask([&f, &serialized_feed = serialized_feed_v[i], range, time_bias]() - { put_async_task(f, range, serialized_feed, time_bias); })); - } - executor.sync(); - auto end_time = std::chrono::steady_clock::now(); - std::chrono::duration elapsed = end_time - start_time; - LOG(info, "%8.2f puts/s for pass=%u", bm_params.get_documents() / elapsed.count(), pass); - time_bias += bm_params.get_documents(); -} - -vespalib::nbostream -make_update_feed(PersistenceProviderFixture &f, BMRange range) -{ - vespalib::nbostream serialized_feed; - LOG(debug, "make_update_feed([%u..%u))", range.get_start(), range.get_end()); - for (unsigned int i = range.get_start(); i < range.get_end(); ++i) { - serialized_feed << f.make_bucket_id(i); - auto document_update = f.make_document_update(i); - document_update->serializeHEAD(serialized_feed); - } - return serialized_feed; -} - -void -update_async_task(PersistenceProviderFixture &f, BMRange range, const vespalib::nbostream &serialized_feed, int64_t time_bias) -{ - LOG(debug, "update_async_task([%u..%u))", range.get_start(), range.get_end()); - MyPendingTracker pending_tracker(100); - auto &provider = *f._persistence_engine; - auto &context = f._context; - auto &repo = *f._repo; - vespalib::nbostream is(serialized_feed.data(), serialized_feed.size()); - BucketId bucket_id; - auto bucket_space = f._bucket_space; - for (unsigned int i = range.get_start(); i < range.get_end(); ++i) { - is >> bucket_id; - Bucket bucket(document::Bucket(bucket_space, bucket_id), PartitionId(0)); - auto document_update = DocumentUpdate::createHEAD(repo, is); - if (f._rpc_client) { - auto cmd = std::make_unique(bucket.getBucket(), std::move(document_update), time_bias + i); - f.send_rpc(std::move(cmd), pending_tracker); - } else { - provider.updateAsync(bucket, Timestamp(time_bias + i), std::move(document_update), context, std::make_unique(pending_tracker)); - } - } - assert(is.empty()); - pending_tracker.drain(); -} - -void -run_update_async_tasks(PersistenceProviderFixture &f, vespalib::ThreadStackExecutor &executor, int pass, int64_t& time_bias, const std::vector &serialized_feed_v, const BMParams& bm_params) -{ - LOG(info, "updateAsync %u small documents, pass=%u", bm_params.get_documents(), pass); - auto start_time = std::chrono::steady_clock::now(); - for (uint32_t i = 0; i < bm_params.get_threads(); ++i) { - auto range = bm_params.get_range(i); - executor.execute(makeLambdaTask([&f, &serialized_feed = serialized_feed_v[i], range, time_bias]() - { update_async_task(f, range, serialized_feed, time_bias); })); - } - executor.sync(); - auto end_time = std::chrono::steady_clock::now(); - std::chrono::duration elapsed = end_time - start_time; - LOG(info, "%8.2f updates/s for pass=%u", bm_params.get_documents() / elapsed.count(), pass); - time_bias += bm_params.get_documents(); -} - -vespalib::nbostream -make_remove_feed(PersistenceProviderFixture &f, BMRange range) -{ - vespalib::nbostream serialized_feed; - LOG(debug, "make_update_feed([%u..%u))", range.get_start(), range.get_end()); - for (unsigned int i = range.get_start(); i < range.get_end(); ++i) { - serialized_feed << f.make_bucket_id(i); - auto document_id = f.make_document_id(i); - vespalib::string raw_id = document_id.toString(); - serialized_feed.write(raw_id.c_str(), raw_id.size() + 1); - } - return serialized_feed; -} - -void -remove_async_task(PersistenceProviderFixture &f, BMRange range, const vespalib::nbostream &serialized_feed, int64_t time_bias) -{ - LOG(debug, "remove_async_task([%u..%u))", range.get_start(), range.get_end()); - MyPendingTracker pending_tracker(100); - auto &provider = *f._persistence_engine; - auto &context = f._context; - vespalib::nbostream is(serialized_feed.data(), serialized_feed.size()); - BucketId bucket_id; - auto bucket_space = f._bucket_space; - for (unsigned int i = range.get_start(); i < range.get_end(); ++i) { - is >> bucket_id; - Bucket bucket(document::Bucket(bucket_space, bucket_id), PartitionId(0)); - DocumentId document_id(is); - if (f._rpc_client) { - auto cmd = std::make_unique(bucket.getBucket(), document_id, time_bias + i); - f.send_rpc(std::move(cmd), pending_tracker); - } else { - provider.removeAsync(bucket, Timestamp(time_bias + i), document_id, context, std::make_unique(pending_tracker)); - } - } - assert(is.empty()); - pending_tracker.drain(); -} - -void -run_remove_async_tasks(PersistenceProviderFixture &f, vespalib::ThreadStackExecutor &executor, int pass, int64_t& time_bias, const std::vector &serialized_feed_v, const BMParams &bm_params) -{ - LOG(info, "removeAsync %u small documents, pass=%u", bm_params.get_documents(), pass); - auto start_time = std::chrono::steady_clock::now(); - for (uint32_t i = 0; i < bm_params.get_threads(); ++i) { - auto range = bm_params.get_range(i); - executor.execute(makeLambdaTask([&f, &serialized_feed = serialized_feed_v[i], range, time_bias]() - { remove_async_task(f, range, serialized_feed, time_bias); })); - } - executor.sync(); - auto end_time = std::chrono::steady_clock::now(); - std::chrono::duration elapsed = end_time - start_time; - LOG(info, "%8.2f removes/s for pass=%u", bm_params.get_documents() / elapsed.count(), pass); - time_bias += bm_params.get_documents(); -} - -void benchmark_async_spi(const BMParams &bm_params) -{ - vespalib::rmdir(base_dir, true); - PersistenceProviderFixture f(bm_params); - auto &provider = *f._persistence_engine; - LOG(info, "start initialize"); - provider.initialize(); - LOG(info, "create %u buckets", f.num_buckets()); - f.create_buckets(); - if (bm_params.get_enable_service_layer()) { - f.start_service_layer(); - } - vespalib::ThreadStackExecutor executor(bm_params.get_threads(), 128 * 1024); - auto put_feed = make_feed(executor, bm_params, [&f](BMRange range) { return make_put_feed(f, range); }, "put"); - auto update_feed = make_feed(executor, bm_params, [&f](BMRange range) { return make_update_feed(f, range); }, "update"); - auto remove_feed = make_feed(executor, bm_params, [&f](BMRange range) { return make_remove_feed(f, range); }, "remove"); - int64_t time_bias = 1; - for (uint32_t pass = 0; pass < bm_params.get_put_passes(); ++pass) { - run_put_async_tasks(f, executor, pass, time_bias, put_feed, bm_params); - } - for (uint32_t pass = 0; pass < bm_params.get_update_passes(); ++pass) { - run_update_async_tasks(f, executor, pass, time_bias, update_feed, bm_params); - } - for (uint32_t pass = 0; pass < bm_params.get_remove_passes(); ++pass) { - run_remove_async_tasks(f, executor, pass, time_bias, remove_feed, bm_params); - } - f.shutdown_service_layer(); -} - -class App : public FastOS_Application -{ - BMParams _bm_params; -public: - App(); - ~App() override; - void usage(); - bool get_options(); - int Main() override; -}; - -App::App() - : _bm_params() -{ -} - -App::~App() = default; - -void -App::usage() -{ - std::cerr << - "vespa-spi-feed-bm version 0.0\n" - "\n" - "USAGE:\n"; - std::cerr << - "vespa-spi-feed-bm\n" - "[--threads threads]\n" - "[--documents documents]\n" - "[--put-passes put-passes]\n" - "[--update-passes update-passes]\n" - "[--remove-passes remove-passes]\n" - "[--rpc-network-threads threads]\n" - "[--enable-service-layer]" << std::endl; -} - -bool -App::get_options() -{ - int c; - const char *opt_argument = nullptr; - int long_opt_index = 0; - static struct option long_opts[] = { - { "threads", 1, nullptr, 0 }, - { "documents", 1, nullptr, 0 }, - { "put-passes", 1, nullptr, 0 }, - { "update-passes", 1, nullptr, 0 }, - { "remove-passes", 1, nullptr, 0 }, - { "rpc-network-threads", 1, nullptr, 0 }, - { "enable-service-layer", 0, nullptr, 0 } - }; - enum longopts_enum { - LONGOPT_THREADS, - LONGOPT_DOCUMENTS, - LONGOPT_PUT_PASSES, - LONGOPT_UPDATE_PASSES, - LONGOPT_REMOVE_PASSES, - LONGOPT_RPC_NETWORK_THREADS, - LONGOPT_ENABLE_SERVICE_LAYER - }; - int opt_index = 1; - resetOptIndex(opt_index); - while ((c = GetOptLong("", opt_argument, opt_index, long_opts, &long_opt_index)) != -1) { - switch (c) { - case 0: - switch(long_opt_index) { - case LONGOPT_THREADS: - _bm_params.set_threads(atoi(opt_argument)); - break; - case LONGOPT_DOCUMENTS: - _bm_params.set_documents(atoi(opt_argument)); - break; - case LONGOPT_PUT_PASSES: - _bm_params.set_put_passes(atoi(opt_argument)); - break; - case LONGOPT_UPDATE_PASSES: - _bm_params.set_update_passes(atoi(opt_argument)); - break; - case LONGOPT_REMOVE_PASSES: - _bm_params.set_remove_passes(atoi(opt_argument)); - break; - case LONGOPT_RPC_NETWORK_THREADS: - _bm_params.set_rpc_network_threads(atoi(opt_argument)); - break; - case LONGOPT_ENABLE_SERVICE_LAYER: - _bm_params.set_enable_service_layer(true); - break; - default: - return false; - } - break; - default: - return false; - } - } - return _bm_params.check(); -} - -int -App::Main() -{ - if (!get_options()) { - usage(); - return 1; - } - benchmark_async_spi(_bm_params); - return 0; -} - -int -main(int argc, char* argv[]) -{ - DummyFileHeaderContext::setCreator("vespa-spi-feed-bm"); - App app; - auto exit_value = app.Entry(argc, argv); - vespalib::rmdir(base_dir, true); - return exit_value; -} -- cgit v1.2.3 From ac6b7f9ce49d56670c963f2d732d468afa0b945c Mon Sep 17 00:00:00 2001 From: kkraune Date: Thu, 24 Sep 2020 13:44:05 +0200 Subject: use US locale --- .../com/yahoo/vespa/model/application/validation/QuotaValidator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java index 7cd5e6b9b07..c0a2fe4fdf2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.VespaModel; +import java.util.Locale; import java.util.stream.Collectors; /** @@ -51,7 +52,7 @@ public class QuotaValidator extends Validator { } private void throwBudgetExceeded(double spend, double budget) { - var message = String.format("Hourly spend for maximum specified resources ($%.2f) exceeds budget from quota ($%.2f)!", spend, budget); + var message = String.format(Locale.US, "Hourly spend for maximum specified resources ($%.2f) exceeds budget from quota ($%.2f)!", spend, budget); throw new IllegalArgumentException(message); } } -- cgit v1.2.3 From a4e7bca517671223e797623c000d7bf4e9de8cc3 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Thu, 24 Sep 2020 13:52:53 +0200 Subject: Add internalRestart to prepareParams --- .../vespa/config/server/session/PrepareParams.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index d962218b63a..914c927698f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -45,6 +45,7 @@ public final class PrepareParams { static final String APPLICATION_CONTAINER_ROLE = "applicationContainerRole"; static final String QUOTA_PARAM_NAME = "quota"; static final String FORCE_PARAM_NAME = "force"; + static final String INTERNAL_RESTART_PARAM_NAME = "internalRestart"; private final ApplicationId applicationId; private final TimeoutBudget timeoutBudget; @@ -53,6 +54,7 @@ public final class PrepareParams { private final boolean verbose; private final boolean isBootstrap; private final boolean force; + private final boolean internalRestart; private final Optional vespaVersion; private final List containerEndpoints; private final Optional tlsSecretsKeyName; @@ -67,7 +69,8 @@ public final class PrepareParams { List containerEndpoints, Optional tlsSecretsKeyName, Optional endpointCertificateMetadata, Optional dockerImageRepository, Optional athenzDomain, - Optional applicationRoles, Optional quota, boolean force) { + Optional applicationRoles, Optional quota, boolean force, + boolean internalRestart) { this.timeoutBudget = timeoutBudget; this.applicationId = Objects.requireNonNull(applicationId); this.ignoreValidationErrors = ignoreValidationErrors; @@ -83,6 +86,7 @@ public final class PrepareParams { this.applicationRoles = applicationRoles; this.quota = quota; this.force = force; + this.internalRestart = internalRestart; } public static class Builder { @@ -92,6 +96,7 @@ public final class PrepareParams { private boolean verbose = false; private boolean isBootstrap = false; private boolean force = false; + private boolean internalRestart = false; private ApplicationId applicationId = null; private TimeoutBudget timeoutBudget = new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(60)); private Optional vespaVersion = Optional.empty(); @@ -208,11 +213,16 @@ public final class PrepareParams { return this; } + public Builder internalRestart(boolean internalRestart) { + this.internalRestart = internalRestart; + return this; + } + public PrepareParams build() { return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun, verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName, endpointCertificateMetadata, dockerImageRepository, athenzDomain, - applicationRoles, quota, force); + applicationRoles, quota, force, internalRestart); } } @@ -231,6 +241,7 @@ public final class PrepareParams { .applicationRoles(ApplicationRoles.fromString(request.getProperty(APPLICATION_HOST_ROLE), request.getProperty(APPLICATION_CONTAINER_ROLE))) .quota(request.getProperty(QUOTA_PARAM_NAME)) .force(request.getBooleanProperty(FORCE_PARAM_NAME)) + .internalRestart(request.getBooleanProperty(INTERNAL_RESTART_PARAM_NAME)) .build(); } @@ -282,6 +293,8 @@ public final class PrepareParams { public boolean force() { return force; } + public boolean internalRestart() { return internalRestart; } + public TimeoutBudget getTimeoutBudget() { return timeoutBudget; } -- cgit v1.2.3 From 069716d610dbea6fa58e009cac8c5495c5e7d06d Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Thu, 24 Sep 2020 13:53:40 +0200 Subject: Expose locks info in REST API --- .../hosted/provision/restapi/LocksResponse.java | 73 ++++++++++++ .../provision/restapi/NodesV2ApiHandler.java | 1 + zkfacade/abi-spec.json | 60 ++++++++++ .../main/java/com/yahoo/vespa/curator/Lock.java | 16 ++- .../java/com/yahoo/vespa/curator/LockInfo.java | 64 +++++++++++ .../com/yahoo/vespa/curator/ThreadLockInfo.java | 125 +++++++++++++++++++++ 6 files changed, 337 insertions(+), 2 deletions(-) create mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java create mode 100644 zkfacade/src/main/java/com/yahoo/vespa/curator/LockInfo.java create mode 100644 zkfacade/src/main/java/com/yahoo/vespa/curator/ThreadLockInfo.java diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java new file mode 100644 index 00000000000..817063592b1 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java @@ -0,0 +1,73 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.curator.LockInfo; +import com.yahoo.vespa.curator.ThreadLockInfo; + +import java.io.IOException; +import java.io.OutputStream; +import java.time.Instant; +import java.util.List; + +public class LocksResponse extends HttpResponse { + + private final Slime slime; + + public LocksResponse() { + super(200); + + this.slime = new Slime(); + Cursor root = slime.setObject(); + + root.setLong("in-critical-region", ThreadLockInfo.inCriticalRegionCount()); + root.setLong("invoke-acquire", ThreadLockInfo.invokeAcquireCount()); + root.setLong("acquire-timed-out", ThreadLockInfo.acquireTimedOutCount()); + root.setLong("lock-acquired", ThreadLockInfo.lockAcquiredCount()); + root.setLong("locks-released", ThreadLockInfo.locksReleasedCount()); + root.setLong("acquire-reentrant-lock-errors", ThreadLockInfo.failedToAcquireReentrantLockCount()); + root.setLong("no-locks-errors", ThreadLockInfo.noLocksErrorCount()); + root.setLong("timeout-on-reentrancy-errors", ThreadLockInfo.timeoutOnReentrancyErrorCount()); + + List threadLockInfos = ThreadLockInfo.getThreadLockInfos(); + if (!threadLockInfos.isEmpty()) { + Cursor threadsCursor = root.setArray("threads"); + threadLockInfos.forEach(threadLockInfo -> { + Cursor threadLockInfoCursor = threadsCursor.addObject(); + threadLockInfoCursor.setString("thread-name", threadLockInfo.getThreadName()); + threadLockInfoCursor.setString("lock-path", threadLockInfo.getLockPath()); + + List lockInfos = threadLockInfo.getLockInfos(); + if (!lockInfos.isEmpty()) { + Cursor lockInfosCursor = threadLockInfoCursor.setArray("locks"); + lockInfos.forEach(lockInfo -> { + Cursor lockInfoCursor = lockInfosCursor.addObject(); + lockInfoCursor.setString("invoke-acquire-time", toString(lockInfo.getTimeAcquiredWasInvoked())); + lockInfoCursor.setLong("reentrancy-hold-count-on-acquire", lockInfo.getThreadHoldCountOnAcquire()); + lockInfoCursor.setString("acquire-timeout", lockInfo.getAcquireTimeout().toString()); + lockInfo.getTimeLockWasAcquired().ifPresent(instant -> lockInfoCursor.setString("lock-acquired-time", toString(instant))); + lockInfoCursor.setString("lock-state", lockInfo.getLockState().name()); + lockInfo.getTimeTerminalStateWasReached().ifPresent(instant -> lockInfoCursor.setString("terminal-state-time", toString(instant))); + }); + } + }); + } + } + + @Override + public void render(OutputStream stream) throws IOException { + new JsonFormat(true).encode(stream, slime); + } + + @Override + public String getContentType() { + return "application/json"; + } + + private static String toString(Instant time) { + return Instant.ofEpochMilli(time.toEpochMilli()).toString(); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index 1891b9d2f82..a2d599eab6e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -116,6 +116,7 @@ public class NodesV2ApiHandler extends LoggingRequestHandler { if (pathS.startsWith("/nodes/v2/state/")) return new NodesResponse(ResponseType.nodesInStateList, request, orchestrator, nodeRepository); if (pathS.startsWith("/nodes/v2/acl/")) return new NodeAclResponse(request, nodeRepository); if (pathS.equals( "/nodes/v2/command/")) return new ResourceResponse(request.getUri(), "restart", "reboot"); + if (pathS.equals( "/nodes/v2/locks")) return new LocksResponse(); if (pathS.equals( "/nodes/v2/maintenance/")) return new JobsResponse(nodeRepository.jobControl()); if (pathS.equals( "/nodes/v2/upgrade/")) return new UpgradeResponse(nodeRepository.infrastructureVersions(), nodeRepository.osVersions(), nodeRepository.dockerImages()); if (pathS.startsWith("/nodes/v2/capacity")) return new HostCapacityResponse(nodeRepository, request); diff --git a/zkfacade/abi-spec.json b/zkfacade/abi-spec.json index f4ad1ab4372..614c58f3cf6 100644 --- a/zkfacade/abi-spec.json +++ b/zkfacade/abi-spec.json @@ -109,5 +109,65 @@ "public void close()" ], "fields": [] + }, + "com.yahoo.vespa.curator.LockInfo$LockState": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static com.yahoo.vespa.curator.LockInfo$LockState[] values()", + "public static com.yahoo.vespa.curator.LockInfo$LockState valueOf(java.lang.String)", + "public boolean isTerminal()" + ], + "fields": [ + "public static final enum com.yahoo.vespa.curator.LockInfo$LockState ACQUIRING", + "public static final enum com.yahoo.vespa.curator.LockInfo$LockState TIMED_OUT", + "public static final enum com.yahoo.vespa.curator.LockInfo$LockState ACQUIRED", + "public static final enum com.yahoo.vespa.curator.LockInfo$LockState FAILED_TO_REENTER", + "public static final enum com.yahoo.vespa.curator.LockInfo$LockState RELEASED" + ] + }, + "com.yahoo.vespa.curator.LockInfo": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public static com.yahoo.vespa.curator.LockInfo invokingAcquire(int, java.time.Duration)", + "public int getThreadHoldCountOnAcquire()", + "public java.time.Instant getTimeAcquiredWasInvoked()", + "public java.time.Duration getAcquireTimeout()", + "public com.yahoo.vespa.curator.LockInfo$LockState getLockState()", + "public java.util.Optional getTimeLockWasAcquired()", + "public java.util.Optional getTimeTerminalStateWasReached()" + ], + "fields": [] + }, + "com.yahoo.vespa.curator.ThreadLockInfo": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public static int invokeAcquireCount()", + "public static int inCriticalRegionCount()", + "public static int acquireTimedOutCount()", + "public static int lockAcquiredCount()", + "public static int locksReleasedCount()", + "public static int noLocksErrorCount()", + "public static int failedToAcquireReentrantLockCount()", + "public static int timeoutOnReentrancyErrorCount()", + "public static java.util.List getThreadLockInfos()", + "public java.lang.String getThreadName()", + "public java.lang.String getLockPath()", + "public java.util.List getLockInfos()" + ], + "fields": [] } } \ No newline at end of file diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java index d97d8f5ed71..299fceb8d7f 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java @@ -32,12 +32,20 @@ public class Lock implements Mutex { /** Take the lock with the given timeout. This may be called multiple times from the same thread - each matched by a close */ public void acquire(Duration timeout) throws UncheckedTimeoutException { + ThreadLockInfo threadLockInfo = getThreadLockInfo(); + threadLockInfo.invokingAcquire(timeout); try { - if ( ! mutex.acquire(timeout.toMillis(), TimeUnit.MILLISECONDS)) + if ( ! mutex.acquire(timeout.toMillis(), TimeUnit.MILLISECONDS)) { + threadLockInfo.acquireTimedOut(); + throw new UncheckedTimeoutException("Timed out after waiting " + timeout + - " to acquire lock '" + lockPath + "'"); + " to acquire lock '" + lockPath + "'"); + } + threadLockInfo.lockAcquired(); + if ( ! lock.tryLock()) { // Should be available to only this thread, while holding the above mutex. release(); + threadLockInfo.failedToAcquireReentrantLock(); throw new IllegalStateException("InterProcessMutex acquired, but guarded lock held by someone else, for lock '" + lockPath + "'"); } } @@ -60,6 +68,7 @@ public class Lock implements Mutex { } private void release() { + getThreadLockInfo().lockReleased(); try { mutex.release(); } @@ -68,6 +77,9 @@ public class Lock implements Mutex { } } + private ThreadLockInfo getThreadLockInfo() { + return ThreadLockInfo.getCurrentThreadLockInfo(lockPath, lock); + } } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/LockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/LockInfo.java new file mode 100644 index 00000000000..870ee12ebda --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/LockInfo.java @@ -0,0 +1,64 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator; + +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; + +/** + * Information about a lock. + * + *

Should be mutated by a single thread. Other threads may see an inconsistent state of this instance.

+ */ +public class LockInfo { + + private final int threadHoldCountOnAcquire; + private final Instant acquireInstant; + private final Duration timeout; + + private volatile Optional lockAcquiredInstant = Optional.empty(); + private volatile Optional terminalStateInstant = Optional.empty(); + + public static LockInfo invokingAcquire(int holdCount, Duration timeout) { + return new LockInfo(holdCount, timeout); + } + + public enum LockState { + ACQUIRING(false), TIMED_OUT(true), ACQUIRED(false), FAILED_TO_REENTER(true), RELEASED(true); + + private final boolean terminal; + + LockState(boolean terminal) { this.terminal = terminal; } + + public boolean isTerminal() { return terminal; } + } + + private volatile LockState lockState = LockState.ACQUIRING; + + private LockInfo(int threadHoldCountOnAcquire, Duration timeout) { + this.threadHoldCountOnAcquire = threadHoldCountOnAcquire; + this.acquireInstant = Instant.now(); + this.timeout = timeout; + } + + public int getThreadHoldCountOnAcquire() { return threadHoldCountOnAcquire; } + public Instant getTimeAcquiredWasInvoked() { return acquireInstant; } + public Duration getAcquireTimeout() { return timeout; } + public LockState getLockState() { return lockState; } + public Optional getTimeLockWasAcquired() { return lockAcquiredInstant; } + public Optional getTimeTerminalStateWasReached() { return terminalStateInstant; } + + void timedOut() { setTerminalState(LockState.TIMED_OUT); } + void failedToAcquireReentrantLock() { setTerminalState(LockState.FAILED_TO_REENTER); } + void released() { setTerminalState(LockState.RELEASED); } + + void lockAcquired() { + lockState = LockState.ACQUIRED; + lockAcquiredInstant = Optional.of(Instant.now()); + } + + void setTerminalState(LockState terminalState) { + lockState = terminalState; + terminalStateInstant = Optional.of(Instant.now()); + } +} diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/ThreadLockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/ThreadLockInfo.java new file mode 100644 index 00000000000..9fbcb550ddd --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/ThreadLockInfo.java @@ -0,0 +1,125 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator; + +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; + +/** + * This class contains process-wide statistics and information related to acquiring and releasing + * {@link Lock}. Instances of this class contain information tied to a specific thread and lock path. + * + *

Instances of this class are thread-safe as long as foreign threads (!= this.thread) avoid mutable methods.

+ */ +public class ThreadLockInfo { + + private static final ConcurrentHashMap locks = new ConcurrentHashMap<>(); + + private static final int MAX_COMPLETED_LOCK_INFOS_SIZE = 10; + private static final ConcurrentLinkedDeque completedLockInfos = new ConcurrentLinkedDeque<>(); + + private static final AtomicInteger invokeAcquireCount = new AtomicInteger(0); + private static final AtomicInteger inCriticalRegionCount = new AtomicInteger(0); + private static final AtomicInteger acquireTimedOutCount = new AtomicInteger(0); + private static final AtomicInteger lockAcquiredCount = new AtomicInteger(0); + private static final AtomicInteger locksReleasedCount = new AtomicInteger(0); + + private static final AtomicInteger failedToAcquireReentrantLockCount = new AtomicInteger(0); + private static final AtomicInteger noLocksErrorCount = new AtomicInteger(0); + private static final AtomicInteger timeoutOnReentrancyErrorCount = new AtomicInteger(0); + + private final Thread thread; + private final String lockPath; + private final ReentrantLock lock; + + /** The locks are reentrant so there may be more than 1 lock for this thread. */ + private final ConcurrentLinkedQueue lockInfos = new ConcurrentLinkedQueue<>(); + + public static int invokeAcquireCount() { return invokeAcquireCount.get(); } + public static int inCriticalRegionCount() { return inCriticalRegionCount.get(); } + public static int acquireTimedOutCount() { return acquireTimedOutCount.get(); } + public static int lockAcquiredCount() { return lockAcquiredCount.get(); } + public static int locksReleasedCount() { return locksReleasedCount.get(); } + public static int noLocksErrorCount() { return noLocksErrorCount.get(); } + public static int failedToAcquireReentrantLockCount() { return failedToAcquireReentrantLockCount.get(); } + public static int timeoutOnReentrancyErrorCount() { return timeoutOnReentrancyErrorCount.get(); } + public static List getThreadLockInfos() { return List.copyOf(locks.values()); } + + /** Returns the per-thread singleton ThreadLockInfo. */ + static ThreadLockInfo getCurrentThreadLockInfo(String lockPath, ReentrantLock lock) { + return locks.computeIfAbsent( + Thread.currentThread(), + currentThread -> new ThreadLockInfo(currentThread, lockPath, lock)); + } + + ThreadLockInfo(Thread currentThread, String lockPath, ReentrantLock lock) { + this.thread = currentThread; + this.lockPath = lockPath; + this.lock = lock; + } + + public String getThreadName() { return thread.getName(); } + public String getLockPath() { return lockPath; } + public List getLockInfos() { return List.copyOf(lockInfos); } + + /** Mutable method (see class doc) */ + void invokingAcquire(Duration timeout) { + invokeAcquireCount.incrementAndGet(); + inCriticalRegionCount.incrementAndGet(); + lockInfos.add(LockInfo.invokingAcquire(lock.getHoldCount(), timeout)); + } + + /** Mutable method (see class doc) */ + void acquireTimedOut() { + if (lockInfos.size() > 1) { + timeoutOnReentrancyErrorCount.incrementAndGet(); + } + + removeLastLockInfo(acquireTimedOutCount, LockInfo::timedOut); + } + + /** Mutable method (see class doc) */ + void lockAcquired() { + lockAcquiredCount.incrementAndGet(); + getLastLockInfo().ifPresent(LockInfo::lockAcquired); + } + + /** Mutable method (see class doc) */ + void failedToAcquireReentrantLock() { + removeLastLockInfo(failedToAcquireReentrantLockCount, LockInfo::failedToAcquireReentrantLock); + } + + /** Mutable method (see class doc) */ + void lockReleased() { + removeLastLockInfo(locksReleasedCount, LockInfo::released); + } + + private Optional getLastLockInfo() { + return lockInfos.isEmpty() ? Optional.empty() : Optional.of(lockInfos.peek()); + } + + private void removeLastLockInfo(AtomicInteger metricToIncrement, Consumer completeLockInfo) { + metricToIncrement.incrementAndGet(); + inCriticalRegionCount.decrementAndGet(); + + if (lockInfos.isEmpty()) { + noLocksErrorCount.incrementAndGet(); + return; + } + + LockInfo lockInfo = lockInfos.poll(); + completeLockInfo.accept(lockInfo); + + if (completedLockInfos.size() >= MAX_COMPLETED_LOCK_INFOS_SIZE) { + // This is thread-safe, as no-one but currentThread mutates completedLockInfos + completedLockInfos.removeLast(); + } + completedLockInfos.addFirst(lockInfo); + } +} -- cgit v1.2.3 From 6c6e773f39bf88b66ef1b4d0782f9a0025b244a0 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Thu, 24 Sep 2020 13:54:32 +0200 Subject: Restart nodes after activate --- .../yahoo/vespa/config/server/ApplicationRepository.java | 15 +++++++++++++++ .../config/server/configchange/ConfigChangeActions.java | 12 ++++++++---- .../vespa/config/server/configchange/RestartActions.java | 2 +- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 3564a6e6da7..fb981eb1c67 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -313,11 +313,26 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } public PrepareResult deploy(File applicationPackage, PrepareParams prepareParams, Instant now) { + if (prepareParams.internalRestart() && hostProvisioner.isEmpty()) + throw new IllegalArgumentException("Internal restart not supported without HostProvisioner"); + ApplicationId applicationId = prepareParams.getApplicationId(); long sessionId = createSession(applicationId, prepareParams.getTimeoutBudget(), applicationPackage); Tenant tenant = getTenant(applicationId); PrepareResult result = prepare(tenant, sessionId, prepareParams, now); activate(tenant, sessionId, prepareParams.getTimeoutBudget(), prepareParams.force()); + + if (prepareParams.internalRestart() && !result.configChangeActions().getRestartActions().isEmpty()) { + Set hostnames = result.configChangeActions().getRestartActions().getEntries().stream() + .flatMap(entry -> entry.getServices().stream()) + .map(ServiceInfo::getHostName) + .collect(Collectors.toUnmodifiableSet()); + + hostProvisioner.get().restart(applicationId, HostFilter.from(hostnames, Set.of(), Set.of(), Set.of())); + ConfigChangeActions newActions = new ConfigChangeActions(new RestartActions(), result.configChangeActions().getRefeedActions()); + return new PrepareResult(result.sessionId(), newActions, result.deployLog()); + } + return result; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActions.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActions.java index d783dc105c3..dd9c8e4b6bb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActions.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActions.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.configchange; import com.yahoo.config.model.api.ConfigChangeAction; import java.util.List; +import java.util.Objects; /** * Contains an aggregated view of which actions that must be performed to handle config @@ -18,13 +19,16 @@ public class ConfigChangeActions { private final RefeedActions refeedActions; public ConfigChangeActions() { - this.restartActions = new RestartActions(); - this.refeedActions = new RefeedActions(); + this(new RestartActions(), new RefeedActions()); } public ConfigChangeActions(List actions) { - this.restartActions = new RestartActions(actions); - this.refeedActions = new RefeedActions(actions); + this(new RestartActions(actions), new RefeedActions(actions)); + } + + public ConfigChangeActions(RestartActions restartActions, RefeedActions refeedActions) { + this.restartActions = Objects.requireNonNull(restartActions); + this.refeedActions = Objects.requireNonNull(refeedActions); } public RestartActions getRestartActions() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java index 29b0b99e42e..4db5a5e125f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java @@ -97,6 +97,6 @@ public class RestartActions { } public boolean isEmpty() { - return getEntries().isEmpty(); + return actions.isEmpty(); } } -- cgit v1.2.3 From 0d3b36d44a88084dd55a33d3995a5f5d39c89a0c Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Thu, 24 Sep 2020 13:55:44 +0200 Subject: Create DeployTester builder --- .../config/server/ApplicationRepositoryTest.java | 2 +- .../config/server/ConfigServerBootstrapTest.java | 23 +-- .../vespa/config/server/deploy/DeployTester.java | 165 +++++++++++++-------- .../config/server/deploy/HostedDeployTest.java | 26 ++-- .../vespa/config/server/deploy/RedeployTest.java | 4 +- 5 files changed, 135 insertions(+), 85 deletions(-) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 3789e540fc3..0a49a19d728 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -384,7 +384,7 @@ public class ApplicationRepositoryTest { .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath()) .fileReferencesDir(temporaryFolder.newFolder("filedistribution").getAbsolutePath()) .sessionLifetime(60)); - DeployTester tester = new DeployTester(configserverConfig, clock); + DeployTester tester = new DeployTester.Builder().configserverConfig(configserverConfig).clock(clock).build(); tester.deployApp("src/test/apps/app", clock.instant()); // session 2 (numbering starts at 2) clock.advance(Duration.ofSeconds(10)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index bda17faec12..e407e2efc41 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.config.model.api.ModelFactory; import com.yahoo.config.model.provision.Host; import com.yahoo.config.model.provision.Hosts; import com.yahoo.config.model.provision.InMemoryProvisioner; @@ -32,7 +31,6 @@ import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; import java.nio.file.Paths; -import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.Arrays; @@ -63,7 +61,8 @@ public class ConfigServerBootstrapTest { public void testBootstrap() throws Exception { ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder); InMemoryProvisioner provisioner = new InMemoryProvisioner(true, "host0", "host1", "host3", "host4"); - DeployTester tester = new DeployTester(List.of(createHostedModelFactory()), configserverConfig, provisioner); + DeployTester tester = new DeployTester.Builder().modelFactory(createHostedModelFactory()) + .configserverConfig(configserverConfig).hostProvisioner(provisioner).build(); tester.deployApp("src/test/apps/hosted/"); File versionFile = temporaryFolder.newFile(); @@ -96,7 +95,8 @@ public class ConfigServerBootstrapTest { public void testBootstrapWithVipStatusFile() throws Exception { ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder); InMemoryProvisioner provisioner = new InMemoryProvisioner(true, "host0", "host1", "host3", "host4"); - DeployTester tester = new DeployTester(List.of(createHostedModelFactory()), configserverConfig, provisioner); + DeployTester tester = new DeployTester.Builder().modelFactory(createHostedModelFactory()) + .configserverConfig(configserverConfig).hostProvisioner(provisioner).build(); tester.deployApp("src/test/apps/hosted/"); File versionFile = temporaryFolder.newFile(); @@ -121,7 +121,8 @@ public class ConfigServerBootstrapTest { @Test public void testBootstrapWhenRedeploymentFails() throws Exception { ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder); - DeployTester tester = new DeployTester(List.of(createHostedModelFactory()), configserverConfig); + DeployTester tester = new DeployTester.Builder().modelFactory(createHostedModelFactory()) + .configserverConfig(configserverConfig).build(); tester.deployApp("src/test/apps/hosted/"); File versionFile = temporaryFolder.newFile(); @@ -158,13 +159,15 @@ public class ConfigServerBootstrapTest { public void testBootstrapNonHostedOneConfigModel() throws Exception { ConfigserverConfig configserverConfig = createConfigserverConfigNonHosted(temporaryFolder); String vespaVersion = "1.2.3"; - List modelFactories = Collections.singletonList(DeployTester.createModelFactory(Version.fromString(vespaVersion))); List hosts = createHosts(vespaVersion); - InMemoryProvisioner provisioner = new InMemoryProvisioner(new Hosts(hosts), true); Curator curator = new MockCurator(); - DeployTester tester = new DeployTester(modelFactories, configserverConfig, - Clock.systemUTC(), new Zone(Environment.dev, RegionName.defaultName()), - provisioner, curator); + DeployTester tester = new DeployTester.Builder() + .modelFactory(DeployTester.createModelFactory(Version.fromString(vespaVersion))) + .hostProvisioner(new InMemoryProvisioner(new Hosts(hosts), true)) + .configserverConfig(configserverConfig) + .zone(new Zone(Environment.dev, RegionName.defaultName())) + .curator(curator) + .build(); tester.deployApp("src/test/apps/app/", vespaVersion, Instant.now()); ApplicationId applicationId = tester.applicationId(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index b657ff2a5f8..3b6e6c00d8b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -42,6 +42,7 @@ import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.VespaModelFactory; +import com.yahoo.vespa.orchestrator.Orchestrator; import java.io.File; import java.nio.file.Files; @@ -70,73 +71,16 @@ public class DeployTester { private final TenantRepository tenantRepository; private final ApplicationRepository applicationRepository; - public DeployTester() { - this(Collections.singletonList(createModelFactory(Clock.systemUTC()))); - } - - public DeployTester(List modelFactories) { - this(modelFactories, - new ConfigserverConfig(new ConfigserverConfig.Builder() - .configServerDBDir(uncheck(() -> Files.createTempDirectory("serverdb")).toString()) - .configDefinitionsDir(uncheck(() -> Files.createTempDirectory("configdefinitions")).toString()) - .fileReferencesDir(uncheck(() -> Files.createTempDirectory("configdefinitions")).toString())), - Clock.systemUTC()); - } - - public DeployTester(ConfigserverConfig configserverConfig, Clock clock) { - this(Collections.singletonList(createModelFactory(clock)), configserverConfig, clock); - } - - public DeployTester(List modelFactories, ConfigserverConfig configserverConfig) { - this(modelFactories, configserverConfig, Clock.systemUTC()); - } - - public DeployTester(List modelFactories, ConfigserverConfig configserverConfig, Clock clock) { - this(modelFactories, configserverConfig, clock, Zone.defaultZone()); - } - - public DeployTester(List modelFactories, ConfigserverConfig configserverConfig, HostProvisioner hostProvisioner) { - this(modelFactories, configserverConfig, Clock.systemUTC(), hostProvisioner); - } - - public DeployTester(List modelFactories, ConfigserverConfig configserverConfig, Clock clock, HostProvisioner provisioner) { - this(modelFactories, configserverConfig, clock, Zone.defaultZone(), provisioner); - } - - public DeployTester(List modelFactories, ConfigserverConfig configserverConfig, Clock clock, Zone zone) { - this(modelFactories, configserverConfig, clock, zone, createProvisioner()); - } - - public DeployTester(List modelFactories, ConfigserverConfig configserverConfig, Clock clock, Zone zone, HostProvisioner provisioner) { - this(modelFactories, configserverConfig, clock, zone, provisioner, new MockCurator()); - } - - public DeployTester(List modelFactories, ConfigserverConfig configserverConfig, Clock clock, Zone zone, - HostProvisioner provisioner, Curator curator) { + private DeployTester(Clock clock, TenantRepository tenantRepository, ApplicationRepository applicationRepository) { this.clock = clock; - TestComponentRegistry componentRegistry = createComponentRegistry(curator, Metrics.createTestMetrics(), - modelFactories, configserverConfig, clock, zone, - provisioner); - try { - this.tenantRepository = new TenantRepository(componentRegistry); - tenantRepository.addTenant(tenantName); - } - catch (Exception e) { - throw new IllegalArgumentException(e); - } - applicationRepository = new ApplicationRepository.Builder() - .withTenantRepository(tenantRepository) - .withProvisioner(new ProvisionerAdapter(provisioner)) - .withConfigserverConfig(configserverConfig) - .withOrchestrator(new OrchestratorMock()) - .withClock(clock) - .build(); + this.tenantRepository = tenantRepository; + this.applicationRepository = applicationRepository; } public Tenant tenant() { return tenantRepository.getTenant(tenantName); } - + /** Create a model factory for the version of this source*/ public static CountingModelFactory createModelFactory(Clock clock) { return new CountingModelFactory(clock); @@ -297,11 +241,11 @@ public class DeployTester { private static class FailingModelFactory implements ModelFactory { private final Version version; - + public FailingModelFactory(Version version) { this.version = version; } - + @Override public Version version() { return version; } @@ -370,4 +314,99 @@ public class DeployTester { } + public static class Builder { + private Clock clock; + private Provisioner provisioner; + private ConfigserverConfig configserverConfig; + private Zone zone; + private Curator curator; + private Metrics metrics; + private List modelFactories; + private Orchestrator orchestrator; + + public DeployTester build() { + Clock clock = Optional.ofNullable(this.clock).orElseGet(Clock::systemUTC); + Zone zone = Optional.ofNullable(this.zone).orElseGet(Zone::defaultZone); + ConfigserverConfig configserverConfig = Optional.ofNullable(this.configserverConfig) + .orElseGet(() -> new ConfigserverConfig(new ConfigserverConfig.Builder() + .configServerDBDir(uncheck(() -> Files.createTempDirectory("serverdb")).toString()) + .configDefinitionsDir(uncheck(() -> Files.createTempDirectory("configdefinitions")).toString()) + .fileReferencesDir(uncheck(() -> Files.createTempDirectory("configdefinitions")).toString()))); + Provisioner provisioner = Optional.ofNullable(this.provisioner) + .orElseGet(() -> new ProvisionerAdapter(createProvisioner())); + List modelFactories = Optional.ofNullable(this.modelFactories) + .orElseGet(() -> List.of(createModelFactory(clock))); + + TestComponentRegistry.Builder testComponentRegistryBuilder = new TestComponentRegistry.Builder() + .clock(clock) + .configServerConfig(configserverConfig) + .curator(Optional.ofNullable(curator).orElseGet(MockCurator::new)) + .modelFactoryRegistry(new ModelFactoryRegistry(modelFactories)) + .metrics(Optional.ofNullable(metrics).orElseGet(Metrics::createTestMetrics)) + .zone(zone); + if (configserverConfig.hostedVespa()) testComponentRegistryBuilder.provisioner(provisioner); + + TenantRepository tenantRepository = new TenantRepository(testComponentRegistryBuilder.build()); + tenantRepository.addTenant(tenantName); + + ApplicationRepository applicationRepository = new ApplicationRepository.Builder() + .withTenantRepository(tenantRepository) + .withConfigserverConfig(configserverConfig) + .withOrchestrator(Optional.ofNullable(orchestrator).orElseGet(OrchestratorMock::new)) + .withClock(clock) + .withProvisioner(provisioner) + .build(); + + return new DeployTester(clock, tenantRepository, applicationRepository); + } + + public Builder clock(Clock clock) { + this.clock = clock; + return this; + } + + public Builder provisioner(Provisioner provisioner) { + this.provisioner = provisioner; + return this; + } + + public Builder hostProvisioner(HostProvisioner hostProvisioner) { + return provisioner(new ProvisionerAdapter(hostProvisioner)); + } + + public Builder configserverConfig(ConfigserverConfig configserverConfig) { + this.configserverConfig = configserverConfig; + return this; + } + + public Builder zone(Zone zone) { + this.zone = zone; + return this; + } + + public Builder curator(Curator curator) { + this.curator = curator; + return this; + } + + public Builder metrics(Metrics metrics) { + this.metrics = metrics; + return this; + } + + public Builder modelFactory(ModelFactory modelFactory) { + return modelFactories(List.of(modelFactory)); + } + + public Builder modelFactories(List modelFactories) { + this.modelFactories = modelFactories; + return this; + } + + public Builder orchestrator(Orchestrator orchestrator) { + this.orchestrator = orchestrator; + return this; + } + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index 5b995ce55e6..04d018b71c0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -65,8 +65,9 @@ public class HostedDeployTest { @Test public void testRedeployWithVersion() throws IOException { - CountingModelFactory modelFactory = createHostedModelFactory(Version.fromString("4.5.6"), Clock.systemUTC()); - DeployTester tester = new DeployTester(List.of(modelFactory), createConfigserverConfig()); + DeployTester tester = new DeployTester.Builder() + .modelFactory(createHostedModelFactory(Version.fromString("4.5.6"), Clock.systemUTC())) + .configserverConfig(createConfigserverConfig()).build(); tester.deployApp("src/test/apps/hosted/", "4.5.6"); Optional deployment = tester.redeployFromLocalActive(tester.applicationId()); @@ -77,7 +78,9 @@ public class HostedDeployTest { @Test public void testRedeploy() throws IOException { - DeployTester tester = new DeployTester(List.of(createHostedModelFactory()), createConfigserverConfig()); + DeployTester tester = new DeployTester.Builder() + .modelFactory(createHostedModelFactory()) + .configserverConfig(createConfigserverConfig()).build(); ApplicationId appId = tester.applicationId(); tester.deployApp("src/test/apps/hosted/"); assertFalse(tester.applicationRepository().getActiveSession(appId).getMetaData().isInternalRedeploy()); @@ -90,8 +93,9 @@ public class HostedDeployTest { @Test public void testReDeployWithWantedDockerImageRepositoryAndAthenzDomain() throws IOException { - CountingModelFactory modelFactory = createHostedModelFactory(Version.fromString("4.5.6"), Clock.systemUTC()); - DeployTester tester = new DeployTester(List.of(modelFactory), createConfigserverConfig()); + DeployTester tester = new DeployTester.Builder() + .modelFactory(createHostedModelFactory(Version.fromString("4.5.6"), Clock.systemUTC())) + .configserverConfig(createConfigserverConfig()).build(); String dockerImageRepository = "docker.foo.com:4443/bar/baz"; tester.deployApp("src/test/apps/hosted/", Instant.now(), new PrepareParams.Builder() .vespaVersion("4.5.6") @@ -111,7 +115,7 @@ public class HostedDeployTest { List modelFactories = List.of(createHostedModelFactory(Version.fromString("6.1.0")), createHostedModelFactory(Version.fromString("6.2.0")), createHostedModelFactory(Version.fromString("7.0.0"))); - DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig()); + DeployTester tester = new DeployTester.Builder().modelFactories(modelFactories).configserverConfig(createConfigserverConfig()).build(); tester.deployApp("src/test/apps/hosted/", "6.2.0"); assertEquals(4, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); } @@ -332,7 +336,7 @@ public class HostedDeployTest { ManualClock clock = new ManualClock("2016-10-09T00:00:00"); List modelFactories = List.of(createHostedModelFactory(clock), createFailingModelFactory(Version.fromString("1.0.0"))); // older than default - DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig()); + DeployTester tester = new DeployTester.Builder().modelFactories(modelFactories).configserverConfig(createConfigserverConfig()).build(); tester.deployApp("src/test/apps/validationOverride/", clock.instant()); // Redeployment from local active works @@ -416,8 +420,12 @@ public class HostedDeployTest { private DeployTester createTester(List hosts, List modelFactories, Zone prodZone, Clock clock) throws IOException { - return new DeployTester(modelFactories, createConfigserverConfig(prodZone), - clock, prodZone, new InMemoryProvisioner(new Hosts(hosts), true)); + return new DeployTester.Builder() + .modelFactories(modelFactories) + .configserverConfig(createConfigserverConfig(prodZone)) + .clock(clock) + .zone(prodZone) + .hostProvisioner(new InMemoryProvisioner(new Hosts(hosts), true)).build(); } private static class ConfigChangeActionsModelFactory extends TestModelFactory { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java index c07c7316930..015cc039a1c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java @@ -27,7 +27,7 @@ public class RedeployTest { @Test public void testRedeploy() { - DeployTester tester = new DeployTester(); + DeployTester tester = new DeployTester.Builder().build(); tester.deployApp("src/test/apps/app"); Optional deployment = tester.redeployFromLocalActive(); @@ -45,7 +45,7 @@ public class RedeployTest { public void testNoRedeploy() { List modelFactories = List.of(createModelFactory(Clock.systemUTC()), createFailingModelFactory(Version.fromString("1.0.0"))); - DeployTester tester = new DeployTester(modelFactories); + DeployTester tester = new DeployTester.Builder().modelFactories(modelFactories).build(); ApplicationId id = ApplicationId.from(tester.tenant().getName(), ApplicationName.from("default"), InstanceName.from("default")); -- cgit v1.2.3 From 00b8a48ebdecacec264b3a457557ca723ec88a21 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Thu, 24 Sep 2020 14:11:30 +0200 Subject: Add flag for adjusting cache size --- flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 6 ++++++ .../main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java | 3 ++- .../vespa/hosted/provision/persistence/CuratorDatabaseClient.java | 5 +++-- .../yahoo/vespa/hosted/provision/persistence/NodeSerializer.java | 5 +++-- .../hosted/provision/persistence/CuratorDatabaseClientTest.java | 2 +- .../vespa/hosted/provision/persistence/NodeSerializerTest.java | 2 +- 6 files changed, 16 insertions(+), 7 deletions(-) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index ce9063dbcce..6ed98d169c9 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -404,6 +404,12 @@ public class Flags { "Takes effect at next run of maintainer", APPLICATION_ID); + public static final UnboundLongFlag NODE_OBJECT_CACHE_SIZE = defineLongFlag( + "node-object-cache-size", + 1000, + "The number of deserialized Node objects to store in-memory.", + "Takes effect on config server restart"); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 8523db7a970..e70793111ea 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -155,7 +155,8 @@ public class NodeRepository extends AbstractComponent { int spareCount) { // Flag is read once here as it shouldn't not change at runtime this.useConfigServerLock = Flags.USE_CONFIG_SERVER_LOCK.bindTo(flagSource).value(); - this.db = new CuratorDatabaseClient(flavors, curator, clock, zone, useCuratorClientCache, useConfigServerLock); + long nodeObjectCacheSize = Flags.NODE_OBJECT_CACHE_SIZE.bindTo(flagSource).value(); + this.db = new CuratorDatabaseClient(flavors, curator, clock, zone, useCuratorClientCache, useConfigServerLock, nodeObjectCacheSize); this.zone = zone; this.clock = clock; this.flavors = flavors; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index ed3c345a6bd..a7386d74c23 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -79,8 +79,9 @@ public class CuratorDatabaseClient { private final CuratorCounter provisionIndexCounter; private final boolean logStackTracesOnLockTimeout; - public CuratorDatabaseClient(NodeFlavors flavors, Curator curator, Clock clock, Zone zone, boolean useCache, boolean logStackTracesOnLockTimeout) { - this.nodeSerializer = new NodeSerializer(flavors); + public CuratorDatabaseClient(NodeFlavors flavors, Curator curator, Clock clock, Zone zone, boolean useCache, boolean logStackTracesOnLockTimeout, + long nodeObjectCacheSize) { + this.nodeSerializer = new NodeSerializer(flavors, nodeObjectCacheSize); this.zone = zone; this.db = new CuratorDatabase(curator, root, useCache); this.clock = clock; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index ff2c1bbcf86..064a9d5d933 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -117,12 +117,13 @@ public class NodeSerializer { // // Deserializing a Node from slime is expensive, and happens frequently. Node instances that have already been // deserialized are returned from this cache instead of being deserialized again. - private final Cache cache = CacheBuilder.newBuilder().maximumSize(1000).build(); + private final Cache cache; // ---------------- Serialization ---------------------------------------------------- - public NodeSerializer(NodeFlavors flavors) { + public NodeSerializer(NodeFlavors flavors, long cacheSize) { this.flavors = flavors; + this.cache = CacheBuilder.newBuilder().maximumSize(cacheSize).build(); } public byte[] toJson(Node node) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java index b3d567c8d58..c7535f04c4f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java @@ -26,7 +26,7 @@ public class CuratorDatabaseClientTest { private final Curator curator = new MockCurator(); private final CuratorDatabaseClient zkClient = new CuratorDatabaseClient( - FlavorConfigBuilder.createDummies("default"), curator, Clock.systemUTC(), Zone.defaultZone(), true, false); + FlavorConfigBuilder.createDummies("default"), curator, Clock.systemUTC(), Zone.defaultZone(), true, false, 1000); @Test public void can_read_stored_host_information() throws Exception { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java index 97c9bef0dd0..1c854068b27 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java @@ -53,7 +53,7 @@ import static org.junit.Assert.assertTrue; public class NodeSerializerTest { private final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default", "large", "ugccloud-container"); - private final NodeSerializer nodeSerializer = new NodeSerializer(nodeFlavors); + private final NodeSerializer nodeSerializer = new NodeSerializer(nodeFlavors, 1000); private final ManualClock clock = new ManualClock(); @Test -- cgit v1.2.3 From b859701325b84fe7372c0d17c729c349a14de2d2 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Thu, 24 Sep 2020 14:14:23 +0200 Subject: Remove debug logging and fallback to file if we fail to read from zk --- .../java/com/yahoo/vespa/config/server/version/VersionState.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/version/VersionState.java b/configserver/src/main/java/com/yahoo/vespa/config/server/version/VersionState.java index a591ca07e5e..b0bc8dc90dd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/version/VersionState.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/version/VersionState.java @@ -46,7 +46,6 @@ public class VersionState { } public boolean isUpgraded() { - System.out.println("current version: " + currentVersion() + ", stored version: " + storedVersion()); return currentVersion().compareTo(storedVersion()) > 0; } @@ -67,16 +66,14 @@ public class VersionState { if (distributeApplicationPackage.value()) { Optional version = curator.getData(versionPath); if(version.isPresent()) { - System.out.println("Found version in zk "); try { return Version.fromString(Utf8.toString(version.get())); } catch (Exception e) { - return new Version(0, 0, 0); // Use an old value to signal we don't know + // continue, use value in file } } } try (FileReader reader = new FileReader(versionFile)) { - System.out.println("Found version in file "); return Version.fromString(IOUtils.readAll(reader)); } catch (Exception e) { return new Version(0, 0, 0); // Use an old value to signal we don't know -- cgit v1.2.3 From cee32bc63ccea339eadaf9b92335d783c86fc500 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Thu, 24 Sep 2020 06:55:50 +0000 Subject: add new generic rename and test --- eval/CMakeLists.txt | 2 + .../instruction/generic_rename/CMakeLists.txt | 9 + .../generic_rename/generic_rename_test.cpp | 128 +++++++++++++ eval/src/vespa/eval/CMakeLists.txt | 1 + eval/src/vespa/eval/instruction/CMakeLists.txt | 6 + eval/src/vespa/eval/instruction/generic_rename.cpp | 197 +++++++++++++++++++++ eval/src/vespa/eval/instruction/generic_rename.h | 77 ++++++++ 7 files changed, 420 insertions(+) create mode 100644 eval/src/tests/instruction/generic_rename/CMakeLists.txt create mode 100644 eval/src/tests/instruction/generic_rename/generic_rename_test.cpp create mode 100644 eval/src/vespa/eval/instruction/CMakeLists.txt create mode 100644 eval/src/vespa/eval/instruction/generic_rename.cpp create mode 100644 eval/src/vespa/eval/instruction/generic_rename.h diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index c69425d5387..cb0a3b3bc68 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -32,6 +32,7 @@ vespa_define_module( src/tests/eval/value_codec src/tests/eval/value_type src/tests/gp/ponder_nov2017 + src/tests/instruction/generic_rename src/tests/tensor/dense_add_dimension_optimizer src/tests/tensor/dense_dimension_combiner src/tests/tensor/dense_dot_product_function @@ -72,6 +73,7 @@ vespa_define_module( src/vespa/eval/eval/test src/vespa/eval/eval/value_cache src/vespa/eval/gp + src/vespa/eval/instruction src/vespa/eval/tensor src/vespa/eval/tensor/dense src/vespa/eval/tensor/mixed diff --git a/eval/src/tests/instruction/generic_rename/CMakeLists.txt b/eval/src/tests/instruction/generic_rename/CMakeLists.txt new file mode 100644 index 00000000000..98af0fe0212 --- /dev/null +++ b/eval/src/tests/instruction/generic_rename/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_generic_rename_test_app TEST + SOURCES + generic_rename_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_generic_rename_test_app COMMAND eval_generic_rename_test_app) diff --git a/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp b/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp new file mode 100644 index 00000000000..31e82a01070 --- /dev/null +++ b/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp @@ -0,0 +1,128 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include +#include +#include +#include +#include +#include +#include + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::eval::test; + +using vespalib::make_string_short::fmt; + +std::vector rename_layouts = { + {x(3)}, + {x(3),y(5)}, + {x(3),y(5),z(7)}, + float_cells({x(3),y(5),z(7)}), + {x({"a","b","c"})}, + {x({"a","b","c"}),y({"foo","bar"})}, + {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), + {x(3),y({"foo", "bar"}),z(7)}, + {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) +}; + +struct FromTo { + std::vector from; + std::vector to; +}; + +std::vector rename_from_to = { + { {"x"}, {"x_renamed"} }, + { {"x"}, {"z_was_x"} }, + { {"x", "y"}, {"y", "x"} }, + { {"x", "z"}, {"z", "x"} }, + { {"x", "y", "z"}, {"a", "b", "c"} }, + { {"z"}, {"a"} }, + { {"y"}, {"z_was_y"} }, + { {"y"}, {"b"} } +}; + + +TEST(GenericRenameTest, dense_rename_plan_can_be_created_and_executed) { + auto lhs = ValueType::from_spec("tensor(a[2],c[3],d{},e[5],g[7],h{})"); + std::vector from({"a", "c", "e"}); + std::vector to({"f", "a", "b"}); + ValueType renamed = lhs.rename(from, to); + auto plan = instruction::DenseRenamePlan(lhs, renamed, from, to); + std::vector expect_loop = {15,2,7}; + std::vector expect_stride = {7,105,1}; + EXPECT_EQ(plan.subspace_size, 210); + EXPECT_EQ(plan.loop_cnt, expect_loop); + EXPECT_EQ(plan.stride, expect_stride); + std::vector out; + int want[3][5][2][7]; + size_t counter = 0; + for (size_t a = 0; a < 2; ++a) { + for (size_t c = 0; c < 3; ++c) { + for (size_t e = 0; e < 5; ++e) { + for (size_t g = 0; g < 7; ++g) { + want[c][e][a][g] = counter++; + } + } + } + } + std::vector expect(210); + memcpy(&expect[0], &want[0], 210*sizeof(int)); + auto move_cell = [&](size_t offset) { out.push_back(offset); }; + plan.execute(0, move_cell); + EXPECT_EQ(out, expect); +} + +TEST(GenericRenameTest, sparse_rename_plan_can_be_created) { + auto lhs = ValueType::from_spec("tensor(a{},c{},d[3],e{},g{},h[5])"); + std::vector from({"a", "c", "e"}); + std::vector to({"f", "a", "b"}); + ValueType renamed = lhs.rename(from, to); + auto plan = instruction::SparseRenamePlan(lhs, renamed, from, to); + EXPECT_EQ(plan.mapped_dims, 4); + std::vector expect = {2,0,1,3}; + EXPECT_EQ(plan.output_dimensions, expect); +} + +TensorSpec simple_tensor_rename(const TensorSpec &a, const FromTo &ft) { + Stash stash; + const auto &engine = SimpleTensorEngine::ref(); + auto lhs = engine.from_spec(a); + const auto &result = engine.rename(*lhs, ft.from, ft.to, stash); + return engine.to_spec(result); +} + +TensorSpec perform_generic_rename(const TensorSpec &a, const ValueType &res_type, + const FromTo &ft, const ValueBuilderFactory &factory) +{ + Stash stash; + auto lhs = value_from_spec(a, factory); + auto my_op = instruction::GenericRename::make_instruction(lhs->type(), res_type, ft.from, ft.to, factory, stash); + InterpretedFunction::EvalSingle single(my_op); + return spec_from_value(single.eval(std::vector({*lhs}))); +} + +void test_generic_rename(const ValueBuilderFactory &factory) { + for (const auto & layout : rename_layouts) { + TensorSpec lhs = spec(layout, N()); + ValueType lhs_type = ValueType::from_spec(lhs.type()); + // printf("lhs_type: %s\n", lhs_type.to_spec().c_str()); + for (const auto & from_to : rename_from_to) { + ValueType renamed_type = lhs_type.rename(from_to.from, from_to.to); + if (renamed_type.is_error()) continue; + // printf("type %s -> %s\n", lhs_type.to_spec().c_str(), renamed_type.to_spec().c_str()); + SCOPED_TRACE(fmt("\n===\nLHS: %s\n===\n", lhs.to_string().c_str())); + auto expect = simple_tensor_rename(lhs, from_to); + auto actual = perform_generic_rename(lhs, renamed_type, from_to, factory); + EXPECT_EQ(actual, expect); + } + } +} + +TEST(GenericRenameTest, generic_rename_works_for_simple_values) { + test_generic_rename(SimpleValueBuilderFactory::get()); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/vespa/eval/CMakeLists.txt b/eval/src/vespa/eval/CMakeLists.txt index b01945cf752..ee9a793bba0 100644 --- a/eval/src/vespa/eval/CMakeLists.txt +++ b/eval/src/vespa/eval/CMakeLists.txt @@ -2,6 +2,7 @@ vespa_add_library(vespaeval SOURCES $ + $ $ $ $ diff --git a/eval/src/vespa/eval/instruction/CMakeLists.txt b/eval/src/vespa/eval/instruction/CMakeLists.txt new file mode 100644 index 00000000000..f5a7de96f0b --- /dev/null +++ b/eval/src/vespa/eval/instruction/CMakeLists.txt @@ -0,0 +1,6 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +vespa_add_library(eval_instruction OBJECT + SOURCES + generic_rename +) diff --git a/eval/src/vespa/eval/instruction/generic_rename.cpp b/eval/src/vespa/eval/instruction/generic_rename.cpp new file mode 100644 index 00000000000..cda0aa6e89a --- /dev/null +++ b/eval/src/vespa/eval/instruction/generic_rename.cpp @@ -0,0 +1,197 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "generic_rename.h" +#include +#include +#include +#include + +namespace vespalib::eval::instruction { + +using State = InterpretedFunction::State; +using Instruction = InterpretedFunction::Instruction; + +namespace { + +template uint64_t wrap_param(const IN &value_in) { + const T &value = value_in; + static_assert(sizeof(uint64_t) == sizeof(&value)); + return (uint64_t)&value; +} + +template const T &unwrap_param(uint64_t param) { + return *((const T *)param); +} + +const vespalib::string & +find_rename(const vespalib::string & original, + const std::vector &from, + const std::vector &to) +{ + for (size_t i = 0; i < from.size(); ++i) { + if (original == from[i]) { + return to[i]; + } + } + return original; +} + +size_t +find_index_of(const vespalib::string & name, + const std::vector & dims) +{ + for (size_t i = 0; i < dims.size(); ++i) { + if (name == dims[i].name) { + return i; + } + } + abort(); // should not happen +} + +struct RenameParam { + ValueType res_type; + SparseRenamePlan sparse_plan; + DenseRenamePlan dense_plan; + const ValueBuilderFactory &factory; + RenameParam(const ValueType &lhs_type, const ValueType &output_type, + const std::vector &rename_dimension_from, + const std::vector &rename_dimension_to, + const ValueBuilderFactory &factory_in) + : res_type(output_type), + sparse_plan(lhs_type, output_type, rename_dimension_from, rename_dimension_to), + dense_plan(lhs_type, output_type, rename_dimension_from, rename_dimension_to), + factory(factory_in) + { + assert(!res_type.is_error()); + assert(lhs_type.cell_type() == res_type.cell_type()); + } + ~RenameParam(); +}; +RenameParam::~RenameParam() = default; + +template +std::unique_ptr +generic_rename(const Value &a, + const SparseRenamePlan &sparse_plan, const DenseRenamePlan &dense_plan, + const ValueType &res_type, const ValueBuilderFactory &factory) +{ + auto cells = a.cells().typify(); + std::vector output_address(sparse_plan.mapped_dims); + std::vector input_address; + for (size_t maps_to : sparse_plan.output_dimensions) { + input_address.push_back(&output_address[maps_to]); + } + auto builder = factory.create_value_builder(res_type, + sparse_plan.mapped_dims, + dense_plan.subspace_size, + a.index().size()); + auto view = a.index().create_view({}); + view->lookup({}); + size_t subspace; + while (view->next_result(input_address, subspace)) { + CT *dst = builder->add_subspace(output_address).begin(); + size_t input_offset = dense_plan.subspace_size * subspace; + auto copy_cells = [&](size_t input_idx) { *dst++ = cells[input_idx]; }; + dense_plan.execute(input_offset, copy_cells); + } + return builder->build(std::move(builder)); +} + +template +void my_generic_rename_op(State &state, uint64_t param_in) { + const auto ¶m = unwrap_param(param_in); + const Value &a = state.peek(0); + auto res_value = generic_rename(a, param.sparse_plan, param.dense_plan, + param.res_type, param.factory); + auto &result = state.stash.create>(std::move(res_value)); + const Value &result_ref = *(result.get()); + state.pop_push(result_ref); +} + +struct SelectGenericRenameOp { + template static auto invoke() { + return my_generic_rename_op; + } +}; + +} // namespace + +//----------------------------------------------------------------------------- + +SparseRenamePlan::SparseRenamePlan(const ValueType &input_type, + const ValueType &output_type, + const std::vector &from, + const std::vector &to) + : output_dimensions() +{ + const auto in_dims = input_type.mapped_dimensions(); + const auto out_dims = output_type.mapped_dimensions(); + mapped_dims = in_dims.size(); + assert(mapped_dims == out_dims.size()); + for (const auto & dim : in_dims) { + const auto & renamed_to = find_rename(dim.name, from, to); + size_t index = find_index_of(renamed_to, out_dims); + assert(index < mapped_dims); + output_dimensions.push_back(index); + } + assert(output_dimensions.size() == mapped_dims); +} + +SparseRenamePlan::~SparseRenamePlan() = default; + +DenseRenamePlan::DenseRenamePlan(const ValueType &lhs_type, + const ValueType &output_type, + const std::vector &from, + const std::vector &to) + : loop_cnt(), + stride(), + subspace_size(output_type.dense_subspace_size()) +{ + assert (subspace_size == lhs_type.dense_subspace_size()); + const auto lhs_dims = lhs_type.nontrivial_indexed_dimensions(); + const auto out_dims = output_type.nontrivial_indexed_dimensions(); + size_t num_dense_dims = lhs_dims.size(); + assert(num_dense_dims == out_dims.size()); + std::vector lhs_loopcnt(num_dense_dims); + std::vector lhs_stride(num_dense_dims, 1); + size_t lhs_size = 1; + for (size_t i = num_dense_dims; i-- > 0; ) { + lhs_stride[i] = lhs_size; + lhs_loopcnt[i] = lhs_dims[i].size; + lhs_size *= lhs_loopcnt[i]; + } + assert(lhs_size == subspace_size); + size_t prev_index = num_dense_dims; + for (const auto & dim : out_dims) { + const auto & renamed_from = find_rename(dim.name, to, from); + size_t index = find_index_of(renamed_from, lhs_dims); + assert(index < num_dense_dims); + if (prev_index + 1 == index) { + assert(stride.back() == lhs_stride[index] * lhs_loopcnt[index]); + loop_cnt.back() *= lhs_loopcnt[index]; + stride.back() = lhs_stride[index]; + } else { + loop_cnt.push_back(lhs_loopcnt[index]); + stride.push_back(lhs_stride[index]); + } + prev_index = index; + } +} + +DenseRenamePlan::~DenseRenamePlan() = default; + +InterpretedFunction::Instruction +GenericRename::make_instruction(const ValueType &lhs_type, const ValueType &output_type, + const std::vector &rename_dimension_from, + const std::vector &rename_dimension_to, + const ValueBuilderFactory &factory, Stash &stash) +{ + auto ¶m = stash.create(lhs_type, output_type, + rename_dimension_from, rename_dimension_to, + factory); + auto fun = typify_invoke<1,TypifyCellType,SelectGenericRenameOp>(output_type.cell_type()); + return Instruction(fun, wrap_param(param)); +} + +} // namespace + diff --git a/eval/src/vespa/eval/instruction/generic_rename.h b/eval/src/vespa/eval/instruction/generic_rename.h new file mode 100644 index 00000000000..e4f0146b9a8 --- /dev/null +++ b/eval/src/vespa/eval/instruction/generic_rename.h @@ -0,0 +1,77 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include +#include +#include +#include + +namespace vespalib::eval { class ValueBuilderFactory; } + +namespace vespalib::eval::instruction { + +struct DenseRenamePlan { + std::vector loop_cnt; + std::vector stride; + const size_t subspace_size; + DenseRenamePlan(const ValueType &lhs_type, + const ValueType &output_type, + const std::vector &from, + const std::vector &to); + ~DenseRenamePlan(); + template void execute(size_t offset, F &&f) const { + switch(loops_left(0)) { + case 0: return execute_few(0, offset, std::forward(f)); + case 1: return execute_few(0, offset, std::forward(f)); + case 2: return execute_few(0, offset, std::forward(f)); + case 3: return execute_few(0, offset, std::forward(f)); + default: return execute_many(0, offset, std::forward(f)); + } + } +private: + size_t loops_left(size_t idx) const { return (loop_cnt.size() - idx); } + + template void execute_few(size_t idx, size_t offset, F &&f) const { + if constexpr (N == 0) { + f(offset); + } else { + for (size_t i = 0; i < loop_cnt[idx]; ++i) { + execute_few(idx + 1, offset, std::forward(f)); + offset += stride[idx]; + } + } + } + template void execute_many(size_t idx, size_t offset, F &&f) const { + for (size_t i = 0; i < loop_cnt[idx]; ++i) { + if (loops_left(idx + 1) == 3) { + execute_few(idx + 1, offset, std::forward(f)); + } else { + execute_many(idx + 1, offset, std::forward(f)); + } + offset += stride[idx]; + } + } +}; + +struct SparseRenamePlan { + size_t mapped_dims; + std::vector output_dimensions; + SparseRenamePlan(const ValueType &input_type, + const ValueType &output_type, + const std::vector &from, + const std::vector &to); + ~SparseRenamePlan(); +}; + +//----------------------------------------------------------------------------- + +struct GenericRename { + static InterpretedFunction::Instruction + make_instruction(const ValueType &lhs_type, const ValueType &output_type, + const std::vector &rename_dimension_from, + const std::vector &rename_dimension_to, + const ValueBuilderFactory &factory, Stash &stash); +}; + +} // namespace -- cgit v1.2.3 From 6fe2e0a4568ef2c6f5b3a08a099fae4edc2385bc Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Thu, 24 Sep 2020 14:42:36 +0200 Subject: Deploy with internalRestart if flag is set --- .../controller/api/application/v4/model/DeploymentData.java | 9 ++++++++- .../yahoo/vespa/hosted/controller/ApplicationController.java | 10 +++++++++- flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 6 ++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java index 717a4296b81..ed54060cf5d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java @@ -32,6 +32,7 @@ public class DeploymentData { private final Optional athenzDomain; private final Optional applicationRoles; private final Optional quota; + private final boolean internalRestart; public DeploymentData(ApplicationId instance, ZoneId zone, byte[] applicationPackage, Version platform, Set containerEndpoints, @@ -39,7 +40,8 @@ public class DeploymentData { Optional dockerImageRepo, Optional athenzDomain, Optional applicationRoles, - Optional quota) { + Optional quota, + boolean internalRestart) { this.instance = requireNonNull(instance); this.zone = requireNonNull(zone); this.applicationPackage = requireNonNull(applicationPackage); @@ -50,6 +52,7 @@ public class DeploymentData { this.athenzDomain = athenzDomain; this.applicationRoles = applicationRoles; this.quota = quota; + this.internalRestart = internalRestart; } public ApplicationId instance() { @@ -91,4 +94,8 @@ public class DeploymentData { public Optional quota() { return quota; } + + public boolean internalRestart() { + return internalRestart; + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 7135cbe77c9..a6c177230d7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -118,6 +118,7 @@ public class ApplicationController { private final EndpointCertificateManager endpointCertificateManager; private final StringFlag dockerImageRepoFlag; private final BooleanFlag provisionApplicationRoles; + private final BooleanFlag deployWithInternalRestart; private final BillingController billingController; ApplicationController(Controller controller, CuratorDb curator, AccessControl accessControl, Clock clock, @@ -132,6 +133,7 @@ public class ApplicationController { this.applicationStore = controller.serviceRegistry().applicationStore(); this.dockerImageRepoFlag = Flags.DOCKER_IMAGE_REPO.bindTo(flagSource); this.provisionApplicationRoles = Flags.PROVISION_APPLICATION_ROLES.bindTo(flagSource); + this.deployWithInternalRestart = Flags.DEPLOY_WITH_INTERNAL_RESTART.bindTo(controller.flagSource()); this.billingController = billingController; deploymentTrigger = new DeploymentTrigger(controller, clock); @@ -521,6 +523,11 @@ public class ApplicationController { .filter(s -> !s.isBlank()) .map(DockerImage::fromString); + boolean internalRestart = deployWithInternalRestart + .with(FetchVector.Dimension.ZONE_ID, zone.value()) + .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()) + .value(); + Optional domain = controller.tenants().get(application.tenant()) .filter(tenant-> tenant instanceof AthenzTenant) .map(tenant -> ((AthenzTenant)tenant).domain()); @@ -534,7 +541,8 @@ public class ApplicationController { ConfigServer.PreparedApplication preparedApplication = configServer.deploy(new DeploymentData(application, zone, applicationPackage.zippedContent(), platform, - endpoints, endpointCertificateMetadata, dockerImageRepo, domain, applicationRoles, quota)); + endpoints, endpointCertificateMetadata, dockerImageRepo, domain, + applicationRoles, quota, internalRestart)); return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), applicationPackage.zippedContent().length); } finally { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index ce9063dbcce..411766ccacd 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -404,6 +404,12 @@ public class Flags { "Takes effect at next run of maintainer", APPLICATION_ID); + public static final UnboundBooleanFlag DEPLOY_WITH_INTERNAL_RESTART = defineFeatureFlag( + "deploy-with-internal-restart", false, + "Whether controller should deploy application with internal restart parameter set", + "Takes effect on next deploy from controller", + APPLICATION_ID, ZONE_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { -- cgit v1.2.3 From 88db94528ecbb96f88e30c5fa69ea9427e8ef0a7 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Thu, 24 Sep 2020 15:15:32 +0200 Subject: Count events per zk path and move to separate package --- .../hosted/provision/restapi/LocksResponse.java | 75 ++++++++----- zkfacade/abi-spec.json | 60 ---------- .../main/java/com/yahoo/vespa/curator/Lock.java | 3 +- .../java/com/yahoo/vespa/curator/LockInfo.java | 64 ----------- .../com/yahoo/vespa/curator/ThreadLockInfo.java | 125 --------------------- .../yahoo/vespa/curator/stats/LockCounters.java | 30 +++++ .../com/yahoo/vespa/curator/stats/LockInfo.java | 66 +++++++++++ .../yahoo/vespa/curator/stats/ThreadLockInfo.java | 122 ++++++++++++++++++++ .../yahoo/vespa/curator/stats/package-info.java | 5 + 9 files changed, 269 insertions(+), 281 deletions(-) delete mode 100644 zkfacade/src/main/java/com/yahoo/vespa/curator/LockInfo.java delete mode 100644 zkfacade/src/main/java/com/yahoo/vespa/curator/ThreadLockInfo.java create mode 100644 zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java create mode 100644 zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java create mode 100644 zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java create mode 100644 zkfacade/src/main/java/com/yahoo/vespa/curator/stats/package-info.java diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java index 817063592b1..bc4401ee03a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java @@ -5,14 +5,22 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; -import com.yahoo.vespa.curator.LockInfo; -import com.yahoo.vespa.curator.ThreadLockInfo; +import com.yahoo.vespa.curator.stats.LockCounters; +import com.yahoo.vespa.curator.stats.LockInfo; +import com.yahoo.vespa.curator.stats.ThreadLockInfo; import java.io.IOException; import java.io.OutputStream; import java.time.Instant; import java.util.List; +import java.util.Map; +import java.util.TreeMap; +/** + * Returns information related to ZooKeeper locks. + * + * @author hakon + */ public class LocksResponse extends HttpResponse { private final Slime slime; @@ -23,38 +31,43 @@ public class LocksResponse extends HttpResponse { this.slime = new Slime(); Cursor root = slime.setObject(); - root.setLong("in-critical-region", ThreadLockInfo.inCriticalRegionCount()); - root.setLong("invoke-acquire", ThreadLockInfo.invokeAcquireCount()); - root.setLong("acquire-timed-out", ThreadLockInfo.acquireTimedOutCount()); - root.setLong("lock-acquired", ThreadLockInfo.lockAcquiredCount()); - root.setLong("locks-released", ThreadLockInfo.locksReleasedCount()); - root.setLong("acquire-reentrant-lock-errors", ThreadLockInfo.failedToAcquireReentrantLockCount()); - root.setLong("no-locks-errors", ThreadLockInfo.noLocksErrorCount()); - root.setLong("timeout-on-reentrancy-errors", ThreadLockInfo.timeoutOnReentrancyErrorCount()); + Map lockCountersByPath = new TreeMap<>(ThreadLockInfo.getLockCountersByPath()); + + Cursor lockPathsCursor = root.setArray("lock-paths"); + lockCountersByPath.forEach((lockPath, lockCounters) -> { + Cursor lockPathCursor = lockPathsCursor.addObject(); + lockPathCursor.setString("path", lockPath); + lockPathCursor.setLong("in-critical-region", lockCounters.inCriticalRegionCount()); + lockPathCursor.setLong("invoke-acquire", lockCounters.invokeAcquireCount()); + lockPathCursor.setLong("acquire-timed-out", lockCounters.acquireTimedOutCount()); + lockPathCursor.setLong("lock-acquired", lockCounters.lockAcquiredCount()); + lockPathCursor.setLong("locks-released", lockCounters.locksReleasedCount()); + lockPathCursor.setLong("acquire-reentrant-lock-errors", lockCounters.failedToAcquireReentrantLockCount()); + lockPathCursor.setLong("no-locks-errors", lockCounters.noLocksErrorCount()); + lockPathCursor.setLong("timeout-on-reentrancy-errors", lockCounters.timeoutOnReentrancyErrorCount()); + }); List threadLockInfos = ThreadLockInfo.getThreadLockInfos(); - if (!threadLockInfos.isEmpty()) { - Cursor threadsCursor = root.setArray("threads"); - threadLockInfos.forEach(threadLockInfo -> { - Cursor threadLockInfoCursor = threadsCursor.addObject(); - threadLockInfoCursor.setString("thread-name", threadLockInfo.getThreadName()); - threadLockInfoCursor.setString("lock-path", threadLockInfo.getLockPath()); + Cursor threadsCursor = root.setArray("threads"); + threadLockInfos.forEach(threadLockInfo -> { + Cursor threadLockInfoCursor = threadsCursor.addObject(); + threadLockInfoCursor.setString("thread-name", threadLockInfo.getThreadName()); + threadLockInfoCursor.setString("lock-path", threadLockInfo.getLockPath()); - List lockInfos = threadLockInfo.getLockInfos(); - if (!lockInfos.isEmpty()) { - Cursor lockInfosCursor = threadLockInfoCursor.setArray("locks"); - lockInfos.forEach(lockInfo -> { - Cursor lockInfoCursor = lockInfosCursor.addObject(); - lockInfoCursor.setString("invoke-acquire-time", toString(lockInfo.getTimeAcquiredWasInvoked())); - lockInfoCursor.setLong("reentrancy-hold-count-on-acquire", lockInfo.getThreadHoldCountOnAcquire()); - lockInfoCursor.setString("acquire-timeout", lockInfo.getAcquireTimeout().toString()); - lockInfo.getTimeLockWasAcquired().ifPresent(instant -> lockInfoCursor.setString("lock-acquired-time", toString(instant))); - lockInfoCursor.setString("lock-state", lockInfo.getLockState().name()); - lockInfo.getTimeTerminalStateWasReached().ifPresent(instant -> lockInfoCursor.setString("terminal-state-time", toString(instant))); - }); - } - }); - } + List lockInfos = threadLockInfo.getLockInfos(); + if (!lockInfos.isEmpty()) { + Cursor lockInfosCursor = threadLockInfoCursor.setArray("locks"); + lockInfos.forEach(lockInfo -> { + Cursor lockInfoCursor = lockInfosCursor.addObject(); + lockInfoCursor.setString("invoke-acquire-time", toString(lockInfo.getTimeAcquiredWasInvoked())); + lockInfoCursor.setLong("reentrancy-hold-count-on-acquire", lockInfo.getThreadHoldCountOnAcquire()); + lockInfoCursor.setString("acquire-timeout", lockInfo.getAcquireTimeout().toString()); + lockInfo.getTimeLockWasAcquired().ifPresent(instant -> lockInfoCursor.setString("lock-acquired-time", toString(instant))); + lockInfoCursor.setString("lock-state", lockInfo.getLockState().name()); + lockInfo.getTimeTerminalStateWasReached().ifPresent(instant -> lockInfoCursor.setString("terminal-state-time", toString(instant))); + }); + } + }); } @Override diff --git a/zkfacade/abi-spec.json b/zkfacade/abi-spec.json index 614c58f3cf6..f4ad1ab4372 100644 --- a/zkfacade/abi-spec.json +++ b/zkfacade/abi-spec.json @@ -109,65 +109,5 @@ "public void close()" ], "fields": [] - }, - "com.yahoo.vespa.curator.LockInfo$LockState": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.vespa.curator.LockInfo$LockState[] values()", - "public static com.yahoo.vespa.curator.LockInfo$LockState valueOf(java.lang.String)", - "public boolean isTerminal()" - ], - "fields": [ - "public static final enum com.yahoo.vespa.curator.LockInfo$LockState ACQUIRING", - "public static final enum com.yahoo.vespa.curator.LockInfo$LockState TIMED_OUT", - "public static final enum com.yahoo.vespa.curator.LockInfo$LockState ACQUIRED", - "public static final enum com.yahoo.vespa.curator.LockInfo$LockState FAILED_TO_REENTER", - "public static final enum com.yahoo.vespa.curator.LockInfo$LockState RELEASED" - ] - }, - "com.yahoo.vespa.curator.LockInfo": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public static com.yahoo.vespa.curator.LockInfo invokingAcquire(int, java.time.Duration)", - "public int getThreadHoldCountOnAcquire()", - "public java.time.Instant getTimeAcquiredWasInvoked()", - "public java.time.Duration getAcquireTimeout()", - "public com.yahoo.vespa.curator.LockInfo$LockState getLockState()", - "public java.util.Optional getTimeLockWasAcquired()", - "public java.util.Optional getTimeTerminalStateWasReached()" - ], - "fields": [] - }, - "com.yahoo.vespa.curator.ThreadLockInfo": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public static int invokeAcquireCount()", - "public static int inCriticalRegionCount()", - "public static int acquireTimedOutCount()", - "public static int lockAcquiredCount()", - "public static int locksReleasedCount()", - "public static int noLocksErrorCount()", - "public static int failedToAcquireReentrantLockCount()", - "public static int timeoutOnReentrancyErrorCount()", - "public static java.util.List getThreadLockInfos()", - "public java.lang.String getThreadName()", - "public java.lang.String getLockPath()", - "public java.util.List getLockInfos()" - ], - "fields": [] } } \ No newline at end of file diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java index 299fceb8d7f..da6e3109695 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.curator; import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.path.Path; import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.curator.stats.ThreadLockInfo; import org.apache.curator.framework.recipes.locks.InterProcessLock; import java.time.Duration; @@ -39,7 +40,7 @@ public class Lock implements Mutex { threadLockInfo.acquireTimedOut(); throw new UncheckedTimeoutException("Timed out after waiting " + timeout + - " to acquire lock '" + lockPath + "'"); + " to acquire lock '" + lockPath + "'"); } threadLockInfo.lockAcquired(); diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/LockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/LockInfo.java deleted file mode 100644 index 870ee12ebda..00000000000 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/LockInfo.java +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.curator; - -import java.time.Duration; -import java.time.Instant; -import java.util.Optional; - -/** - * Information about a lock. - * - *

Should be mutated by a single thread. Other threads may see an inconsistent state of this instance.

- */ -public class LockInfo { - - private final int threadHoldCountOnAcquire; - private final Instant acquireInstant; - private final Duration timeout; - - private volatile Optional lockAcquiredInstant = Optional.empty(); - private volatile Optional terminalStateInstant = Optional.empty(); - - public static LockInfo invokingAcquire(int holdCount, Duration timeout) { - return new LockInfo(holdCount, timeout); - } - - public enum LockState { - ACQUIRING(false), TIMED_OUT(true), ACQUIRED(false), FAILED_TO_REENTER(true), RELEASED(true); - - private final boolean terminal; - - LockState(boolean terminal) { this.terminal = terminal; } - - public boolean isTerminal() { return terminal; } - } - - private volatile LockState lockState = LockState.ACQUIRING; - - private LockInfo(int threadHoldCountOnAcquire, Duration timeout) { - this.threadHoldCountOnAcquire = threadHoldCountOnAcquire; - this.acquireInstant = Instant.now(); - this.timeout = timeout; - } - - public int getThreadHoldCountOnAcquire() { return threadHoldCountOnAcquire; } - public Instant getTimeAcquiredWasInvoked() { return acquireInstant; } - public Duration getAcquireTimeout() { return timeout; } - public LockState getLockState() { return lockState; } - public Optional getTimeLockWasAcquired() { return lockAcquiredInstant; } - public Optional getTimeTerminalStateWasReached() { return terminalStateInstant; } - - void timedOut() { setTerminalState(LockState.TIMED_OUT); } - void failedToAcquireReentrantLock() { setTerminalState(LockState.FAILED_TO_REENTER); } - void released() { setTerminalState(LockState.RELEASED); } - - void lockAcquired() { - lockState = LockState.ACQUIRED; - lockAcquiredInstant = Optional.of(Instant.now()); - } - - void setTerminalState(LockState terminalState) { - lockState = terminalState; - terminalStateInstant = Optional.of(Instant.now()); - } -} diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/ThreadLockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/ThreadLockInfo.java deleted file mode 100644 index 9fbcb550ddd..00000000000 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/ThreadLockInfo.java +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.curator; - -import java.time.Duration; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Consumer; - -/** - * This class contains process-wide statistics and information related to acquiring and releasing - * {@link Lock}. Instances of this class contain information tied to a specific thread and lock path. - * - *

Instances of this class are thread-safe as long as foreign threads (!= this.thread) avoid mutable methods.

- */ -public class ThreadLockInfo { - - private static final ConcurrentHashMap locks = new ConcurrentHashMap<>(); - - private static final int MAX_COMPLETED_LOCK_INFOS_SIZE = 10; - private static final ConcurrentLinkedDeque completedLockInfos = new ConcurrentLinkedDeque<>(); - - private static final AtomicInteger invokeAcquireCount = new AtomicInteger(0); - private static final AtomicInteger inCriticalRegionCount = new AtomicInteger(0); - private static final AtomicInteger acquireTimedOutCount = new AtomicInteger(0); - private static final AtomicInteger lockAcquiredCount = new AtomicInteger(0); - private static final AtomicInteger locksReleasedCount = new AtomicInteger(0); - - private static final AtomicInteger failedToAcquireReentrantLockCount = new AtomicInteger(0); - private static final AtomicInteger noLocksErrorCount = new AtomicInteger(0); - private static final AtomicInteger timeoutOnReentrancyErrorCount = new AtomicInteger(0); - - private final Thread thread; - private final String lockPath; - private final ReentrantLock lock; - - /** The locks are reentrant so there may be more than 1 lock for this thread. */ - private final ConcurrentLinkedQueue lockInfos = new ConcurrentLinkedQueue<>(); - - public static int invokeAcquireCount() { return invokeAcquireCount.get(); } - public static int inCriticalRegionCount() { return inCriticalRegionCount.get(); } - public static int acquireTimedOutCount() { return acquireTimedOutCount.get(); } - public static int lockAcquiredCount() { return lockAcquiredCount.get(); } - public static int locksReleasedCount() { return locksReleasedCount.get(); } - public static int noLocksErrorCount() { return noLocksErrorCount.get(); } - public static int failedToAcquireReentrantLockCount() { return failedToAcquireReentrantLockCount.get(); } - public static int timeoutOnReentrancyErrorCount() { return timeoutOnReentrancyErrorCount.get(); } - public static List getThreadLockInfos() { return List.copyOf(locks.values()); } - - /** Returns the per-thread singleton ThreadLockInfo. */ - static ThreadLockInfo getCurrentThreadLockInfo(String lockPath, ReentrantLock lock) { - return locks.computeIfAbsent( - Thread.currentThread(), - currentThread -> new ThreadLockInfo(currentThread, lockPath, lock)); - } - - ThreadLockInfo(Thread currentThread, String lockPath, ReentrantLock lock) { - this.thread = currentThread; - this.lockPath = lockPath; - this.lock = lock; - } - - public String getThreadName() { return thread.getName(); } - public String getLockPath() { return lockPath; } - public List getLockInfos() { return List.copyOf(lockInfos); } - - /** Mutable method (see class doc) */ - void invokingAcquire(Duration timeout) { - invokeAcquireCount.incrementAndGet(); - inCriticalRegionCount.incrementAndGet(); - lockInfos.add(LockInfo.invokingAcquire(lock.getHoldCount(), timeout)); - } - - /** Mutable method (see class doc) */ - void acquireTimedOut() { - if (lockInfos.size() > 1) { - timeoutOnReentrancyErrorCount.incrementAndGet(); - } - - removeLastLockInfo(acquireTimedOutCount, LockInfo::timedOut); - } - - /** Mutable method (see class doc) */ - void lockAcquired() { - lockAcquiredCount.incrementAndGet(); - getLastLockInfo().ifPresent(LockInfo::lockAcquired); - } - - /** Mutable method (see class doc) */ - void failedToAcquireReentrantLock() { - removeLastLockInfo(failedToAcquireReentrantLockCount, LockInfo::failedToAcquireReentrantLock); - } - - /** Mutable method (see class doc) */ - void lockReleased() { - removeLastLockInfo(locksReleasedCount, LockInfo::released); - } - - private Optional getLastLockInfo() { - return lockInfos.isEmpty() ? Optional.empty() : Optional.of(lockInfos.peek()); - } - - private void removeLastLockInfo(AtomicInteger metricToIncrement, Consumer completeLockInfo) { - metricToIncrement.incrementAndGet(); - inCriticalRegionCount.decrementAndGet(); - - if (lockInfos.isEmpty()) { - noLocksErrorCount.incrementAndGet(); - return; - } - - LockInfo lockInfo = lockInfos.poll(); - completeLockInfo.accept(lockInfo); - - if (completedLockInfos.size() >= MAX_COMPLETED_LOCK_INFOS_SIZE) { - // This is thread-safe, as no-one but currentThread mutates completedLockInfos - completedLockInfos.removeLast(); - } - completedLockInfos.addFirst(lockInfo); - } -} diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java new file mode 100644 index 00000000000..9052db5ce5a --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java @@ -0,0 +1,30 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator.stats; + +import java.util.concurrent.atomic.AtomicInteger; + +/** + * A collection of counters for events related to lock acquisition and release. + * + * @author hakon + */ +public class LockCounters { + final AtomicInteger invokeAcquireCount = new AtomicInteger(0); + final AtomicInteger inCriticalRegionCount = new AtomicInteger(0); + final AtomicInteger acquireTimedOutCount = new AtomicInteger(0); + final AtomicInteger lockAcquiredCount = new AtomicInteger(0); + final AtomicInteger locksReleasedCount = new AtomicInteger(0); + + final AtomicInteger failedToAcquireReentrantLockCount = new AtomicInteger(0); + final AtomicInteger noLocksErrorCount = new AtomicInteger(0); + final AtomicInteger timeoutOnReentrancyErrorCount = new AtomicInteger(0); + + public int invokeAcquireCount() { return invokeAcquireCount.get(); } + public int inCriticalRegionCount() { return inCriticalRegionCount.get(); } + public int acquireTimedOutCount() { return acquireTimedOutCount.get(); } + public int lockAcquiredCount() { return lockAcquiredCount.get(); } + public int locksReleasedCount() { return locksReleasedCount.get(); } + public int failedToAcquireReentrantLockCount() { return failedToAcquireReentrantLockCount.get(); } + public int noLocksErrorCount() { return noLocksErrorCount.get(); } + public int timeoutOnReentrancyErrorCount() { return timeoutOnReentrancyErrorCount.get(); } +} diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java new file mode 100644 index 00000000000..b192210c3de --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java @@ -0,0 +1,66 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator.stats; + +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; + +/** + * Information about a lock. + * + *

Should be mutated by a single thread. Other threads may see an inconsistent state of this instance.

+ * + * @author hakon + */ +public class LockInfo { + + private final int threadHoldCountOnAcquire; + private final Instant acquireInstant; + private final Duration timeout; + + private volatile Optional lockAcquiredInstant = Optional.empty(); + private volatile Optional terminalStateInstant = Optional.empty(); + + public static LockInfo invokingAcquire(int holdCount, Duration timeout) { + return new LockInfo(holdCount, timeout); + } + + public enum LockState { + ACQUIRING(false), TIMED_OUT(true), ACQUIRED(false), FAILED_TO_REENTER(true), RELEASED(true); + + private final boolean terminal; + + LockState(boolean terminal) { this.terminal = terminal; } + + public boolean isTerminal() { return terminal; } + } + + private volatile LockState lockState = LockState.ACQUIRING; + + private LockInfo(int threadHoldCountOnAcquire, Duration timeout) { + this.threadHoldCountOnAcquire = threadHoldCountOnAcquire; + this.acquireInstant = Instant.now(); + this.timeout = timeout; + } + + public int getThreadHoldCountOnAcquire() { return threadHoldCountOnAcquire; } + public Instant getTimeAcquiredWasInvoked() { return acquireInstant; } + public Duration getAcquireTimeout() { return timeout; } + public LockState getLockState() { return lockState; } + public Optional getTimeLockWasAcquired() { return lockAcquiredInstant; } + public Optional getTimeTerminalStateWasReached() { return terminalStateInstant; } + + void timedOut() { setTerminalState(LockState.TIMED_OUT); } + void failedToAcquireReentrantLock() { setTerminalState(LockState.FAILED_TO_REENTER); } + void released() { setTerminalState(LockState.RELEASED); } + + void lockAcquired() { + lockState = LockState.ACQUIRED; + lockAcquiredInstant = Optional.of(Instant.now()); + } + + void setTerminalState(LockState terminalState) { + lockState = terminalState; + terminalStateInstant = Optional.of(Instant.now()); + } +} diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java new file mode 100644 index 00000000000..bba39e6dc49 --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java @@ -0,0 +1,122 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator.stats; + +import com.yahoo.vespa.curator.Lock; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; + +/** + * This class contains process-wide statistics and information related to acquiring and releasing + * {@link Lock}. Instances of this class contain information tied to a specific thread and lock path. + * + *

Instances of this class are thread-safe as long as foreign threads (!= this.thread) avoid mutable methods.

+ * + * @author hakon + */ +public class ThreadLockInfo { + + private static final ConcurrentHashMap locks = new ConcurrentHashMap<>(); + + private static final int MAX_COMPLETED_LOCK_INFOS_SIZE = 10; + private static final ConcurrentLinkedDeque completedLockInfos = new ConcurrentLinkedDeque<>(); + + private static final ConcurrentHashMap countersByLockPath = new ConcurrentHashMap<>(); + + private final Thread thread; + private final String lockPath; + private final ReentrantLock lock; + private final LockCounters lockCountersForPath; + + /** The locks are reentrant so there may be more than 1 lock for this thread. */ + private final ConcurrentLinkedQueue lockInfos = new ConcurrentLinkedQueue<>(); + + public static Map getLockCountersByPath() { return Map.copyOf(countersByLockPath); } + + public static List getThreadLockInfos() { return List.copyOf(locks.values()); } + + /** Returns the per-thread singleton ThreadLockInfo. */ + public static ThreadLockInfo getCurrentThreadLockInfo(String lockPath, ReentrantLock lock) { + return locks.computeIfAbsent( + Thread.currentThread(), + currentThread -> { + LockCounters lockCounters = countersByLockPath.computeIfAbsent(lockPath, ignored -> new LockCounters()); + return new ThreadLockInfo(currentThread, lockPath, lock, lockCounters); + }); + } + + ThreadLockInfo(Thread currentThread, String lockPath, ReentrantLock lock, LockCounters lockCountersForPath) { + this.thread = currentThread; + this.lockPath = lockPath; + this.lock = lock; + this.lockCountersForPath = lockCountersForPath; + } + + public String getThreadName() { return thread.getName(); } + public String getLockPath() { return lockPath; } + public List getLockInfos() { return List.copyOf(lockInfos); } + + /** Mutable method (see class doc) */ + public void invokingAcquire(Duration timeout) { + lockCountersForPath.invokeAcquireCount.incrementAndGet(); + lockCountersForPath.inCriticalRegionCount.incrementAndGet(); + lockInfos.add(LockInfo.invokingAcquire(lock.getHoldCount(), timeout)); + } + + /** Mutable method (see class doc) */ + public void acquireTimedOut() { + if (lockInfos.size() > 1) { + lockCountersForPath.timeoutOnReentrancyErrorCount.incrementAndGet(); + } + + removeLastLockInfo(lockCountersForPath.timeoutOnReentrancyErrorCount, LockInfo::timedOut); + } + + /** Mutable method (see class doc) */ + public void lockAcquired() { + lockCountersForPath.lockAcquiredCount.incrementAndGet(); + + getLastLockInfo().ifPresent(LockInfo::lockAcquired); + } + + /** Mutable method (see class doc) */ + public void failedToAcquireReentrantLock() { + removeLastLockInfo(lockCountersForPath.failedToAcquireReentrantLockCount, LockInfo::failedToAcquireReentrantLock); + } + + /** Mutable method (see class doc) */ + public void lockReleased() { + removeLastLockInfo(lockCountersForPath.locksReleasedCount, LockInfo::released); + } + + private Optional getLastLockInfo() { + return lockInfos.isEmpty() ? Optional.empty() : Optional.of(lockInfos.peek()); + } + + private void removeLastLockInfo(AtomicInteger metricToIncrement, Consumer completeLockInfo) { + metricToIncrement.incrementAndGet(); + lockCountersForPath.inCriticalRegionCount.decrementAndGet(); + + if (lockInfos.isEmpty()) { + lockCountersForPath.noLocksErrorCount.incrementAndGet(); + return; + } + + LockInfo lockInfo = lockInfos.poll(); + completeLockInfo.accept(lockInfo); + + if (completedLockInfos.size() >= MAX_COMPLETED_LOCK_INFOS_SIZE) { + // This is thread-safe, as no-one but currentThread mutates completedLockInfos + completedLockInfos.removeLast(); + } + completedLockInfos.addFirst(lockInfo); + } +} diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/package-info.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/package-info.java new file mode 100644 index 00000000000..15a81ffea70 --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/package-info.java @@ -0,0 +1,5 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.curator.stats; + +import com.yahoo.osgi.annotation.ExportPackage; -- cgit v1.2.3 From 5fd7195145a2ab4c5c57cfe7d770b228e0d8ea48 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Thu, 24 Sep 2020 15:20:15 +0200 Subject: Use DaemonThreadFactory and make sure to shutdown executors --- .../vespa/config/server/ConfigServerBootstrap.java | 19 +++++++------------ .../server/application/FileDistributionStatus.java | 5 +++++ .../config/server/filedistribution/FileServer.java | 9 +++++++-- .../config/server/rpc/DelayedConfigResponses.java | 4 +++- .../com/yahoo/vespa/config/server/rpc/RpcServer.java | 2 +- .../vespa/config/server/tenant/TenantRepository.java | 7 +++++-- 6 files changed, 28 insertions(+), 18 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index 609ff4473c6..3275dc42477 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -57,7 +57,6 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable private final ApplicationRepository applicationRepository; private final RpcServer server; - private final Optional serverThread; private final VersionState versionState; private final StateMonitor stateMonitor; private final VipStatus vipStatus; @@ -66,6 +65,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable private final Duration sleepTimeWhenRedeployingFails; private final RedeployingApplicationsFails exitIfRedeployingApplicationsFails; private final ExecutorService rpcServerExecutor; + private final Optional bootstrapExecutor; @Inject public ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, @@ -96,20 +96,21 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable this.sleepTimeWhenRedeployingFails = Duration.ofSeconds(configserverConfig.sleepTimeWhenRedeployingFails()); this.exitIfRedeployingApplicationsFails = exitIfRedeployingApplicationsFails; rpcServerExecutor = Executors.newSingleThreadExecutor(new DaemonThreadFactory("config server RPC server")); + log.log(Level.FINE, "Bootstrap mode: " + mode + ", VIP status mode: " + vipStatusMode); initializing(vipStatusMode); switch (mode) { case BOOTSTRAP_IN_SEPARATE_THREAD: - this.serverThread = Optional.of(new Thread(this, "config server bootstrap thread")); - serverThread.get().start(); + bootstrapExecutor = Optional.of(Executors.newSingleThreadExecutor(new DaemonThreadFactory("config server bootstrap"))); + bootstrapExecutor.get().execute(this); break; case BOOTSTRAP_IN_CONSTRUCTOR: - this.serverThread = Optional.empty(); + bootstrapExecutor = Optional.empty(); start(); break; case INITIALIZE_ONLY: - this.serverThread = Optional.empty(); + bootstrapExecutor = Optional.empty(); break; default: throw new IllegalArgumentException("Unknown bootstrap mode " + mode + ", legal values: " + Arrays.toString(Mode.values())); @@ -123,13 +124,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable server.stop(); log.log(Level.FINE, "RPC server stopped"); rpcServerExecutor.shutdown(); - serverThread.ifPresent(thread -> { - try { - thread.join(); - } catch (InterruptedException e) { - log.log(Level.WARNING, "Error joining server thread on shutdown: " + e.getMessage()); - } - }); + bootstrapExecutor.ifPresent(ExecutorService::shutdownNow); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java index 012d0d52275..ab1bd79d498 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java @@ -147,6 +147,11 @@ public class FileDistributionStatus extends AbstractComponent { } } + @Override + public void deconstruct() { + rpcExecutor.shutdownNow(); + } + static class HostStatus { private final String hostname; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index 99cdb0a74dc..0be91c2d5e1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.filedistribution; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.FileReference; import com.yahoo.jrt.Int32Value; import com.yahoo.jrt.Request; @@ -79,8 +80,10 @@ public class FileServer { private FileServer(ConnectionPool connectionPool, File rootDir) { this.downloader = new FileDownloader(connectionPool); this.root = new FileDirectory(rootDir); - this.pushExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors())); - this.pullExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors())); + this.pushExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), + new DaemonThreadFactory("file server push")); + this.pullExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), + new DaemonThreadFactory("file server push")); } boolean hasFile(String fileReference) { @@ -191,6 +194,8 @@ public class FileServer { public void close() { downloader.close(); + pullExecutor.shutdown(); + pushExecutor.shutdown(); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java index 7ab55b9af72..9820eac2f30 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java @@ -52,7 +52,9 @@ public class DelayedConfigResponses { // Since JRT does not allow adding watcher for "fake" requests, we must be able to disable it for unit tests :( DelayedConfigResponses(RpcServer rpcServer, int numTimerThreads, boolean useJrtWatcher) { this.rpcServer = rpcServer; - ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(numTimerThreads, ThreadFactoryFactory.getThreadFactory(DelayedConfigResponses.class.getName())); + ScheduledThreadPoolExecutor executor = + new ScheduledThreadPoolExecutor(numTimerThreads, + ThreadFactoryFactory.getDaemonThreadFactory("delayed config responses")); executor.setRemoveOnCancelPolicy(true); this.executorService = executor; this.useJrtWatcher = useJrtWatcher; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index a96ae16a20b..55ee45482f9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -133,7 +133,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { BlockingQueue workQueue = new LinkedBlockingQueue<>(config.maxgetconfigclients()); int rpcWorkerThreads = (config.numRpcThreads() == 0) ? threadsToUse() : config.numRpcThreads(); executorService = new ThreadPoolExecutor(rpcWorkerThreads, rpcWorkerThreads, - 0, TimeUnit.SECONDS, workQueue, ThreadFactoryFactory.getThreadFactory(THREADPOOL_NAME)); + 0, TimeUnit.SECONDS, workQueue, ThreadFactoryFactory.getDaemonThreadFactory(THREADPOOL_NAME)); delayedConfigResponses = new DelayedConfigResponses(this, config.numDelayedResponseThreads()); spec = new Spec(null, config.rpcport()); hostRegistry = hostRegistries.getTenantHostRegistry(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 41377bdf317..57e49fe365a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.tenant; import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.concurrent.StripedExecutor; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; @@ -84,7 +85,8 @@ public class TenantRepository { private final ExecutorService zkCacheExecutor; private final StripedExecutor zkWatcherExecutor; private final ExecutorService bootstrapExecutor; - private final ScheduledExecutorService checkForRemovedApplicationsService = new ScheduledThreadPoolExecutor(1); + private final ScheduledExecutorService checkForRemovedApplicationsService = + new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("check for removed applications")); private final Optional directoryCache; /** @@ -96,7 +98,8 @@ public class TenantRepository { public TenantRepository(GlobalComponentRegistry componentRegistry) { this.componentRegistry = componentRegistry; ConfigserverConfig configserverConfig = componentRegistry.getConfigserverConfig(); - this.bootstrapExecutor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders()); + this.bootstrapExecutor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders(), + new DaemonThreadFactory("bootstrap tenants")); this.curator = componentRegistry.getCurator(); metricUpdater = componentRegistry.getMetrics().getOrCreateMetricUpdater(Collections.emptyMap()); this.tenantListeners.add(componentRegistry.getTenantListener()); -- cgit v1.2.3 From fb7bceeb2537d57e4ce6543201d8ed8afe400d4c Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Thu, 24 Sep 2020 15:36:11 +0200 Subject: Fix name --- .../java/com/yahoo/vespa/config/server/filedistribution/FileServer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index 0be91c2d5e1..31294faad05 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -83,7 +83,7 @@ public class FileServer { this.pushExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), new DaemonThreadFactory("file server push")); this.pullExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), - new DaemonThreadFactory("file server push")); + new DaemonThreadFactory("file server pull")); } boolean hasFile(String fileReference) { -- cgit v1.2.3 From 7cedd2af51c97567dc02cb3c3a39cbbbed40f584 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Thu, 24 Sep 2020 15:29:24 +0200 Subject: Report Curator cache metrics --- .../provision/maintenance/MetricsReporter.java | 19 +++++++----- .../hosted/provision/persistence/CacheStats.java | 36 ++++++++++++++++++++++ .../provision/persistence/CuratorDatabase.java | 31 +++++++++++++++++-- .../persistence/CuratorDatabaseClient.java | 6 +++- .../provision/persistence/NodeSerializer.java | 31 ++----------------- .../provision/maintenance/MetricsReporterTest.java | 5 +++ 6 files changed, 88 insertions(+), 40 deletions(-) create mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CacheStats.java diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index d15bc9ede90..cd05c7e36d3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -15,7 +15,7 @@ import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.History; -import com.yahoo.vespa.hosted.provision.persistence.NodeSerializer; +import com.yahoo.vespa.hosted.provision.persistence.CacheStats; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ServiceMonitor; @@ -69,15 +69,20 @@ public class MetricsReporter extends NodeRepositoryMaintainer { updateMaintenanceMetrics(); updateDockerMetrics(nodes); updateTenantUsageMetrics(nodes); - updateNodeSerializerCacheMetrics(); + updateCacheMetrics(); return true; } - private void updateNodeSerializerCacheMetrics() { - NodeSerializer.CacheStats cacheStats = nodeRepository().database().nodeSerializerCacheStats(); - metric.set("cache.nodeObject.hitRate", cacheStats.hitRate(), null); - metric.set("cache.nodeObject.evictionCount", cacheStats.evictionCount(), null); - metric.set("cache.nodeObject.size", cacheStats.size(), null); + private void updateCacheMetrics() { + CacheStats nodeCacheStats = nodeRepository().database().nodeSerializerCacheStats(); + metric.set("cache.nodeObject.hitRate", nodeCacheStats.hitRate(), null); + metric.set("cache.nodeObject.evictionCount", nodeCacheStats.evictionCount(), null); + metric.set("cache.nodeObject.size", nodeCacheStats.size(), null); + + CacheStats curatorCacheStats = nodeRepository().database().cacheStats(); + metric.set("cache.curator.hitRate", curatorCacheStats.hitRate(), null); + metric.set("cache.curator.evictionCount", curatorCacheStats.evictionCount(), null); + metric.set("cache.curator.size", curatorCacheStats.size(), null); } private void updateMaintenanceMetrics() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CacheStats.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CacheStats.java new file mode 100644 index 00000000000..a2a2bcf3ab4 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CacheStats.java @@ -0,0 +1,36 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +/** + * Statistics for caches used by {@link com.yahoo.vespa.hosted.provision.NodeRepository}. + * + * @author mpolden + */ +public class CacheStats { + + private final double hitRate; + private final long evictionCount; + private final long size; + + public CacheStats(double hitRate, long evictionCount, long size) { + this.hitRate = hitRate; + this.evictionCount = evictionCount; + this.size = size; + } + + /** The fraction of lookups that resulted in a hit */ + public double hitRate() { + return hitRate; + } + + /** The number of entries that have been evicted */ + public long evictionCount() { + return evictionCount; + } + + /** The current size of the cache */ + public long size() { + return size; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java index 08f2cfec40f..fa5a72eea52 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; +import com.google.common.cache.AbstractCache; import com.google.common.collect.ImmutableList; import com.yahoo.config.provision.HostName; import com.yahoo.path.Path; @@ -17,6 +18,7 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -116,6 +118,10 @@ public class CuratorDatabase { return cache.get(); } + CacheStats cacheStats() { + return cache.get().stats(); + } + /** Caches must only be instantiated using this method */ private Cache newCache(long generation) { return useCache ? new Cache(generation, curator) : new NoCache(generation, curator); @@ -139,6 +145,8 @@ public class CuratorDatabase { private final Map> children = new ConcurrentHashMap<>(); private final Map> data = new ConcurrentHashMap<>(); + private final AbstractCache.SimpleStatsCounter stats = new AbstractCache.SimpleStatsCounter(); + /** Create an empty snapshot at a given generation (as an empty snapshot is a valid partial snapshot) */ private Cache(long generation, Curator curator) { this.generation = generation; @@ -146,13 +154,29 @@ public class CuratorDatabase { } @Override - public List getChildren(Path path) { - return children.computeIfAbsent(path, key -> ImmutableList.copyOf(curator.getChildren(path))); + public List getChildren(Path path) { + return get(children, path, () -> ImmutableList.copyOf(curator.getChildren(path))); } @Override public Optional getData(Path path) { - return data.computeIfAbsent(path, key -> curator.getData(path)).map(data -> Arrays.copyOf(data, data.length)); + return get(data, path, () -> curator.getData(path)).map(data -> Arrays.copyOf(data, data.length)); + } + + private T get(Map values, Path path, Supplier loader) { + return values.compute(path, (key, value) -> { + if (value == null) { + stats.recordMisses(1); + return loader.get(); + } + stats.recordHits(1); + return value; + }); + } + + public CacheStats stats() { + var stats = this.stats.snapshot(); + return new CacheStats(stats.hitRate(), stats.evictionCount(), children.size() + data.size()); } } @@ -183,4 +207,5 @@ public class CuratorDatabase { Optional getData(Path path); } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index dc6c2ee173b..56f7c951025 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -589,7 +589,11 @@ public class CuratorDatabaseClient { .collect(Collectors.toList()); } - public NodeSerializer.CacheStats nodeSerializerCacheStats() { + public CacheStats cacheStats() { + return db.cacheStats(); + } + + public CacheStats nodeSerializerCacheStats() { return nodeSerializer.cacheStats(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 064a9d5d933..21ec3d48aef 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -139,8 +139,8 @@ public class NodeSerializer { /** Returns cache statistics for this serializer */ public CacheStats cacheStats() { - com.google.common.cache.CacheStats cacheStats = cache.stats(); - return new CacheStats(cacheStats.hitRate(), cacheStats.evictionCount(), cache.size()); + var stats = cache.stats(); + return new CacheStats(stats.hitRate(), stats.evictionCount(), cache.size()); } private void toSlime(Node node, Cursor object) { @@ -483,31 +483,4 @@ public class NodeSerializer { throw new IllegalArgumentException("Serialized form of '" + type + "' not defined"); } - /** Cache statistics for this serializer's object cache */ - public static class CacheStats { - - private final double hitRate; - private final long evictionCount; - private final long size; - - public CacheStats(double hitRate, long evictionCount, long size) { - this.hitRate = hitRate; - this.evictionCount = evictionCount; - this.size = size; - } - - public double hitRate() { - return hitRate; - } - - public long evictionCount() { - return evictionCount; - } - - public long size() { - return size; - } - - } - } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 1c5149702e7..557453daa49 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -127,6 +127,11 @@ public class MetricsReporterTest { expectedMetrics.put("cache.nodeObject.evictionCount", 0L); expectedMetrics.put("cache.nodeObject.size", 2L); + nodeRepository.list(); + expectedMetrics.put("cache.curator.hitRate", 0.5D); + expectedMetrics.put("cache.curator.evictionCount", 0L); + expectedMetrics.put("cache.curator.size", 11L); + ManualClock clock = new ManualClock(Instant.ofEpochSecond(124)); Orchestrator orchestrator = mock(Orchestrator.class); -- cgit v1.2.3 From 34f461e913a40db6db18e5d98a3ef9e8880de0cb Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Thu, 24 Sep 2020 13:32:33 +0000 Subject: move generic join into its own files --- eval/CMakeLists.txt | 1 + .../tests/eval/simple_value/simple_value_test.cpp | 48 +---- .../tests/instruction/generic_join/CMakeLists.txt | 9 + .../instruction/generic_join/generic_join_test.cpp | 119 +++++++++++ .../generic_rename/generic_rename_test.cpp | 7 +- eval/src/vespa/eval/eval/CMakeLists.txt | 2 - eval/src/vespa/eval/eval/tensor_instructions.cpp | 148 ------------- eval/src/vespa/eval/eval/tensor_instructions.h | 24 --- eval/src/vespa/eval/eval/tensor_plans.cpp | 86 -------- eval/src/vespa/eval/eval/tensor_plans.h | 77 ------- eval/src/vespa/eval/instruction/CMakeLists.txt | 1 + eval/src/vespa/eval/instruction/generic_join.cpp | 232 +++++++++++++++++++++ eval/src/vespa/eval/instruction/generic_join.h | 87 ++++++++ 13 files changed, 456 insertions(+), 385 deletions(-) create mode 100644 eval/src/tests/instruction/generic_join/CMakeLists.txt create mode 100644 eval/src/tests/instruction/generic_join/generic_join_test.cpp delete mode 100644 eval/src/vespa/eval/eval/tensor_instructions.cpp delete mode 100644 eval/src/vespa/eval/eval/tensor_instructions.h delete mode 100644 eval/src/vespa/eval/eval/tensor_plans.cpp delete mode 100644 eval/src/vespa/eval/eval/tensor_plans.h create mode 100644 eval/src/vespa/eval/instruction/generic_join.cpp create mode 100644 eval/src/vespa/eval/instruction/generic_join.h diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index cb0a3b3bc68..66ee9916c83 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -32,6 +32,7 @@ vespa_define_module( src/tests/eval/value_codec src/tests/eval/value_type src/tests/gp/ponder_nov2017 + src/tests/instruction/generic_join src/tests/instruction/generic_rename src/tests/tensor/dense_add_dimension_optimizer src/tests/tensor/dense_dimension_combiner diff --git a/eval/src/tests/eval/simple_value/simple_value_test.cpp b/eval/src/tests/eval/simple_value/simple_value_test.cpp index 6fa89989bb7..4827fa3be3c 100644 --- a/eval/src/tests/eval/simple_value/simple_value_test.cpp +++ b/eval/src/tests/eval/simple_value/simple_value_test.cpp @@ -2,8 +2,7 @@ #include #include -#include -#include +#include #include #include #include @@ -11,6 +10,7 @@ using namespace vespalib; using namespace vespalib::eval; +using namespace vespalib::eval::instruction; using namespace vespalib::eval::test; using vespalib::make_string_short::fmt; @@ -70,7 +70,7 @@ TensorSpec simple_value_new_join(const TensorSpec &a, const TensorSpec &b, join_ const auto &factory = SimpleValueBuilderFactory::get(); auto lhs = value_from_spec(a, factory); auto rhs = value_from_spec(b, factory); - auto my_op = tensor_instruction::make_join(lhs->type(), rhs->type(), function, factory, stash); + auto my_op = GenericJoin::make_instruction(lhs->type(), rhs->type(), function, factory, stash); InterpretedFunction::EvalSingle single(my_op); return spec_from_value(single.eval(std::vector({*lhs,*rhs}))); } @@ -115,48 +115,6 @@ TEST(SimpleValueTest, simple_value_can_be_built_and_inspected) { EXPECT_FALSE(view->next_result({&label}, subspace)); } -TEST(SimpleValueTest, dense_join_plan_can_be_created) { - auto lhs = ValueType::from_spec("tensor(a{},b[6],c[5],e[3],f[2],g{})"); - auto rhs = ValueType::from_spec("tensor(a{},b[6],c[5],d[4],h{})"); - auto plan = DenseJoinPlan(lhs, rhs); - std::vector expect_loop = {30,4,6}; - std::vector expect_lhs_stride = {6,0,1}; - std::vector expect_rhs_stride = {4,1,0}; - EXPECT_EQ(plan.lhs_size, 180); - EXPECT_EQ(plan.rhs_size, 120); - EXPECT_EQ(plan.out_size, 720); - EXPECT_EQ(plan.loop_cnt, expect_loop); - EXPECT_EQ(plan.lhs_stride, expect_lhs_stride); - EXPECT_EQ(plan.rhs_stride, expect_rhs_stride); -} - -TEST(SimpleValueTest, sparse_join_plan_can_be_created) { - auto lhs = ValueType::from_spec("tensor(a{},b[6],c[5],e[3],f[2],g{})"); - auto rhs = ValueType::from_spec("tensor(b[6],c[5],d[4],g{},h{})"); - auto plan = SparseJoinPlan(lhs, rhs); - using SRC = SparseJoinPlan::Source; - std::vector expect_sources = {SRC::LHS,SRC::BOTH,SRC::RHS}; - std::vector expect_lhs_overlap = {1}; - std::vector expect_rhs_overlap = {0}; - EXPECT_EQ(plan.sources, expect_sources); - EXPECT_EQ(plan.lhs_overlap, expect_lhs_overlap); - EXPECT_EQ(plan.rhs_overlap, expect_rhs_overlap); -} - -TEST(SimpleValueTest, dense_join_plan_can_be_executed) { - auto plan = DenseJoinPlan(ValueType::from_spec("tensor(a[2])"), - ValueType::from_spec("tensor(b[3])")); - std::vector a({1, 2}); - std::vector b({3, 4, 5}); - std::vector c(6, 0); - std::vector expect = {3,4,5,6,8,10}; - ASSERT_EQ(plan.out_size, 6); - int *dst = &c[0]; - auto cell_join = [&](size_t a_idx, size_t b_idx) { *dst++ = (a[a_idx] * b[b_idx]); }; - plan.execute(0, 0, cell_join); - EXPECT_EQ(c, expect); -} - TEST(SimpleValueTest, new_generic_join_works_for_simple_values) { ASSERT_TRUE((join_layouts.size() % 2) == 0); for (size_t i = 0; i < join_layouts.size(); i += 2) { diff --git a/eval/src/tests/instruction/generic_join/CMakeLists.txt b/eval/src/tests/instruction/generic_join/CMakeLists.txt new file mode 100644 index 00000000000..13fc6550d3c --- /dev/null +++ b/eval/src/tests/instruction/generic_join/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_generic_join_test_app TEST + SOURCES + generic_join_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_generic_join_test_app COMMAND eval_generic_join_test_app) diff --git a/eval/src/tests/instruction/generic_join/generic_join_test.cpp b/eval/src/tests/instruction/generic_join/generic_join_test.cpp new file mode 100644 index 00000000000..53df23d77be --- /dev/null +++ b/eval/src/tests/instruction/generic_join/generic_join_test.cpp @@ -0,0 +1,119 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include +#include +#include +#include +#include +#include +#include + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::eval::instruction; +using namespace vespalib::eval::test; + +using vespalib::make_string_short::fmt; + +std::vector join_layouts = { + {}, {}, + {x(5)}, {x(5)}, + {x(5)}, {y(5)}, + {x(5)}, {x(5),y(5)}, + {y(3)}, {x(2),z(3)}, + {x(3),y(5)}, {y(5),z(7)}, + float_cells({x(3),y(5)}), {y(5),z(7)}, + {x(3),y(5)}, float_cells({y(5),z(7)}), + float_cells({x(3),y(5)}), float_cells({y(5),z(7)}), + {x({"a","b","c"})}, {x({"a","b","c"})}, + {x({"a","b","c"})}, {x({"a","b"})}, + {x({"a","b","c"})}, {y({"foo","bar","baz"})}, + {x({"a","b","c"})}, {x({"a","b","c"}),y({"foo","bar","baz"})}, + {x({"a","b"}),y({"foo","bar","baz"})}, {x({"a","b","c"}),y({"foo","bar"})}, + {x({"a","b"}),y({"foo","bar","baz"})}, {y({"foo","bar"}),z({"i","j","k","l"})}, + float_cells({x({"a","b"}),y({"foo","bar","baz"})}), {y({"foo","bar"}),z({"i","j","k","l"})}, + {x({"a","b"}),y({"foo","bar","baz"})}, float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), + float_cells({x({"a","b"}),y({"foo","bar","baz"})}), float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), + {x(3),y({"foo", "bar"})}, {y({"foo", "bar"}),z(7)}, + {x({"a","b","c"}),y(5)}, {y(5),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y(5)}), {y(5),z({"i","j","k","l"})}, + {x({"a","b","c"}),y(5)}, float_cells({y(5),z({"i","j","k","l"})}), + float_cells({x({"a","b","c"}),y(5)}), float_cells({y(5),z({"i","j","k","l"})}) +}; + +TensorSpec simple_tensor_join(const TensorSpec &a, const TensorSpec &b, join_fun_t function) { + Stash stash; + const auto &engine = SimpleTensorEngine::ref(); + auto lhs = engine.from_spec(a); + auto rhs = engine.from_spec(b); + const auto &result = engine.join(*lhs, *rhs, function, stash); + return engine.to_spec(result); +} + +TensorSpec perform_generic_join(const TensorSpec &a, const TensorSpec &b, join_fun_t function) { + Stash stash; + const auto &factory = SimpleValueBuilderFactory::get(); + auto lhs = value_from_spec(a, factory); + auto rhs = value_from_spec(b, factory); + auto my_op = GenericJoin::make_instruction(lhs->type(), rhs->type(), function, factory, stash); + InterpretedFunction::EvalSingle single(my_op); + return spec_from_value(single.eval(std::vector({*lhs,*rhs}))); +} + +TEST(GenericJoinTest, dense_join_plan_can_be_created) { + auto lhs = ValueType::from_spec("tensor(a{},b[6],c[5],e[3],f[2],g{})"); + auto rhs = ValueType::from_spec("tensor(a{},b[6],c[5],d[4],h{})"); + auto plan = DenseJoinPlan(lhs, rhs); + std::vector expect_loop = {30,4,6}; + std::vector expect_lhs_stride = {6,0,1}; + std::vector expect_rhs_stride = {4,1,0}; + EXPECT_EQ(plan.lhs_size, 180); + EXPECT_EQ(plan.rhs_size, 120); + EXPECT_EQ(plan.out_size, 720); + EXPECT_EQ(plan.loop_cnt, expect_loop); + EXPECT_EQ(plan.lhs_stride, expect_lhs_stride); + EXPECT_EQ(plan.rhs_stride, expect_rhs_stride); +} + +TEST(GenericJoinTest, sparse_join_plan_can_be_created) { + auto lhs = ValueType::from_spec("tensor(a{},b[6],c[5],e[3],f[2],g{})"); + auto rhs = ValueType::from_spec("tensor(b[6],c[5],d[4],g{},h{})"); + auto plan = SparseJoinPlan(lhs, rhs); + using SRC = SparseJoinPlan::Source; + std::vector expect_sources = {SRC::LHS,SRC::BOTH,SRC::RHS}; + std::vector expect_lhs_overlap = {1}; + std::vector expect_rhs_overlap = {0}; + EXPECT_EQ(plan.sources, expect_sources); + EXPECT_EQ(plan.lhs_overlap, expect_lhs_overlap); + EXPECT_EQ(plan.rhs_overlap, expect_rhs_overlap); +} + +TEST(GenericJoinTest, dense_join_plan_can_be_executed) { + auto plan = DenseJoinPlan(ValueType::from_spec("tensor(a[2])"), + ValueType::from_spec("tensor(b[3])")); + std::vector a({1, 2}); + std::vector b({3, 4, 5}); + std::vector c(6, 0); + std::vector expect = {3,4,5,6,8,10}; + ASSERT_EQ(plan.out_size, 6); + int *dst = &c[0]; + auto cell_join = [&](size_t a_idx, size_t b_idx) { *dst++ = (a[a_idx] * b[b_idx]); }; + plan.execute(0, 0, cell_join); + EXPECT_EQ(c, expect); +} + +TEST(GenericJoinTest, generic_join_works_for_simple_values) { + ASSERT_TRUE((join_layouts.size() % 2) == 0); + for (size_t i = 0; i < join_layouts.size(); i += 2) { + TensorSpec lhs = spec(join_layouts[i], Div16(N())); + TensorSpec rhs = spec(join_layouts[i + 1], Div16(N())); + for (auto fun: {operation::Add::f, operation::Sub::f, operation::Mul::f, operation::Div::f}) { + SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); + auto expect = simple_tensor_join(lhs, rhs, fun); + auto actual = perform_generic_join(lhs, rhs, fun); + EXPECT_EQ(actual, expect); + } + } +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp b/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp index 31e82a01070..1139d35d847 100644 --- a/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp +++ b/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp @@ -10,6 +10,7 @@ using namespace vespalib; using namespace vespalib::eval; +using namespace vespalib::eval::instruction; using namespace vespalib::eval::test; using vespalib::make_string_short::fmt; @@ -50,7 +51,7 @@ TEST(GenericRenameTest, dense_rename_plan_can_be_created_and_executed) { std::vector from({"a", "c", "e"}); std::vector to({"f", "a", "b"}); ValueType renamed = lhs.rename(from, to); - auto plan = instruction::DenseRenamePlan(lhs, renamed, from, to); + auto plan = DenseRenamePlan(lhs, renamed, from, to); std::vector expect_loop = {15,2,7}; std::vector expect_stride = {7,105,1}; EXPECT_EQ(plan.subspace_size, 210); @@ -80,7 +81,7 @@ TEST(GenericRenameTest, sparse_rename_plan_can_be_created) { std::vector from({"a", "c", "e"}); std::vector to({"f", "a", "b"}); ValueType renamed = lhs.rename(from, to); - auto plan = instruction::SparseRenamePlan(lhs, renamed, from, to); + auto plan = SparseRenamePlan(lhs, renamed, from, to); EXPECT_EQ(plan.mapped_dims, 4); std::vector expect = {2,0,1,3}; EXPECT_EQ(plan.output_dimensions, expect); @@ -99,7 +100,7 @@ TensorSpec perform_generic_rename(const TensorSpec &a, const ValueType &res_type { Stash stash; auto lhs = value_from_spec(a, factory); - auto my_op = instruction::GenericRename::make_instruction(lhs->type(), res_type, ft.from, ft.to, factory, stash); + auto my_op = GenericRename::make_instruction(lhs->type(), res_type, ft.from, ft.to, factory, stash); InterpretedFunction::EvalSingle single(my_op); return spec_from_value(single.eval(std::vector({*lhs}))); } diff --git a/eval/src/vespa/eval/eval/CMakeLists.txt b/eval/src/vespa/eval/eval/CMakeLists.txt index 53ee6ed87c0..84cebe8cfd0 100644 --- a/eval/src/vespa/eval/eval/CMakeLists.txt +++ b/eval/src/vespa/eval/eval/CMakeLists.txt @@ -25,9 +25,7 @@ vespa_add_library(eval_eval OBJECT tensor.cpp tensor_engine.cpp tensor_function.cpp - tensor_instructions.cpp tensor_nodes.cpp - tensor_plans.cpp tensor_spec.cpp value.cpp value_codec.cpp diff --git a/eval/src/vespa/eval/eval/tensor_instructions.cpp b/eval/src/vespa/eval/eval/tensor_instructions.cpp deleted file mode 100644 index d839bfdad50..00000000000 --- a/eval/src/vespa/eval/eval/tensor_instructions.cpp +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "tensor_instructions.h" -#include "tensor_plans.h" -#include "inline_operation.h" -#include -#include - -namespace vespalib::eval::tensor_instruction { - -using State = InterpretedFunction::State; -using Instruction = InterpretedFunction::Instruction; - -//----------------------------------------------------------------------------- - -namespace { - -//----------------------------------------------------------------------------- - -template uint64_t wrap_param(const IN &value_in) { - const T &value = value_in; - static_assert(sizeof(uint64_t) == sizeof(&value)); - return (uint64_t)&value; -} - -template const T &unwrap_param(uint64_t param) { - return *((const T *)param); -} - -//----------------------------------------------------------------------------- - -struct JoinParam { - ValueType res_type; - SparseJoinPlan sparse_plan; - DenseJoinPlan dense_plan; - join_fun_t function; - const ValueBuilderFactory &factory; - JoinParam(const ValueType &lhs_type, const ValueType &rhs_type, - join_fun_t function_in, const ValueBuilderFactory &factory_in) - : res_type(ValueType::join(lhs_type, rhs_type)), - sparse_plan(lhs_type, rhs_type), - dense_plan(lhs_type, rhs_type), - function(function_in), - factory(factory_in) - { - assert(!res_type.is_error()); - } - ~JoinParam(); -}; -JoinParam::~JoinParam() = default; - -//----------------------------------------------------------------------------- - -// Contains various state needed to perform the sparse part (all -// mapped dimensions) of the join operation. Performs swapping of -// sparse indexes to ensure that we look up entries from the smallest -// index in the largest index. -struct SparseJoinState { - bool swapped; - const Value::Index &first_index; - const Value::Index &second_index; - const std::vector &second_view_dims; - std::vector full_address; - std::vector first_address; - std::vector address_overlap; - std::vector second_only_address; - size_t lhs_subspace; - size_t rhs_subspace; - size_t &first_subspace; - size_t &second_subspace; - - SparseJoinState(const SparseJoinPlan &plan, const Value::Index &lhs, const Value::Index &rhs) - : swapped(rhs.size() < lhs.size()), - first_index(swapped ? rhs : lhs), second_index(swapped ? lhs : rhs), - second_view_dims(swapped ? plan.lhs_overlap : plan.rhs_overlap), - full_address(plan.sources.size()), - first_address(), address_overlap(), second_only_address(), - lhs_subspace(), rhs_subspace(), - first_subspace(swapped ? rhs_subspace : lhs_subspace), - second_subspace(swapped ? lhs_subspace : rhs_subspace) - { - auto first_source = swapped ? SparseJoinPlan::Source::RHS : SparseJoinPlan::Source::LHS; - for (size_t i = 0; i < full_address.size(); ++i) { - if (plan.sources[i] == SparseJoinPlan::Source::BOTH) { - first_address.push_back(&full_address[i]); - address_overlap.push_back(&full_address[i]); - } else if (plan.sources[i] == first_source) { - first_address.push_back(&full_address[i]); - } else { - second_only_address.push_back(&full_address[i]); - } - } - } - ~SparseJoinState(); -}; -SparseJoinState::~SparseJoinState() = default; - -template -void my_generic_join(State &state, uint64_t param_in) { - const auto ¶m = unwrap_param(param_in); - Fun fun(param.function); - const Value &lhs = state.peek(1); - const Value &rhs = state.peek(0); - auto lhs_cells = lhs.cells().typify(); - auto rhs_cells = rhs.cells().typify(); - SparseJoinState sparse(param.sparse_plan, lhs.index(), rhs.index()); - auto builder = param.factory.create_value_builder(param.res_type, param.sparse_plan.sources.size(), param.dense_plan.out_size, sparse.first_index.size()); - auto outer = sparse.first_index.create_view({}); - auto inner = sparse.second_index.create_view(sparse.second_view_dims); - outer->lookup({}); - while (outer->next_result(sparse.first_address, sparse.first_subspace)) { - inner->lookup(sparse.address_overlap); - while (inner->next_result(sparse.second_only_address, sparse.second_subspace)) { - OCT *dst = builder->add_subspace(sparse.full_address).begin(); - auto join_cells = [&](size_t lhs_idx, size_t rhs_idx) { *dst++ = fun(lhs_cells[lhs_idx], rhs_cells[rhs_idx]); }; - param.dense_plan.execute(param.dense_plan.lhs_size * sparse.lhs_subspace, param.dense_plan.rhs_size * sparse.rhs_subspace, join_cells); - } - } - auto &result = state.stash.create>(builder->build(std::move(builder))); - const Value &result_ref = *(result.get()); - state.pop_pop_push(result_ref); -}; - -struct SelectGenericJoin { - template static auto invoke() { - return my_generic_join; - } -}; - -//----------------------------------------------------------------------------- - -} // namespace - -//----------------------------------------------------------------------------- - -using JoinTypify = TypifyValue; - -Instruction make_join(const ValueType &lhs_type, const ValueType &rhs_type, join_fun_t function, - const ValueBuilderFactory &factory, Stash &stash) -{ - auto ¶m = stash.create(lhs_type, rhs_type, function, factory); - auto fun = typify_invoke<4,JoinTypify,SelectGenericJoin>(lhs_type.cell_type(), rhs_type.cell_type(), param.res_type.cell_type(), function); - return Instruction(fun, wrap_param(param)); -} - -//----------------------------------------------------------------------------- - -} diff --git a/eval/src/vespa/eval/eval/tensor_instructions.h b/eval/src/vespa/eval/eval/tensor_instructions.h deleted file mode 100644 index f5c5e88d07c..00000000000 --- a/eval/src/vespa/eval/eval/tensor_instructions.h +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "interpreted_function.h" - -namespace vespalib { - -class Stash; - -namespace eval { - -struct JoinPlan; - -namespace tensor_instruction { - -using join_fun_t = double (*)(double, double); - -InterpretedFunction::Instruction make_join(const ValueType &lhs_type, const ValueType &rhs_type, join_fun_t function, - const ValueBuilderFactory &factory, Stash &stash); - -} -} -} diff --git a/eval/src/vespa/eval/eval/tensor_plans.cpp b/eval/src/vespa/eval/eval/tensor_plans.cpp deleted file mode 100644 index 203ca72c92a..00000000000 --- a/eval/src/vespa/eval/eval/tensor_plans.cpp +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "tensor_plans.h" -#include -#include -#include - -namespace vespalib::eval { - -//----------------------------------------------------------------------------- - -DenseJoinPlan::DenseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type) - : lhs_size(1), rhs_size(1), out_size(1), loop_cnt(), lhs_stride(), rhs_stride() -{ - enum class Case { NONE, LHS, RHS, BOTH }; - Case prev_case = Case::NONE; - auto update_plan = [&](Case my_case, size_t my_size, size_t in_lhs, size_t in_rhs) { - if (my_case == prev_case) { - assert(!loop_cnt.empty()); - loop_cnt.back() *= my_size; - } else { - loop_cnt.push_back(my_size); - lhs_stride.push_back(in_lhs); - rhs_stride.push_back(in_rhs); - prev_case = my_case; - } - }; - auto visitor = overload - { - [&](visit_ranges_first, const auto &a) { update_plan(Case::LHS, a.size, 1, 0); }, - [&](visit_ranges_second, const auto &b) { update_plan(Case::RHS, b.size, 0, 1); }, - [&](visit_ranges_both, const auto &a, const auto &) { update_plan(Case::BOTH, a.size, 1, 1); } - }; - auto lhs_dims = lhs_type.nontrivial_indexed_dimensions(); - auto rhs_dims = rhs_type.nontrivial_indexed_dimensions(); - visit_ranges(visitor, lhs_dims.begin(), lhs_dims.end(), rhs_dims.begin(), rhs_dims.end(), - [](const auto &a, const auto &b){ return (a.name < b.name); }); - for (size_t i = loop_cnt.size(); i-- > 0; ) { - out_size *= loop_cnt[i]; - if (lhs_stride[i] != 0) { - lhs_stride[i] = lhs_size; - lhs_size *= loop_cnt[i]; - } - if (rhs_stride[i] != 0) { - rhs_stride[i] = rhs_size; - rhs_size *= loop_cnt[i]; - } - } -} - -DenseJoinPlan::~DenseJoinPlan() = default; - -//----------------------------------------------------------------------------- - -SparseJoinPlan::SparseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type) - : sources(), lhs_overlap(), rhs_overlap() -{ - size_t lhs_idx = 0; - size_t rhs_idx = 0; - auto visitor = overload - { - [&](visit_ranges_first, const auto &) { - sources.push_back(Source::LHS); - ++lhs_idx; - }, - [&](visit_ranges_second, const auto &) { - sources.push_back(Source::RHS); - ++rhs_idx; - }, - [&](visit_ranges_both, const auto &, const auto &) { - sources.push_back(Source::BOTH); - lhs_overlap.push_back(lhs_idx++); - rhs_overlap.push_back(rhs_idx++); - } - }; - auto lhs_dims = lhs_type.mapped_dimensions(); - auto rhs_dims = rhs_type.mapped_dimensions(); - visit_ranges(visitor, lhs_dims.begin(), lhs_dims.end(), rhs_dims.begin(), rhs_dims.end(), - [](const auto &a, const auto &b){ return (a.name < b.name); }); -} - -SparseJoinPlan::~SparseJoinPlan() = default; - -//----------------------------------------------------------------------------- - -} diff --git a/eval/src/vespa/eval/eval/tensor_plans.h b/eval/src/vespa/eval/eval/tensor_plans.h deleted file mode 100644 index 9a924a8ba48..00000000000 --- a/eval/src/vespa/eval/eval/tensor_plans.h +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "value_type.h" -#include -#include - -namespace vespalib::eval { - -class ValueBuilderFactory; - -//----------------------------------------------------------------------------- - -/** - * Plan for how to traverse two partially overlapping dense subspaces - * in parallel, identifying all matching cell index combinations, in - * the exact order the joined cells will be stored in the result. The - * plan can be made up-front during tensor function compilation. - **/ -struct DenseJoinPlan { - size_t lhs_size; - size_t rhs_size; - size_t out_size; - std::vector loop_cnt; - std::vector lhs_stride; - std::vector rhs_stride; - DenseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type); - ~DenseJoinPlan(); - template void execute(size_t lhs, size_t rhs, F &&f) const { - switch(loops_left(0)) { - case 0: return execute_few(0, lhs, rhs, std::forward(f)); - case 1: return execute_few(0, lhs, rhs, std::forward(f)); - case 2: return execute_few(0, lhs, rhs, std::forward(f)); - case 3: return execute_few(0, lhs, rhs, std::forward(f)); - default: return execute_many(0, lhs, rhs, std::forward(f)); - } - } -private: - size_t loops_left(size_t idx) const { return (loop_cnt.size() - idx); } - template void execute_few(size_t idx, size_t lhs, size_t rhs, F &&f) const { - if constexpr (N == 0) { - f(lhs, rhs); - } else { - for (size_t i = 0; i < loop_cnt[idx]; ++i, lhs += lhs_stride[idx], rhs += rhs_stride[idx]) { - execute_few(idx + 1, lhs, rhs, std::forward(f)); - } - } - } - template void execute_many(size_t idx, size_t lhs, size_t rhs, F &&f) const { - for (size_t i = 0; i < loop_cnt[idx]; ++i, lhs += lhs_stride[idx], rhs += rhs_stride[idx]) { - if (loops_left(idx + 1) == 3) { - execute_few(idx + 1, lhs, rhs, std::forward(f)); - } else { - execute_many(idx + 1, lhs, rhs, std::forward(f)); - } - } - } -}; - -/** - * Plan for how to join the sparse part (all mapped dimensions) - * between two values. The plan can be made up-front during tensor - * function compilation. - **/ -struct SparseJoinPlan { - enum class Source { LHS, RHS, BOTH }; - std::vector sources; - std::vector lhs_overlap; - std::vector rhs_overlap; - SparseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type); - ~SparseJoinPlan(); -}; - -//----------------------------------------------------------------------------- - -} diff --git a/eval/src/vespa/eval/instruction/CMakeLists.txt b/eval/src/vespa/eval/instruction/CMakeLists.txt index f5a7de96f0b..e5aae50750d 100644 --- a/eval/src/vespa/eval/instruction/CMakeLists.txt +++ b/eval/src/vespa/eval/instruction/CMakeLists.txt @@ -2,5 +2,6 @@ vespa_add_library(eval_instruction OBJECT SOURCES + generic_join generic_rename ) diff --git a/eval/src/vespa/eval/instruction/generic_join.cpp b/eval/src/vespa/eval/instruction/generic_join.cpp new file mode 100644 index 00000000000..8a1a199effa --- /dev/null +++ b/eval/src/vespa/eval/instruction/generic_join.cpp @@ -0,0 +1,232 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "generic_join.h" +#include +#include +#include +#include +#include +#include + +namespace vespalib::eval::instruction { + +using State = InterpretedFunction::State; +using Instruction = InterpretedFunction::Instruction; + +namespace { + +//----------------------------------------------------------------------------- + +template uint64_t wrap_param(const IN &value_in) { + const T &value = value_in; + static_assert(sizeof(uint64_t) == sizeof(&value)); + return (uint64_t)&value; +} + +template const T &unwrap_param(uint64_t param) { + return *((const T *)param); +} + +//----------------------------------------------------------------------------- + +struct JoinParam { + ValueType res_type; + SparseJoinPlan sparse_plan; + DenseJoinPlan dense_plan; + join_fun_t function; + const ValueBuilderFactory &factory; + JoinParam(const ValueType &lhs_type, const ValueType &rhs_type, + join_fun_t function_in, const ValueBuilderFactory &factory_in) + : res_type(ValueType::join(lhs_type, rhs_type)), + sparse_plan(lhs_type, rhs_type), + dense_plan(lhs_type, rhs_type), + function(function_in), + factory(factory_in) + { + assert(!res_type.is_error()); + } + ~JoinParam(); +}; +JoinParam::~JoinParam() = default; + +//----------------------------------------------------------------------------- + +// Contains various state needed to perform the sparse part (all +// mapped dimensions) of the join operation. Performs swapping of +// sparse indexes to ensure that we look up entries from the smallest +// index in the largest index. +struct SparseJoinState { + bool swapped; + const Value::Index &first_index; + const Value::Index &second_index; + const std::vector &second_view_dims; + std::vector full_address; + std::vector first_address; + std::vector address_overlap; + std::vector second_only_address; + size_t lhs_subspace; + size_t rhs_subspace; + size_t &first_subspace; + size_t &second_subspace; + + SparseJoinState(const SparseJoinPlan &plan, const Value::Index &lhs, const Value::Index &rhs) + : swapped(rhs.size() < lhs.size()), + first_index(swapped ? rhs : lhs), second_index(swapped ? lhs : rhs), + second_view_dims(swapped ? plan.lhs_overlap : plan.rhs_overlap), + full_address(plan.sources.size()), + first_address(), address_overlap(), second_only_address(), + lhs_subspace(), rhs_subspace(), + first_subspace(swapped ? rhs_subspace : lhs_subspace), + second_subspace(swapped ? lhs_subspace : rhs_subspace) + { + auto first_source = swapped ? SparseJoinPlan::Source::RHS : SparseJoinPlan::Source::LHS; + for (size_t i = 0; i < full_address.size(); ++i) { + if (plan.sources[i] == SparseJoinPlan::Source::BOTH) { + first_address.push_back(&full_address[i]); + address_overlap.push_back(&full_address[i]); + } else if (plan.sources[i] == first_source) { + first_address.push_back(&full_address[i]); + } else { + second_only_address.push_back(&full_address[i]); + } + } + } + ~SparseJoinState(); +}; +SparseJoinState::~SparseJoinState() = default; + +/* +template +void generic_join() +*/ + +template +void my_generic_join_op(State &state, uint64_t param_in) { + const auto ¶m = unwrap_param(param_in); + Fun fun(param.function); + const Value &lhs = state.peek(1); + const Value &rhs = state.peek(0); + auto lhs_cells = lhs.cells().typify(); + auto rhs_cells = rhs.cells().typify(); + SparseJoinState sparse(param.sparse_plan, lhs.index(), rhs.index()); + auto builder = param.factory.create_value_builder(param.res_type, param.sparse_plan.sources.size(), param.dense_plan.out_size, sparse.first_index.size()); + auto outer = sparse.first_index.create_view({}); + auto inner = sparse.second_index.create_view(sparse.second_view_dims); + outer->lookup({}); + while (outer->next_result(sparse.first_address, sparse.first_subspace)) { + inner->lookup(sparse.address_overlap); + while (inner->next_result(sparse.second_only_address, sparse.second_subspace)) { + OCT *dst = builder->add_subspace(sparse.full_address).begin(); + auto join_cells = [&](size_t lhs_idx, size_t rhs_idx) { *dst++ = fun(lhs_cells[lhs_idx], rhs_cells[rhs_idx]); }; + param.dense_plan.execute(param.dense_plan.lhs_size * sparse.lhs_subspace, param.dense_plan.rhs_size * sparse.rhs_subspace, join_cells); + } + } + auto &result = state.stash.create>(builder->build(std::move(builder))); + const Value &result_ref = *(result.get()); + state.pop_pop_push(result_ref); +}; + +struct SelectGenericJoinOp { + template static auto invoke() { + return my_generic_join_op; + } +}; + +//----------------------------------------------------------------------------- + +} // namespace + +//----------------------------------------------------------------------------- + +DenseJoinPlan::DenseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type) + : lhs_size(1), rhs_size(1), out_size(1), loop_cnt(), lhs_stride(), rhs_stride() +{ + enum class Case { NONE, LHS, RHS, BOTH }; + Case prev_case = Case::NONE; + auto update_plan = [&](Case my_case, size_t my_size, size_t in_lhs, size_t in_rhs) { + if (my_case == prev_case) { + assert(!loop_cnt.empty()); + loop_cnt.back() *= my_size; + } else { + loop_cnt.push_back(my_size); + lhs_stride.push_back(in_lhs); + rhs_stride.push_back(in_rhs); + prev_case = my_case; + } + }; + auto visitor = overload + { + [&](visit_ranges_first, const auto &a) { update_plan(Case::LHS, a.size, 1, 0); }, + [&](visit_ranges_second, const auto &b) { update_plan(Case::RHS, b.size, 0, 1); }, + [&](visit_ranges_both, const auto &a, const auto &) { update_plan(Case::BOTH, a.size, 1, 1); } + }; + auto lhs_dims = lhs_type.nontrivial_indexed_dimensions(); + auto rhs_dims = rhs_type.nontrivial_indexed_dimensions(); + visit_ranges(visitor, lhs_dims.begin(), lhs_dims.end(), rhs_dims.begin(), rhs_dims.end(), + [](const auto &a, const auto &b){ return (a.name < b.name); }); + for (size_t i = loop_cnt.size(); i-- > 0; ) { + out_size *= loop_cnt[i]; + if (lhs_stride[i] != 0) { + lhs_stride[i] = lhs_size; + lhs_size *= loop_cnt[i]; + } + if (rhs_stride[i] != 0) { + rhs_stride[i] = rhs_size; + rhs_size *= loop_cnt[i]; + } + } +} + +DenseJoinPlan::~DenseJoinPlan() = default; + +//----------------------------------------------------------------------------- + +SparseJoinPlan::SparseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type) + : sources(), lhs_overlap(), rhs_overlap() +{ + size_t lhs_idx = 0; + size_t rhs_idx = 0; + auto visitor = overload + { + [&](visit_ranges_first, const auto &) { + sources.push_back(Source::LHS); + ++lhs_idx; + }, + [&](visit_ranges_second, const auto &) { + sources.push_back(Source::RHS); + ++rhs_idx; + }, + [&](visit_ranges_both, const auto &, const auto &) { + sources.push_back(Source::BOTH); + lhs_overlap.push_back(lhs_idx++); + rhs_overlap.push_back(rhs_idx++); + } + }; + auto lhs_dims = lhs_type.mapped_dimensions(); + auto rhs_dims = rhs_type.mapped_dimensions(); + visit_ranges(visitor, lhs_dims.begin(), lhs_dims.end(), rhs_dims.begin(), rhs_dims.end(), + [](const auto &a, const auto &b){ return (a.name < b.name); }); +} + +SparseJoinPlan::~SparseJoinPlan() = default; + +//----------------------------------------------------------------------------- + +} + +//----------------------------------------------------------------------------- + +namespace vespalib::eval::instruction { + +using JoinTypify = TypifyValue; + +Instruction +GenericJoin::make_instruction(const ValueType &lhs_type, const ValueType &rhs_type, join_fun_t function, + const ValueBuilderFactory &factory, Stash &stash) +{ + auto ¶m = stash.create(lhs_type, rhs_type, function, factory); + auto fun = typify_invoke<4,JoinTypify,SelectGenericJoinOp>(lhs_type.cell_type(), rhs_type.cell_type(), param.res_type.cell_type(), function); + return Instruction(fun, wrap_param(param)); +} + +} // namespace diff --git a/eval/src/vespa/eval/instruction/generic_join.h b/eval/src/vespa/eval/instruction/generic_join.h new file mode 100644 index 00000000000..81e56494897 --- /dev/null +++ b/eval/src/vespa/eval/instruction/generic_join.h @@ -0,0 +1,87 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include +#include + +namespace vespalib { class Stash; } +namespace vespalib::eval { class ValueBuilderFactory; } + +namespace vespalib::eval::instruction { + +using join_fun_t = double (*)(double, double); + +//----------------------------------------------------------------------------- + +struct GenericJoin { + static InterpretedFunction::Instruction + make_instruction(const ValueType &lhs_type, const ValueType &rhs_type, join_fun_t function, + const ValueBuilderFactory &factory, Stash &stash); +}; + +//----------------------------------------------------------------------------- + +/** + * Plan for how to traverse two partially overlapping dense subspaces + * in parallel, identifying all matching cell index combinations, in + * the exact order the joined cells will be stored in the result. The + * plan can be made up-front during tensor function compilation. + **/ +struct DenseJoinPlan { + size_t lhs_size; + size_t rhs_size; + size_t out_size; + std::vector loop_cnt; + std::vector lhs_stride; + std::vector rhs_stride; + DenseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type); + ~DenseJoinPlan(); + template void execute(size_t lhs, size_t rhs, F &&f) const { + switch(loops_left(0)) { + case 0: return execute_few(0, lhs, rhs, std::forward(f)); + case 1: return execute_few(0, lhs, rhs, std::forward(f)); + case 2: return execute_few(0, lhs, rhs, std::forward(f)); + case 3: return execute_few(0, lhs, rhs, std::forward(f)); + default: return execute_many(0, lhs, rhs, std::forward(f)); + } + } +private: + size_t loops_left(size_t idx) const { return (loop_cnt.size() - idx); } + template void execute_few(size_t idx, size_t lhs, size_t rhs, F &&f) const { + if constexpr (N == 0) { + f(lhs, rhs); + } else { + for (size_t i = 0; i < loop_cnt[idx]; ++i, lhs += lhs_stride[idx], rhs += rhs_stride[idx]) { + execute_few(idx + 1, lhs, rhs, std::forward(f)); + } + } + } + template void execute_many(size_t idx, size_t lhs, size_t rhs, F &&f) const { + for (size_t i = 0; i < loop_cnt[idx]; ++i, lhs += lhs_stride[idx], rhs += rhs_stride[idx]) { + if (loops_left(idx + 1) == 3) { + execute_few(idx + 1, lhs, rhs, std::forward(f)); + } else { + execute_many(idx + 1, lhs, rhs, std::forward(f)); + } + } + } +}; + +/** + * Plan for how to join the sparse part (all mapped dimensions) + * between two values. The plan can be made up-front during tensor + * function compilation. + **/ +struct SparseJoinPlan { + enum class Source { LHS, RHS, BOTH }; + std::vector sources; + std::vector lhs_overlap; + std::vector rhs_overlap; + SparseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type); + ~SparseJoinPlan(); +}; + +//----------------------------------------------------------------------------- + +} // namespace -- cgit v1.2.3 From fee153c0d1dc84b9fdbb4d926bc35a7dd6e11a05 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Thu, 24 Sep 2020 14:03:46 +0000 Subject: full names for PackedMixedTensorBuilderFactory etc --- .../packed_mappings/packed_mappings_test.cpp | 12 ++--- .../tensor/packed_mappings/packed_mixed_test.cpp | 6 +-- eval/src/vespa/eval/tensor/mixed/CMakeLists.txt | 4 +- .../eval/tensor/mixed/packed_mixed_builder.cpp | 53 ---------------------- .../vespa/eval/tensor/mixed/packed_mixed_builder.h | 40 ---------------- .../eval/tensor/mixed/packed_mixed_factory.cpp | 33 -------------- .../vespa/eval/tensor/mixed/packed_mixed_factory.h | 20 -------- .../vespa/eval/tensor/mixed/packed_mixed_tensor.h | 4 +- .../tensor/mixed/packed_mixed_tensor_builder.cpp | 53 ++++++++++++++++++++++ .../tensor/mixed/packed_mixed_tensor_builder.h | 40 ++++++++++++++++ .../mixed/packed_mixed_tensor_builder_factory.cpp | 33 ++++++++++++++ .../mixed/packed_mixed_tensor_builder_factory.h | 20 ++++++++ 12 files changed, 159 insertions(+), 159 deletions(-) delete mode 100644 eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.cpp delete mode 100644 eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h delete mode 100644 eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.cpp delete mode 100644 eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.h create mode 100644 eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.cpp create mode 100644 eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.h create mode 100644 eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.cpp create mode 100644 eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h diff --git a/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp b/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp index 70f98904809..c8814372bf5 100644 --- a/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp +++ b/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp @@ -4,8 +4,8 @@ #include #include #include -#include -#include +#include +#include #include #include #include @@ -137,7 +137,7 @@ TEST_F(MappingsBuilderTest, some_random) class MixedBuilderTest : public ::testing::Test { public: - std::unique_ptr> builder; + std::unique_ptr> builder; std::unique_ptr built; MixedBuilderTest() = default; @@ -170,7 +170,7 @@ TEST_F(MixedBuilderTest, empty_mapping) size_t dsss = type.dense_subspace_size(); EXPECT_GT(dims, 0); EXPECT_GT(dsss, 0); - builder = std::make_unique>(type, dims, dsss, 3); + builder = std::make_unique>(type, dims, dsss, 3); build_and_compare(0); } } @@ -183,7 +183,7 @@ TEST_F(MixedBuilderTest, just_one) size_t dims = type.count_mapped_dimensions(); size_t dsss = type.dense_subspace_size(); EXPECT_GT(dsss, 0); - builder = std::make_unique>(type, dims, dsss, 3); + builder = std::make_unique>(type, dims, dsss, 3); auto address = generate_random_address(dims); auto ref = builder->add_subspace(address); EXPECT_EQ(ref.size(), dsss); @@ -203,7 +203,7 @@ TEST_F(MixedBuilderTest, some_random) uint32_t dsss = type.dense_subspace_size(); EXPECT_GT(dims, 0); EXPECT_GT(dsss, 0); - builder = std::make_unique>(type, dims, dsss, 3); + builder = std::make_unique>(type, dims, dsss, 3); uint32_t cnt = random_range(dims*5, dims*20); printf("MixBuild: generate %u addresses for %u dims\n", cnt, dims); diff --git a/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp b/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp index eb7607c13b8..8ebed853df6 100644 --- a/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp +++ b/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include using namespace vespalib::eval; @@ -27,7 +27,7 @@ std::vector layouts = { TEST(PackedMixedTest, packed_mixed_tensors_can_be_converted_from_and_to_tensor_spec) { for (const auto &layout: layouts) { TensorSpec expect = spec(layout, N()); - std::unique_ptr value = value_from_spec(expect, PackedMixedFactory()); + std::unique_ptr value = value_from_spec(expect, PackedMixedTensorBuilderFactory()); TensorSpec actual = spec_from_value(*value); EXPECT_EQ(actual, expect); } @@ -35,7 +35,7 @@ TEST(PackedMixedTest, packed_mixed_tensors_can_be_converted_from_and_to_tensor_s TEST(PackedMixedTest, packed_mixed_tensors_can_be_built_and_inspected) { ValueType type = ValueType::from_spec("tensor(x{},y[2],z{})"); - PackedMixedFactory factory; + PackedMixedTensorBuilderFactory factory; std::unique_ptr> builder = factory.create_value_builder(type); float seq = 0.0; for (vespalib::string x: {"a", "b", "c"}) { diff --git a/eval/src/vespa/eval/tensor/mixed/CMakeLists.txt b/eval/src/vespa/eval/tensor/mixed/CMakeLists.txt index 2621ec616c0..ceded3a7380 100644 --- a/eval/src/vespa/eval/tensor/mixed/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/mixed/CMakeLists.txt @@ -5,7 +5,7 @@ vespa_add_library(eval_tensor_mixed OBJECT packed_labels.cpp packed_mappings.cpp packed_mappings_builder.cpp - packed_mixed_factory.cpp + packed_mixed_tensor_builder_factory.cpp packed_mixed_tensor.cpp - packed_mixed_builder.cpp + packed_mixed_tensor_builder.cpp ) diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.cpp deleted file mode 100644 index 1816e06bf25..00000000000 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.cpp +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "packed_mixed_builder.h" - -namespace vespalib::eval::packed_mixed_tensor { - -template -ArrayRef -PackedMixedBuilder::add_subspace(const std::vector &addr) -{ - uint32_t idx = _mappings_builder.add_mapping_for(addr); - size_t offset = idx * _subspace_size; - assert(offset <= _cells.size()); - if (offset == _cells.size()) { - _cells.resize(offset + _subspace_size); - } - return ArrayRef(&_cells[offset], _subspace_size); -} - - -template -std::unique_ptr -PackedMixedBuilder::build(std::unique_ptr>) -{ - size_t self_size = sizeof(PackedMixedTensor); - size_t mappings_size = _mappings_builder.extra_memory(); - // align: - mappings_size += 15ul; - mappings_size &= ~15ul; - size_t cells_size = sizeof(T) * _cells.size(); - size_t total_size = self_size + mappings_size + cells_size; - - char *mem = (char *) operator new(total_size); - char *mappings_mem = mem + self_size; - char *cells_mem = mappings_mem + mappings_size; - - // fill mapping data: - auto mappings = _mappings_builder.target_memory(mappings_mem, cells_mem); - - // copy cells: - memcpy(cells_mem, &_cells[0], cells_size); - ConstArrayRef cells((T *)cells_mem, _cells.size()); - - PackedMixedTensor * built = - new (mem) PackedMixedTensor(_type, TypedCells(cells), mappings); - - return std::unique_ptr(built); -} - -template class PackedMixedBuilder; -template class PackedMixedBuilder; - -} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h deleted file mode 100644 index c851b839756..00000000000 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_builder.h +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "packed_mixed_tensor.h" - -namespace vespalib::eval::packed_mixed_tensor { - -/** - * A builder for PackedMixedTensor objects - * appropriate for cell type T. - **/ -template -class PackedMixedBuilder : public ValueBuilder -{ -private: - const ValueType & _type; - size_t _subspace_size; - std::vector _cells; - PackedMappingsBuilder _mappings_builder; -public: - PackedMixedBuilder(const ValueType &type, - size_t num_mapped_in, - size_t subspace_size_in, - size_t expected_subspaces) - : _type(type), - _subspace_size(subspace_size_in), - _cells(), - _mappings_builder(num_mapped_in) - { - _cells.reserve(_subspace_size * expected_subspaces); - } - - ~PackedMixedBuilder() override = default; - - ArrayRef add_subspace(const std::vector &addr) override; - std::unique_ptr build(std::unique_ptr> self) override; -}; - -} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.cpp deleted file mode 100644 index 75e6b1e996e..00000000000 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.cpp +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "packed_mixed_factory.h" -#include "packed_mixed_builder.h" - -#include - -namespace vespalib::eval { - -namespace { - -struct CreatePackedMixedBuilder { - template - static std::unique_ptr invoke(const ValueType &type, Args &&...args) - { - assert(check_cell_type(type.cell_type())); - return std::make_unique>(type, std::forward(args)...); - } -}; - -} // namespace - -std::unique_ptr -PackedMixedFactory::create_value_builder_base(const ValueType &type, - size_t num_mapped_in, - size_t subspace_size_in, - size_t expected_subspaces) const -{ - return typify_invoke<1,TypifyCellType,CreatePackedMixedBuilder>(type.cell_type(), - type, num_mapped_in, subspace_size_in, expected_subspaces); -} - -} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.h deleted file mode 100644 index ae90e5e243b..00000000000 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_factory.h +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include -#include - -namespace vespalib::eval { - -/** - * A factory that can generate PackedMixedBuilder - * objects appropriate for the requested CellType. - */ -struct PackedMixedFactory : ValueBuilderFactory { - ~PackedMixedFactory() override {} -protected: - std::unique_ptr create_value_builder_base(const ValueType &type, - size_t num_mapped_in, size_t subspace_size_in, size_t expect_subspaces) const override; -}; - -} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h index 60916b23565..604b1c94aeb 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h @@ -15,7 +15,7 @@ namespace vespalib::eval::packed_mixed_tensor { * An implementation of Value modeling a mixed tensor, * where all the data (cells and sparse address mappings) * can reside in a self-contained, contigous block of memory. - * Currently must be built by a PackedMixedBuilder. + * Currently must be built by a PackedMixedTensorBuilder. * Immutable (all data always const). **/ class PackedMixedTensor : public Value, public Value::Index @@ -33,7 +33,7 @@ private: _mappings(mappings) {} - template friend class PackedMixedBuilder; + template friend class PackedMixedTensorBuilder; public: ~PackedMixedTensor() override; diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.cpp new file mode 100644 index 00000000000..d311d589a27 --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.cpp @@ -0,0 +1,53 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "packed_mixed_tensor_builder.h" + +namespace vespalib::eval::packed_mixed_tensor { + +template +ArrayRef +PackedMixedTensorBuilder::add_subspace(const std::vector &addr) +{ + uint32_t idx = _mappings_builder.add_mapping_for(addr); + size_t offset = idx * _subspace_size; + assert(offset <= _cells.size()); + if (offset == _cells.size()) { + _cells.resize(offset + _subspace_size); + } + return ArrayRef(&_cells[offset], _subspace_size); +} + + +template +std::unique_ptr +PackedMixedTensorBuilder::build(std::unique_ptr>) +{ + size_t self_size = sizeof(PackedMixedTensor); + size_t mappings_size = _mappings_builder.extra_memory(); + // align: + mappings_size += 15ul; + mappings_size &= ~15ul; + size_t cells_size = sizeof(T) * _cells.size(); + size_t total_size = self_size + mappings_size + cells_size; + + char *mem = (char *) operator new(total_size); + char *mappings_mem = mem + self_size; + char *cells_mem = mappings_mem + mappings_size; + + // fill mapping data: + auto mappings = _mappings_builder.target_memory(mappings_mem, cells_mem); + + // copy cells: + memcpy(cells_mem, &_cells[0], cells_size); + ConstArrayRef cells((T *)cells_mem, _cells.size()); + + PackedMixedTensor * built = + new (mem) PackedMixedTensor(_type, TypedCells(cells), mappings); + + return std::unique_ptr(built); +} + +template class PackedMixedTensorBuilder; +template class PackedMixedTensorBuilder; + +} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.h new file mode 100644 index 00000000000..c99762b7e8b --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.h @@ -0,0 +1,40 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "packed_mixed_tensor.h" + +namespace vespalib::eval::packed_mixed_tensor { + +/** + * A builder for PackedMixedTensor objects + * appropriate for cell type T. + **/ +template +class PackedMixedTensorBuilder : public ValueBuilder +{ +private: + const ValueType & _type; + size_t _subspace_size; + std::vector _cells; + PackedMappingsBuilder _mappings_builder; +public: + PackedMixedTensorBuilder(const ValueType &type, + size_t num_mapped_in, + size_t subspace_size_in, + size_t expected_subspaces) + : _type(type), + _subspace_size(subspace_size_in), + _cells(), + _mappings_builder(num_mapped_in) + { + _cells.reserve(_subspace_size * expected_subspaces); + } + + ~PackedMixedTensorBuilder() override = default; + + ArrayRef add_subspace(const std::vector &addr) override; + std::unique_ptr build(std::unique_ptr> self) override; +}; + +} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.cpp new file mode 100644 index 00000000000..132eca9dc8c --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.cpp @@ -0,0 +1,33 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "packed_mixed_tensor_builder_factory.h" +#include "packed_mixed_tensor_builder.h" + +#include + +namespace vespalib::eval { + +namespace { + +struct CreatePackedMixedTensorBuilder { + template + static std::unique_ptr invoke(const ValueType &type, Args &&...args) + { + assert(check_cell_type(type.cell_type())); + return std::make_unique>(type, std::forward(args)...); + } +}; + +} // namespace + +std::unique_ptr +PackedMixedTensorBuilderFactory::create_value_builder_base(const ValueType &type, + size_t num_mapped_in, + size_t subspace_size_in, + size_t expected_subspaces) const +{ + return typify_invoke<1,TypifyCellType,CreatePackedMixedTensorBuilder>(type.cell_type(), + type, num_mapped_in, subspace_size_in, expected_subspaces); +} + +} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h new file mode 100644 index 00000000000..b7f107d39b4 --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h @@ -0,0 +1,20 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include +#include +#include + +namespace vespalib::eval { + +/** + * A factory that can generate PackedMixedTensorBuilder + * objects appropriate for the requested CellType. + */ +struct PackedMixedTensorBuilderFactory : ValueBuilderFactory { + ~PackedMixedTensorBuilderFactory() override {} +protected: + std::unique_ptr create_value_builder_base(const ValueType &type, + size_t num_mapped_in, size_t subspace_size_in, size_t expect_subspaces) const override; +}; + +} // namespace -- cgit v1.2.3 From f6e339cce76c190b9007b608f1f139e60a88ff59 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Thu, 24 Sep 2020 14:14:49 +0000 Subject: make a PackedMixedTensorBuilderFactory singleton also --- eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp | 4 ++-- .../vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.cpp | 3 +++ .../vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h | 5 +++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp b/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp index 8ebed853df6..bc1efdaba1d 100644 --- a/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp +++ b/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp @@ -27,7 +27,7 @@ std::vector layouts = { TEST(PackedMixedTest, packed_mixed_tensors_can_be_converted_from_and_to_tensor_spec) { for (const auto &layout: layouts) { TensorSpec expect = spec(layout, N()); - std::unique_ptr value = value_from_spec(expect, PackedMixedTensorBuilderFactory()); + std::unique_ptr value = value_from_spec(expect, PackedMixedTensorBuilderFactory::get()); TensorSpec actual = spec_from_value(*value); EXPECT_EQ(actual, expect); } @@ -35,7 +35,7 @@ TEST(PackedMixedTest, packed_mixed_tensors_can_be_converted_from_and_to_tensor_s TEST(PackedMixedTest, packed_mixed_tensors_can_be_built_and_inspected) { ValueType type = ValueType::from_spec("tensor(x{},y[2],z{})"); - PackedMixedTensorBuilderFactory factory; + const auto & factory = PackedMixedTensorBuilderFactory::get(); std::unique_ptr> builder = factory.create_value_builder(type); float seq = 0.0; for (vespalib::string x: {"a", "b", "c"}) { diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.cpp index 132eca9dc8c..48eedd86f7f 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.cpp @@ -20,6 +20,9 @@ struct CreatePackedMixedTensorBuilder { } // namespace +PackedMixedTensorBuilderFactory::PackedMixedTensorBuilderFactory() = default; +PackedMixedTensorBuilderFactory PackedMixedTensorBuilderFactory::_factory; + std::unique_ptr PackedMixedTensorBuilderFactory::create_value_builder_base(const ValueType &type, size_t num_mapped_in, diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h index b7f107d39b4..20a581e2b35 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h @@ -11,10 +11,15 @@ namespace vespalib::eval { * objects appropriate for the requested CellType. */ struct PackedMixedTensorBuilderFactory : ValueBuilderFactory { +private: + PackedMixedTensorBuilderFactory(); + static PackedMixedTensorBuilderFactory _factory; ~PackedMixedTensorBuilderFactory() override {} protected: std::unique_ptr create_value_builder_base(const ValueType &type, size_t num_mapped_in, size_t subspace_size_in, size_t expect_subspaces) const override; +public: + static const PackedMixedTensorBuilderFactory &get() { return _factory; } }; } // namespace -- cgit v1.2.3 From 0eacb27ebbf5e03123c0605ab54a289a98037516 Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Thu, 24 Sep 2020 14:50:12 +0000 Subject: Handle innerproduct distance metric when reading attribute header. --- .../src/tests/attribute/attribute_header/attribute_header_test.cpp | 2 ++ searchlib/src/vespa/searchlib/attribute/attribute_header.cpp | 2 ++ 2 files changed, 4 insertions(+) diff --git a/searchlib/src/tests/attribute/attribute_header/attribute_header_test.cpp b/searchlib/src/tests/attribute/attribute_header/attribute_header_test.cpp index 1e89fbe7501..aca9b652b94 100644 --- a/searchlib/src/tests/attribute/attribute_header/attribute_header_test.cpp +++ b/searchlib/src/tests/attribute/attribute_header/attribute_header_test.cpp @@ -72,6 +72,8 @@ TEST(AttributeHeaderTest, can_be_added_to_and_extracted_from_generic_header) verify_roundtrip_serialization(HnswIPO({16, 100, DistanceMetric::Euclidean})); verify_roundtrip_serialization(HnswIPO({16, 100, DistanceMetric::Angular})); verify_roundtrip_serialization(HnswIPO({16, 100, DistanceMetric::GeoDegrees})); + verify_roundtrip_serialization(HnswIPO({16, 100, DistanceMetric::InnerProduct})); + verify_roundtrip_serialization(HnswIPO({16, 100, DistanceMetric::Hamming})); verify_roundtrip_serialization(HnswIPO()); } diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp index 430f2eaa560..cfbca71bd5e 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp @@ -114,6 +114,8 @@ to_distance_metric(const vespalib::string& metric) return DistanceMetric::Angular; } else if (metric == geodegrees) { return DistanceMetric::GeoDegrees; + } else if (metric == innerproduct) { + return DistanceMetric::InnerProduct; } else if (metric == hamming) { return DistanceMetric::Hamming; } else { -- cgit v1.2.3 From ee92e3bd735b67fd839029dabbd6eebd2b56e42d Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Thu, 24 Sep 2020 16:15:22 +0200 Subject: Factor out feed handlers for benchmark feed. --- searchcore/src/apps/vespa-feed-bm/CMakeLists.txt | 2 + .../src/apps/vespa-feed-bm/i_bm_feed_handler.h | 30 +++ .../src/apps/vespa-feed-bm/pending_tracker.h | 57 +++++ .../src/apps/vespa-feed-bm/spi_bm_feed_handler.cpp | 93 ++++++++ .../src/apps/vespa-feed-bm/spi_bm_feed_handler.h | 26 +++ .../storage_api_rpc_bm_feed_handler.cpp | 145 ++++++++++++ .../storage_api_rpc_bm_feed_handler.h | 43 ++++ .../src/apps/vespa-feed-bm/vespa_feed_bm.cpp | 252 ++------------------- 8 files changed, 420 insertions(+), 228 deletions(-) create mode 100644 searchcore/src/apps/vespa-feed-bm/i_bm_feed_handler.h create mode 100644 searchcore/src/apps/vespa-feed-bm/pending_tracker.h create mode 100644 searchcore/src/apps/vespa-feed-bm/spi_bm_feed_handler.cpp create mode 100644 searchcore/src/apps/vespa-feed-bm/spi_bm_feed_handler.h create mode 100644 searchcore/src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.cpp create mode 100644 searchcore/src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.h diff --git a/searchcore/src/apps/vespa-feed-bm/CMakeLists.txt b/searchcore/src/apps/vespa-feed-bm/CMakeLists.txt index 9fa47c77b03..1d031e2ea30 100644 --- a/searchcore/src/apps/vespa-feed-bm/CMakeLists.txt +++ b/searchcore/src/apps/vespa-feed-bm/CMakeLists.txt @@ -2,6 +2,8 @@ vespa_add_executable(searchcore_vespa_feed_bm_app SOURCES vespa_feed_bm.cpp + spi_bm_feed_handler.cpp + storage_api_rpc_bm_feed_handler.cpp OUTPUT_NAME vespa-feed-bm DEPENDS searchcore_server diff --git a/searchcore/src/apps/vespa-feed-bm/i_bm_feed_handler.h b/searchcore/src/apps/vespa-feed-bm/i_bm_feed_handler.h new file mode 100644 index 00000000000..a3341bf14c9 --- /dev/null +++ b/searchcore/src/apps/vespa-feed-bm/i_bm_feed_handler.h @@ -0,0 +1,30 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include + +namespace document { +class Bucket; +class Document; +class DocumentUpdate; +class DocumentId; +} + +namespace feedbm { + +class PendingTracker; + +/* + * Interface class for benchmark feed handler. + */ +class IBmFeedHandler +{ +public: + virtual ~IBmFeedHandler() = default; + virtual void put(const document::Bucket& bucket, std::unique_ptr document, uint64_t timestamp, PendingTracker& tracker) = 0; + virtual void update(const document::Bucket& bucket, std::unique_ptr document_update, uint64_t timestamp, PendingTracker& tracker) = 0; + virtual void remove(const document::Bucket& bucket, const document::DocumentId& document_id, uint64_t timestamp, PendingTracker& tracker) = 0; +}; + +} diff --git a/searchcore/src/apps/vespa-feed-bm/pending_tracker.h b/searchcore/src/apps/vespa-feed-bm/pending_tracker.h new file mode 100644 index 00000000000..3698832068f --- /dev/null +++ b/searchcore/src/apps/vespa-feed-bm/pending_tracker.h @@ -0,0 +1,57 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include +#include + +namespace feedbm { + +/* + * Class to track number of pending operations, used as backpressure during + * benchmark feeding. + */ +class PendingTracker { + uint32_t _pending; + uint32_t _limit; + std::mutex _mutex; + std::condition_variable _cond; + +public: + PendingTracker(uint32_t limit) + : _pending(0u), + _limit(limit), + _mutex(), + _cond() + { + } + + ~PendingTracker() + { + drain(); + } + + void release() { + std::unique_lock guard(_mutex); + --_pending; + if (_pending < _limit) { + _cond.notify_all(); + } + } + void retain() { + std::unique_lock guard(_mutex); + while (_pending >= _limit) { + _cond.wait(guard); + } + ++_pending; + } + + void drain() { + std::unique_lock guard(_mutex); + while (_pending > 0) { + _cond.wait(guard); + } + } +}; + +} diff --git a/searchcore/src/apps/vespa-feed-bm/spi_bm_feed_handler.cpp b/searchcore/src/apps/vespa-feed-bm/spi_bm_feed_handler.cpp new file mode 100644 index 00000000000..d53ece2fc42 --- /dev/null +++ b/searchcore/src/apps/vespa-feed-bm/spi_bm_feed_handler.cpp @@ -0,0 +1,93 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "spi_bm_feed_handler.h" +#include "pending_tracker.h" +#include +#include +#include +#include + +using document::Document; +using document::DocumentId; +using document::DocumentUpdate; +using storage::spi::Bucket; +using storage::spi::PartitionId; +using storage::spi::PersistenceProvider; +using storage::spi::Timestamp; + +namespace feedbm { + +namespace { + +storage::spi::LoadType default_load_type(0, "default"); +storage::spi::Context context(default_load_type, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + +class MyOperationComplete : public storage::spi::OperationComplete +{ + PendingTracker& _tracker; +public: + MyOperationComplete(PendingTracker& tracker); + ~MyOperationComplete(); + void onComplete(std::unique_ptr result) override; + void addResultHandler(const storage::spi::ResultHandler* resultHandler) override; +}; + +MyOperationComplete::MyOperationComplete(PendingTracker& tracker) + : _tracker(tracker) +{ + _tracker.retain(); +} + +MyOperationComplete::~MyOperationComplete() +{ + _tracker.release(); +} + +void +MyOperationComplete::onComplete(std::unique_ptr result) +{ + (void) result; +} + +void +MyOperationComplete::addResultHandler(const storage::spi::ResultHandler * resultHandler) +{ + (void) resultHandler; +} + +} + +SpiBmFeedHandler::SpiBmFeedHandler(PersistenceProvider& provider) + : IBmFeedHandler(), + _provider(provider) +{ +} + +SpiBmFeedHandler::~SpiBmFeedHandler() = default; + +void +SpiBmFeedHandler::put(const document::Bucket& bucket, std::unique_ptr document, uint64_t timestamp, PendingTracker& tracker) +{ + _provider.putAsync(Bucket(bucket, PartitionId(0)), Timestamp(timestamp), std::move(document), context, std::make_unique(tracker)); +} + +void +SpiBmFeedHandler::update(const document::Bucket& bucket, std::unique_ptr document_update, uint64_t timestamp, PendingTracker& tracker) +{ + _provider.updateAsync(Bucket(bucket, PartitionId(0)), Timestamp(timestamp), std::move(document_update), context, std::make_unique(tracker)); +} + +void +SpiBmFeedHandler::remove(const document::Bucket& bucket, const DocumentId& document_id, uint64_t timestamp, PendingTracker& tracker) +{ + _provider.removeAsync(Bucket(bucket, PartitionId(0)), Timestamp(timestamp), document_id, context, std::make_unique(tracker)); + +} + +void +SpiBmFeedHandler::create_bucket(const document::Bucket& bucket) +{ + _provider.createBucket(Bucket(bucket, PartitionId(0)), context); +} + +} diff --git a/searchcore/src/apps/vespa-feed-bm/spi_bm_feed_handler.h b/searchcore/src/apps/vespa-feed-bm/spi_bm_feed_handler.h new file mode 100644 index 00000000000..5b56a4f21dd --- /dev/null +++ b/searchcore/src/apps/vespa-feed-bm/spi_bm_feed_handler.h @@ -0,0 +1,26 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_bm_feed_handler.h" + +namespace storage::spi { struct PersistenceProvider; } + +namespace feedbm { + +/* + * Benchmark feed handler for feed directly to persistence provider + */ +class SpiBmFeedHandler : public IBmFeedHandler +{ + storage::spi::PersistenceProvider& _provider; +public: + SpiBmFeedHandler(storage::spi::PersistenceProvider& provider); + ~SpiBmFeedHandler(); + void put(const document::Bucket& bucket, std::unique_ptr document, uint64_t timestamp, PendingTracker& tracker) override; + void update(const document::Bucket& bucket, std::unique_ptr document_update, uint64_t timestamp, PendingTracker& tracker) override; + void remove(const document::Bucket& bucket, const document::DocumentId& document_id, uint64_t timestamp, PendingTracker& tracker) override; + void create_bucket(const document::Bucket& bucket); +}; + +} diff --git a/searchcore/src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.cpp b/searchcore/src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.cpp new file mode 100644 index 00000000000..e2f3a951b99 --- /dev/null +++ b/searchcore/src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.cpp @@ -0,0 +1,145 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "storage_api_rpc_bm_feed_handler.h" +#include "pending_tracker.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using document::Document; +using document::DocumentId; +using document::DocumentUpdate; +using document::DocumentTypeRepo; +using storage::api::StorageMessageAddress; +using storage::rpc::SharedRpcResources; + +namespace feedbm { + +namespace { + +FRT_RPCRequest *make_set_cluster_state_request() { + storage::lib::ClusterStateBundle bundle(storage::lib::ClusterState("version:2 distributor:1 storage:1")); + storage::rpc::SlimeClusterStateBundleCodec codec; + auto encoded_bundle = codec.encode(bundle); + auto *req = new FRT_RPCRequest(); + auto* params = req->GetParams(); + params->AddInt8(static_cast(encoded_bundle._compression_type)); + params->AddInt32(encoded_bundle._uncompressed_length); + const auto buf_len = encoded_bundle._buffer->getDataLen(); + params->AddData(encoded_bundle._buffer->stealBuffer(), buf_len); + req->SetMethodName("setdistributionstates"); + return req; +} + +void set_cluster_up(SharedRpcResources &shared_rpc_resources, storage::api::StorageMessageAddress &storage_address) { + auto req = make_set_cluster_state_request(); + auto target_resolver = std::make_unique(shared_rpc_resources.slobrok_mirror(), shared_rpc_resources.target_factory()); + auto target = target_resolver->resolve_rpc_target(storage_address); + target->_target->get()->InvokeSync(req, 10.0); // 10 seconds timeout + assert(!req->IsError()); + req->SubRef(); +} + +} + +class StorageApiRpcBmFeedHandler::MyMessageDispatcher : public storage::MessageDispatcher +{ + std::mutex _mutex; + vespalib::hash_map _pending; +public: + MyMessageDispatcher() + : storage::MessageDispatcher(), + _mutex(), + _pending() + { + } + ~MyMessageDispatcher() override; + void dispatch_sync(std::shared_ptr msg) override { + release(msg->getMsgId()); + } + void dispatch_async(std::shared_ptr msg) override { + release(msg->getMsgId()); + } + void retain(uint64_t msg_id, PendingTracker &tracker) { + tracker.retain(); + std::lock_guard lock(_mutex); + _pending.insert(std::make_pair(msg_id, &tracker)); + } + void release(uint64_t msg_id) { + PendingTracker *tracker = nullptr; + { + std::lock_guard lock(_mutex); + auto itr = _pending.find(msg_id); + assert(itr != _pending.end()); + tracker = itr->second; + _pending.erase(itr); + } + tracker->release(); + } +}; + +StorageApiRpcBmFeedHandler::MyMessageDispatcher::~MyMessageDispatcher() +{ + std::lock_guard lock(_mutex); + assert(_pending.empty()); +} + +StorageApiRpcBmFeedHandler::StorageApiRpcBmFeedHandler(SharedRpcResources& shared_rpc_resources_in, std::shared_ptr repo) + : IBmFeedHandler(), + _storage_address(std::make_unique("storage", storage::lib::NodeType::STORAGE, 0)), + _shared_rpc_resources(shared_rpc_resources_in), + _message_dispatcher(std::make_unique()), + _message_codec_provider(std::make_unique(repo, std::make_shared())), + _rpc_client(std::make_unique(*_message_dispatcher, _shared_rpc_resources, *_message_codec_provider, storage::rpc::StorageApiRpcService::Params())) +{ + set_cluster_up(_shared_rpc_resources, *_storage_address); +} + +StorageApiRpcBmFeedHandler::~StorageApiRpcBmFeedHandler() = default; + +void +StorageApiRpcBmFeedHandler::send_rpc(std::shared_ptr cmd, PendingTracker& pending_tracker) +{ + cmd->setSourceIndex(0); + cmd->setAddress(*_storage_address); + _message_dispatcher->retain(cmd->getMsgId(), pending_tracker); + _rpc_client->send_rpc_v1_request(std::move(cmd)); +} + +void +StorageApiRpcBmFeedHandler::put(const document::Bucket& bucket, std::unique_ptr document, uint64_t timestamp, PendingTracker& tracker) +{ + auto cmd = std::make_unique(bucket, std::move(document), timestamp); + send_rpc(std::move(cmd), tracker); +} + +void +StorageApiRpcBmFeedHandler::update(const document::Bucket& bucket, std::unique_ptr document_update, uint64_t timestamp, PendingTracker& tracker) +{ + auto cmd = std::make_unique(bucket, std::move(document_update), timestamp); + send_rpc(std::move(cmd), tracker); +} + +void +StorageApiRpcBmFeedHandler::remove(const document::Bucket& bucket, const DocumentId& document_id, uint64_t timestamp, PendingTracker& tracker) +{ + auto cmd = std::make_unique(bucket, document_id, timestamp); + send_rpc(std::move(cmd), tracker); +} + +} diff --git a/searchcore/src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.h b/searchcore/src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.h new file mode 100644 index 00000000000..0fe92350eb2 --- /dev/null +++ b/searchcore/src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.h @@ -0,0 +1,43 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_bm_feed_handler.h" + +namespace document { class DocumentTypeRepo; } +namespace storage::api { +class StorageMessageAddress; +class StorageCommand; +} + +namespace storage::rpc { +class MessageCodecProvider; +class SharedRpcResources; +class StorageApiRpcService; +} + +namespace feedbm { + +/* + * Benchmark feed handler for feed to service layer using storage api protocol + * over rpc. + */ +class StorageApiRpcBmFeedHandler : public IBmFeedHandler +{ + class MyMessageDispatcher; + std::unique_ptr _storage_address; + storage::rpc::SharedRpcResources& _shared_rpc_resources; + std::unique_ptr _message_dispatcher; + std::unique_ptr _message_codec_provider; + std::unique_ptr _rpc_client; + + void send_rpc(std::shared_ptr cmd, PendingTracker& tracker); +public: + StorageApiRpcBmFeedHandler(storage::rpc::SharedRpcResources& shared_rpc_resources_in, std::shared_ptr repo); + ~StorageApiRpcBmFeedHandler(); + void put(const document::Bucket& bucket, std::unique_ptr document, uint64_t timestamp, PendingTracker& tracker) override; + void update(const document::Bucket& bucket, std::unique_ptr document_update, uint64_t timestamp, PendingTracker& tracker) override; + void remove(const document::Bucket& bucket, const document::DocumentId& document_id, uint64_t timestamp, PendingTracker& tracker) override; +}; + +} diff --git a/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp b/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp index 85b310ccbad..4208e9448ff 100644 --- a/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp +++ b/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp @@ -1,7 +1,9 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "pending_tracker.h" +#include "spi_bm_feed_handler.h" +#include "storage_api_rpc_bm_feed_handler.h" #include - #include #include #include @@ -57,21 +59,7 @@ #include #include #include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include #include @@ -123,29 +111,22 @@ using document::Field; using document::FieldUpdate; using document::IntFieldValue; using document::test::makeBucketSpace; -using documentapi::LoadTypeSet; using search::TuneFileDocumentDB; using search::index::DummyFileHeaderContext; using search::index::Schema; using search::index::SchemaBuilder; using search::transactionlog::TransLogServer; -using storage::rpc::MessageCodecProvider; using storage::rpc::SharedRpcResources; -using storage::rpc::StorageApiRpcService; -using storage::spi::Bucket; -using storage::spi::PartitionId; using storage::spi::PersistenceProvider; -using storage::spi::Priority; -using storage::spi::Timestamp; -using storage::spi::Trace; using vespalib::makeLambdaTask; +using feedbm::IBmFeedHandler; +using feedbm::SpiBmFeedHandler; +using feedbm::StorageApiRpcBmFeedHandler; using DocumentDBMap = std::map>; namespace { -storage::spi::LoadType default_load_type(0, "default"); - vespalib::string base_dir = "testdb"; std::shared_ptr make_document_type() { @@ -206,84 +187,6 @@ struct MyResourceWriteFilter : public IResourceWriteFilter State getAcceptState() const override { return IResourceWriteFilter::State(); } }; -class MyPendingTracker { - uint32_t _pending; - uint32_t _limit; - std::mutex _mutex; - std::condition_variable _cond; - -public: - MyPendingTracker(uint32_t limit) - : _pending(0u), - _limit(limit), - _mutex(), - _cond() - { - } - - ~MyPendingTracker() - { - drain(); - } - - void release() { - std::unique_lock guard(_mutex); - --_pending; - if (_pending < _limit) { - _cond.notify_all(); - } - //LOG(info, "release, pending is now %u", _pending); - } - void retain() { - std::unique_lock guard(_mutex); - while (_pending >= _limit) { - _cond.wait(guard); - } - ++_pending; - //LOG(info, "retain, pending is now %u", _pending); - } - - void drain() { - std::unique_lock guard(_mutex); - while (_pending > 0) { - _cond.wait(guard); - } - } -}; - -class MyOperationComplete : public storage::spi::OperationComplete -{ - MyPendingTracker& _tracker; -public: - MyOperationComplete(MyPendingTracker &tracker); - ~MyOperationComplete(); - void onComplete(std::unique_ptr result) override; - void addResultHandler(const storage::spi::ResultHandler* resultHandler) override; -}; - -MyOperationComplete::MyOperationComplete(MyPendingTracker& tracker) - : _tracker(tracker) -{ - _tracker.retain(); -} - -MyOperationComplete::~MyOperationComplete() -{ - _tracker.release(); -} - -void -MyOperationComplete::onComplete(std::unique_ptr result) -{ - (void) result; -} - -void -MyOperationComplete::addResultHandler(const storage::spi::ResultHandler * resultHandler) -{ - (void) resultHandler; -} - class BMRange { uint32_t _start; @@ -525,71 +428,6 @@ struct MyRpcClientConfig { MyRpcClientConfig::~MyRpcClientConfig() = default; -class MyMessageDispatcher : public storage::MessageDispatcher -{ - std::mutex _mutex; - vespalib::hash_map _pending; -public: - MyMessageDispatcher() - : storage::MessageDispatcher(), - _mutex(), - _pending() - { - } - ~MyMessageDispatcher() override; - void dispatch_sync(std::shared_ptr msg) override { - release(msg->getMsgId()); - } - void dispatch_async(std::shared_ptr msg) override { - release(msg->getMsgId()); - } - void retain(uint64_t msg_id, MyPendingTracker &tracker) { - tracker.retain(); - std::lock_guard lock(_mutex); - _pending.insert(std::make_pair(msg_id, &tracker)); - } - void release(uint64_t msg_id) { - MyPendingTracker *tracker = nullptr; - { - std::lock_guard lock(_mutex); - auto itr = _pending.find(msg_id); - assert(itr != _pending.end()); - tracker = itr->second; - _pending.erase(itr); - } - tracker->release(); - } -}; - -MyMessageDispatcher::~MyMessageDispatcher() -{ - std::lock_guard lock(_mutex); - assert(_pending.empty()); -} - -FRT_RPCRequest *make_set_cluster_state_request() { - storage::lib::ClusterStateBundle bundle(storage::lib::ClusterState("version:2 distributor:1 storage:1")); - storage::rpc::SlimeClusterStateBundleCodec codec; - auto encoded_bundle = codec.encode(bundle); - auto *req = new FRT_RPCRequest(); - auto* params = req->GetParams(); - params->AddInt8(static_cast(encoded_bundle._compression_type)); - params->AddInt32(encoded_bundle._uncompressed_length); - const auto buf_len = encoded_bundle._buffer->getDataLen(); - params->AddData(encoded_bundle._buffer->stealBuffer(), buf_len); - req->SetMethodName("setdistributionstates"); - return req; -} - -void set_cluster_up(SharedRpcResources &shared_rpc_resources, storage::api::StorageMessageAddress &storage_address) { - auto req = make_set_cluster_state_request(); - auto target_resolver = std::make_unique(shared_rpc_resources.slobrok_mirror(), shared_rpc_resources.target_factory()); - auto target = target_resolver->resolve_rpc_target(storage_address); - target->_target->get()->InvokeSync(req, 10.0); // 10 seconds timeout - assert(!req->IsError()); - req->SubRef(); -} - } struct PersistenceProviderFixture { @@ -618,33 +456,28 @@ struct PersistenceProviderFixture { MyPersistenceEngineOwner _persistence_owner; MyResourceWriteFilter _write_filter; std::shared_ptr _persistence_engine; - storage::spi::Context _context; uint32_t _bucket_bits; MyStorageConfig _service_layer_config; MyRpcClientConfig _rpc_client_config; ConfigSet _config_set; std::shared_ptr _config_context; - storage::api::StorageMessageAddress _storage_address; + std::unique_ptr _feed_handler; std::unique_ptr _slobrok; std::unique_ptr _service_layer; - std::unique_ptr _message_codec_provider; std::unique_ptr _rpc_client_shared_rpc_resources; - std::unique_ptr _rpc_message_dispatcher; - std::unique_ptr _rpc_client; PersistenceProviderFixture(const BMParams& params); ~PersistenceProviderFixture(); void create_document_db(); uint32_t num_buckets() const { return (1u << _bucket_bits); } BucketId make_bucket_id(uint32_t i) const { return BucketId(_bucket_bits, i & (num_buckets() - 1)); } - Bucket make_bucket(uint32_t i) const { return Bucket(document::Bucket(_bucket_space, BucketId(_bucket_bits, i & (num_buckets() - 1))), PartitionId(0)); } + document::Bucket make_bucket(uint32_t i) const { return document::Bucket(_bucket_space, BucketId(_bucket_bits, i & (num_buckets() - 1))); } DocumentId make_document_id(uint32_t i) const; std::unique_ptr make_document(uint32_t i) const; std::unique_ptr make_document_update(uint32_t i) const; void create_buckets(); void start_service_layer(); void shutdown_service_layer(); - void send_rpc(std::shared_ptr cmd, MyPendingTracker& pending_tracker); }; PersistenceProviderFixture::PersistenceProviderFixture(const BMParams& params) @@ -673,19 +506,15 @@ PersistenceProviderFixture::PersistenceProviderFixture(const BMParams& params) _persistence_owner(), _write_filter(), _persistence_engine(), - _context(default_load_type, Priority(0), Trace::TraceLevel(0)), _bucket_bits(16), _service_layer_config("bm-servicelayer", *_document_types, _slobrok_port, _status_port, params.get_rpc_network_threads()), _rpc_client_config("bm-rpc-client", _slobrok_port), _config_set(), _config_context(std::make_shared(_config_set)), - _storage_address("storage", storage::lib::NodeType::STORAGE, 0), + _feed_handler(), _slobrok(), _service_layer(), - _message_codec_provider(), - _rpc_client_shared_rpc_resources(), - _rpc_message_dispatcher(), - _rpc_client() + _rpc_client_shared_rpc_resources() { create_document_db(); _persistence_engine = std::make_unique(_persistence_owner, _write_filter, -1, false); @@ -693,6 +522,7 @@ PersistenceProviderFixture::PersistenceProviderFixture(const BMParams& params) _persistence_engine->putHandler(_persistence_engine->getWLock(), _bucket_space, _doc_type_name, proxy); _service_layer_config.add_builders(_config_set); _rpc_client_config.add_builders(_config_set); + _feed_handler = std::make_unique(*_persistence_engine); } PersistenceProviderFixture::~PersistenceProviderFixture() @@ -778,9 +608,9 @@ PersistenceProviderFixture::make_document_update(uint32_t i) const void PersistenceProviderFixture::create_buckets() { - auto &provider = *_persistence_engine; + SpiBmFeedHandler feed_handler(*_persistence_engine); for (unsigned int i = 0; i < num_buckets(); ++i) { - provider.createBucket(make_bucket(i), _context); + feed_handler.create_bucket(make_bucket(i)); } } @@ -796,21 +626,17 @@ PersistenceProviderFixture::start_service_layer() _service_layer->setupConfig(100ms); _service_layer->createNode(); _service_layer->getNode().waitUntilInitialized(); - _message_codec_provider = std::make_unique(_repo, std::make_shared()); LOG(info, "start rpc client shared resources"); config::ConfigUri client_config_uri("bm-rpc-client", _config_context); _rpc_client_shared_rpc_resources = std::make_unique(client_config_uri, _rpc_client_port, 100); _rpc_client_shared_rpc_resources->start_server_and_register_slobrok("bm-rpc-client"); - _rpc_message_dispatcher = std::make_unique(); - _rpc_client = std::make_unique(*_rpc_message_dispatcher, *_rpc_client_shared_rpc_resources, *_message_codec_provider, StorageApiRpcService::Params()); - set_cluster_up(*_rpc_client_shared_rpc_resources, _storage_address); + _feed_handler = std::make_unique(*_rpc_client_shared_rpc_resources, _repo); } void PersistenceProviderFixture::shutdown_service_layer() { - _rpc_client.reset(); - _rpc_message_dispatcher.reset(); + _feed_handler.reset(); if (_rpc_client_shared_rpc_resources) { LOG(info, "stop rpc client shared resources"); _rpc_client_shared_rpc_resources->shutdown(); @@ -827,15 +653,6 @@ PersistenceProviderFixture::shutdown_service_layer() } } -void -PersistenceProviderFixture::send_rpc(std::shared_ptr cmd, MyPendingTracker& pending_tracker) -{ - cmd->setSourceIndex(0); - cmd->setAddress(_storage_address); - _rpc_message_dispatcher->retain(cmd->getMsgId(), pending_tracker); - _rpc_client->send_rpc_v1_request(std::move(cmd)); -} - vespalib::nbostream make_put_feed(PersistenceProviderFixture &f, BMRange range) { @@ -872,23 +689,16 @@ void put_async_task(PersistenceProviderFixture &f, BMRange range, const vespalib::nbostream &serialized_feed, int64_t time_bias) { LOG(debug, "put_async_task([%u..%u))", range.get_start(), range.get_end()); - MyPendingTracker pending_tracker(100); - auto &provider = *f._persistence_engine; - auto &context = f._context; + feedbm::PendingTracker pending_tracker(100); auto &repo = *f._repo; vespalib::nbostream is(serialized_feed.data(), serialized_feed.size()); BucketId bucket_id; auto bucket_space = f._bucket_space; for (unsigned int i = range.get_start(); i < range.get_end(); ++i) { is >> bucket_id; - Bucket bucket(document::Bucket(bucket_space, bucket_id), PartitionId(0)); + document::Bucket bucket(bucket_space, bucket_id); auto document = std::make_unique(repo, is); - if (f._rpc_client) { - auto cmd = std::make_unique(bucket.getBucket(), std::move(document), time_bias + i); - f.send_rpc(std::move(cmd), pending_tracker); - } else { - provider.putAsync(bucket, Timestamp(time_bias + i), std::move(document), context, std::make_unique(pending_tracker)); - } + f._feed_handler->put(bucket, std::move(document), time_bias + i, pending_tracker); } assert(is.empty()); pending_tracker.drain(); @@ -928,23 +738,16 @@ void update_async_task(PersistenceProviderFixture &f, BMRange range, const vespalib::nbostream &serialized_feed, int64_t time_bias) { LOG(debug, "update_async_task([%u..%u))", range.get_start(), range.get_end()); - MyPendingTracker pending_tracker(100); - auto &provider = *f._persistence_engine; - auto &context = f._context; + feedbm::PendingTracker pending_tracker(100); auto &repo = *f._repo; vespalib::nbostream is(serialized_feed.data(), serialized_feed.size()); BucketId bucket_id; auto bucket_space = f._bucket_space; for (unsigned int i = range.get_start(); i < range.get_end(); ++i) { is >> bucket_id; - Bucket bucket(document::Bucket(bucket_space, bucket_id), PartitionId(0)); + document::Bucket bucket(bucket_space, bucket_id); auto document_update = DocumentUpdate::createHEAD(repo, is); - if (f._rpc_client) { - auto cmd = std::make_unique(bucket.getBucket(), std::move(document_update), time_bias + i); - f.send_rpc(std::move(cmd), pending_tracker); - } else { - provider.updateAsync(bucket, Timestamp(time_bias + i), std::move(document_update), context, std::make_unique(pending_tracker)); - } + f._feed_handler->update(bucket, std::move(document_update), time_bias + i, pending_tracker); } assert(is.empty()); pending_tracker.drain(); @@ -985,22 +788,15 @@ void remove_async_task(PersistenceProviderFixture &f, BMRange range, const vespalib::nbostream &serialized_feed, int64_t time_bias) { LOG(debug, "remove_async_task([%u..%u))", range.get_start(), range.get_end()); - MyPendingTracker pending_tracker(100); - auto &provider = *f._persistence_engine; - auto &context = f._context; + feedbm::PendingTracker pending_tracker(100); vespalib::nbostream is(serialized_feed.data(), serialized_feed.size()); BucketId bucket_id; auto bucket_space = f._bucket_space; for (unsigned int i = range.get_start(); i < range.get_end(); ++i) { is >> bucket_id; - Bucket bucket(document::Bucket(bucket_space, bucket_id), PartitionId(0)); + document::Bucket bucket(bucket_space, bucket_id); DocumentId document_id(is); - if (f._rpc_client) { - auto cmd = std::make_unique(bucket.getBucket(), document_id, time_bias + i); - f.send_rpc(std::move(cmd), pending_tracker); - } else { - provider.removeAsync(bucket, Timestamp(time_bias + i), document_id, context, std::make_unique(pending_tracker)); - } + f._feed_handler->remove(bucket, document_id, time_bias + i, pending_tracker); } assert(is.empty()); pending_tracker.drain(); -- cgit v1.2.3 From 731ec8fb898164b066b64d26019a74aa13b08710 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Thu, 24 Sep 2020 17:05:54 +0200 Subject: Also show the longest-living historical locks, with stack trace --- .../hosted/provision/restapi/LocksResponse.java | 32 ++++++++++------- .../com/yahoo/vespa/curator/stats/LockInfo.java | 40 ++++++++++++++++++++-- .../yahoo/vespa/curator/stats/ThreadLockInfo.java | 30 +++++++++++----- 3 files changed, 79 insertions(+), 23 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java index bc4401ee03a..502bd6f3cd7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java @@ -55,19 +55,13 @@ public class LocksResponse extends HttpResponse { threadLockInfoCursor.setString("lock-path", threadLockInfo.getLockPath()); List lockInfos = threadLockInfo.getLockInfos(); - if (!lockInfos.isEmpty()) { - Cursor lockInfosCursor = threadLockInfoCursor.setArray("locks"); - lockInfos.forEach(lockInfo -> { - Cursor lockInfoCursor = lockInfosCursor.addObject(); - lockInfoCursor.setString("invoke-acquire-time", toString(lockInfo.getTimeAcquiredWasInvoked())); - lockInfoCursor.setLong("reentrancy-hold-count-on-acquire", lockInfo.getThreadHoldCountOnAcquire()); - lockInfoCursor.setString("acquire-timeout", lockInfo.getAcquireTimeout().toString()); - lockInfo.getTimeLockWasAcquired().ifPresent(instant -> lockInfoCursor.setString("lock-acquired-time", toString(instant))); - lockInfoCursor.setString("lock-state", lockInfo.getLockState().name()); - lockInfo.getTimeTerminalStateWasReached().ifPresent(instant -> lockInfoCursor.setString("terminal-state-time", toString(instant))); - }); - } + Cursor lockInfosCursor = threadLockInfoCursor.setArray("locks"); + lockInfos.forEach(lockInfo -> setLockInfo(lockInfosCursor.addObject(), lockInfo, false)); }); + + List slowLockInfos = ThreadLockInfo.getSlowLockInfos(); + Cursor slowLocksCursor = root.setArray("slow-locks"); + slowLockInfos.forEach(lockInfo -> setLockInfo(slowLocksCursor.addObject(), lockInfo, true)); } @Override @@ -80,6 +74,20 @@ public class LocksResponse extends HttpResponse { return "application/json"; } + private void setLockInfo(Cursor lockInfoCursor, LockInfo lockInfo, boolean includeThreadInfo) { + if (includeThreadInfo) { + lockInfoCursor.setString("thread-name", lockInfo.getThreadName()); + lockInfoCursor.setString("lock-path", lockInfo.getLockPath()); + } + lockInfoCursor.setString("invoke-acquire-time", toString(lockInfo.getTimeAcquiredWasInvoked())); + lockInfoCursor.setLong("reentrancy-hold-count-on-acquire", lockInfo.getThreadHoldCountOnAcquire()); + lockInfoCursor.setString("acquire-timeout", lockInfo.getAcquireTimeout().toString()); + lockInfo.getTimeLockWasAcquired().ifPresent(instant -> lockInfoCursor.setString("lock-acquired-time", toString(instant))); + lockInfoCursor.setString("lock-state", lockInfo.getLockState().name()); + lockInfo.getTimeTerminalStateWasReached().ifPresent(instant -> lockInfoCursor.setString("terminal-state-time", toString(instant))); + lockInfo.getStackTrace().ifPresent(stackTrace -> lockInfoCursor.setString("stack-trace", stackTrace)); + } + private static String toString(Instant time) { return Instant.ofEpochMilli(time.toEpochMilli()).toString(); } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java index b192210c3de..6e58c1ea81b 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.curator.stats; import java.time.Duration; import java.time.Instant; import java.util.Optional; +import java.util.stream.Stream; /** * Information about a lock. @@ -14,15 +15,18 @@ import java.util.Optional; */ public class LockInfo { + private final String threadName; + private final String lockPath; private final int threadHoldCountOnAcquire; private final Instant acquireInstant; private final Duration timeout; private volatile Optional lockAcquiredInstant = Optional.empty(); private volatile Optional terminalStateInstant = Optional.empty(); + private volatile Optional stackTrace = Optional.empty(); - public static LockInfo invokingAcquire(int holdCount, Duration timeout) { - return new LockInfo(holdCount, timeout); + public static LockInfo invokingAcquire(String threadName, String lockPath, int holdCount, Duration timeout) { + return new LockInfo(threadName, lockPath, holdCount, timeout); } public enum LockState { @@ -37,18 +41,28 @@ public class LockInfo { private volatile LockState lockState = LockState.ACQUIRING; - private LockInfo(int threadHoldCountOnAcquire, Duration timeout) { + private LockInfo(String threadName, String lockPath, int threadHoldCountOnAcquire, Duration timeout) { + this.threadName = threadName; + this.lockPath = lockPath; this.threadHoldCountOnAcquire = threadHoldCountOnAcquire; this.acquireInstant = Instant.now(); this.timeout = timeout; } + public String getThreadName() { return threadName; } + public String getLockPath() { return lockPath; } public int getThreadHoldCountOnAcquire() { return threadHoldCountOnAcquire; } public Instant getTimeAcquiredWasInvoked() { return acquireInstant; } public Duration getAcquireTimeout() { return timeout; } public LockState getLockState() { return lockState; } public Optional getTimeLockWasAcquired() { return lockAcquiredInstant; } public Optional getTimeTerminalStateWasReached() { return terminalStateInstant; } + public Optional getStackTrace() { return stackTrace; } + + /** Get time from just before trying to acquire lock to the time the terminal state was reached, or ZERO. */ + public Duration getTotalTime() { + return terminalStateInstant.map(instant -> Duration.between(acquireInstant, instant)).orElse(Duration.ZERO); + } void timedOut() { setTerminalState(LockState.TIMED_OUT); } void failedToAcquireReentrantLock() { setTerminalState(LockState.FAILED_TO_REENTER); } @@ -63,4 +77,24 @@ public class LockInfo { lockState = terminalState; terminalStateInstant = Optional.of(Instant.now()); } + + /** Fill in the stack trace starting at the caller's stack frame. */ + void fillStackTrace() { + final int elementsToIgnore = 1; + + var stackTrace = new StringBuilder(); + + StackTraceElement[] elements = Thread.currentThread().getStackTrace(); + for (int i = elementsToIgnore; i < elements.length; ++i) { + Stream.of(elements).forEach(element -> + stackTrace.append(element.getClassName()) + .append('(') + .append(element.getFileName()) + .append(':') + .append(element.getLineNumber()) + .append(")\n")); + } + + this.stackTrace = Optional.of(stackTrace.toString()); + } } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java index bba39e6dc49..5e77ec76290 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java @@ -4,11 +4,12 @@ package com.yahoo.vespa.curator.stats; import com.yahoo.vespa.curator.Lock; import java.time.Duration; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.PriorityQueue; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; @@ -26,8 +27,10 @@ public class ThreadLockInfo { private static final ConcurrentHashMap locks = new ConcurrentHashMap<>(); - private static final int MAX_COMPLETED_LOCK_INFOS_SIZE = 10; - private static final ConcurrentLinkedDeque completedLockInfos = new ConcurrentLinkedDeque<>(); + private static final int MAX_COMPLETED_LOCK_INFOS_SIZE = 5; + /** Would have used a thread-safe priority queue. */ + private static final Object completedLockInfosMonitor = new Object(); + private static final PriorityQueue completedLockInfos = new PriorityQueue<>(Comparator.comparing(LockInfo::getTotalTime)); private static final ConcurrentHashMap countersByLockPath = new ConcurrentHashMap<>(); @@ -43,6 +46,12 @@ public class ThreadLockInfo { public static List getThreadLockInfos() { return List.copyOf(locks.values()); } + public static List getSlowLockInfos() { + synchronized (completedLockInfosMonitor) { + return List.copyOf(completedLockInfos); + } + } + /** Returns the per-thread singleton ThreadLockInfo. */ public static ThreadLockInfo getCurrentThreadLockInfo(String lockPath, ReentrantLock lock) { return locks.computeIfAbsent( @@ -68,7 +77,7 @@ public class ThreadLockInfo { public void invokingAcquire(Duration timeout) { lockCountersForPath.invokeAcquireCount.incrementAndGet(); lockCountersForPath.inCriticalRegionCount.incrementAndGet(); - lockInfos.add(LockInfo.invokingAcquire(lock.getHoldCount(), timeout)); + lockInfos.add(LockInfo.invokingAcquire(getThreadName(), lockPath, lock.getHoldCount(), timeout)); } /** Mutable method (see class doc) */ @@ -113,10 +122,15 @@ public class ThreadLockInfo { LockInfo lockInfo = lockInfos.poll(); completeLockInfo.accept(lockInfo); - if (completedLockInfos.size() >= MAX_COMPLETED_LOCK_INFOS_SIZE) { - // This is thread-safe, as no-one but currentThread mutates completedLockInfos - completedLockInfos.removeLast(); + synchronized (completedLockInfosMonitor) { + if (completedLockInfos.size() < MAX_COMPLETED_LOCK_INFOS_SIZE) { + lockInfo.fillStackTrace(); + completedLockInfos.add(lockInfo); + } else if (lockInfo.getTotalTime().compareTo(completedLockInfos.peek().getTotalTime()) > 0) { + completedLockInfos.poll(); + lockInfo.fillStackTrace(); + completedLockInfos.add(lockInfo); + } } - completedLockInfos.addFirst(lockInfo); } } -- cgit v1.2.3 From 18cf8e6154be4d6d1fb0399ba9fec2d26408740d Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Thu, 24 Sep 2020 17:25:11 +0200 Subject: Make stacktraces for active locks during request handling --- .../hosted/provision/restapi/LocksResponse.java | 24 ++++++---- .../com/yahoo/vespa/curator/stats/LockInfo.java | 54 ++++++++++++---------- .../yahoo/vespa/curator/stats/ThreadLockInfo.java | 2 +- 3 files changed, 46 insertions(+), 34 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java index 502bd6f3cd7..24321cd8a20 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java @@ -49,15 +49,23 @@ public class LocksResponse extends HttpResponse { List threadLockInfos = ThreadLockInfo.getThreadLockInfos(); Cursor threadsCursor = root.setArray("threads"); - threadLockInfos.forEach(threadLockInfo -> { - Cursor threadLockInfoCursor = threadsCursor.addObject(); - threadLockInfoCursor.setString("thread-name", threadLockInfo.getThreadName()); - threadLockInfoCursor.setString("lock-path", threadLockInfo.getLockPath()); - + int numberOfStackTraces = 0; + for (var threadLockInfo : threadLockInfos) { List lockInfos = threadLockInfo.getLockInfos(); - Cursor lockInfosCursor = threadLockInfoCursor.setArray("locks"); - lockInfos.forEach(lockInfo -> setLockInfo(lockInfosCursor.addObject(), lockInfo, false)); - }); + if (!lockInfos.isEmpty()) { + Cursor threadLockInfoCursor = threadsCursor.addObject(); + threadLockInfoCursor.setString("thread-name", threadLockInfo.getThreadName()); + threadLockInfoCursor.setString("lock-path", threadLockInfo.getLockPath()); + + Cursor lockInfosCursor = threadLockInfoCursor.setArray("active-locks"); + for (var lockInfo : lockInfos) { + if (numberOfStackTraces++ < 10) { // Expensive to generate stack traces? + lockInfo.fillStackTrace(); + } + setLockInfo(lockInfosCursor.addObject(), lockInfo, false); + } + } + } List slowLockInfos = ThreadLockInfo.getSlowLockInfos(); Cursor slowLocksCursor = root.setArray("slow-locks"); diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java index 6e58c1ea81b..c9f3ec4b757 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java @@ -9,13 +9,14 @@ import java.util.stream.Stream; /** * Information about a lock. * - *

Should be mutated by a single thread. Other threads may see an inconsistent state of this instance.

+ *

Should be mutated by a single thread, except {@link #fillStackTrace()} which can be + * invoked by any threads. Other threads may see an inconsistent state of this instance.

* * @author hakon */ public class LockInfo { - private final String threadName; + private final Thread thread; private final String lockPath; private final int threadHoldCountOnAcquire; private final Instant acquireInstant; @@ -25,8 +26,8 @@ public class LockInfo { private volatile Optional terminalStateInstant = Optional.empty(); private volatile Optional stackTrace = Optional.empty(); - public static LockInfo invokingAcquire(String threadName, String lockPath, int holdCount, Duration timeout) { - return new LockInfo(threadName, lockPath, holdCount, timeout); + public static LockInfo invokingAcquire(Thread thread, String lockPath, int holdCount, Duration timeout) { + return new LockInfo(thread, lockPath, holdCount, timeout); } public enum LockState { @@ -41,15 +42,15 @@ public class LockInfo { private volatile LockState lockState = LockState.ACQUIRING; - private LockInfo(String threadName, String lockPath, int threadHoldCountOnAcquire, Duration timeout) { - this.threadName = threadName; + private LockInfo(Thread thread, String lockPath, int threadHoldCountOnAcquire, Duration timeout) { + this.thread = thread; this.lockPath = lockPath; this.threadHoldCountOnAcquire = threadHoldCountOnAcquire; this.acquireInstant = Instant.now(); this.timeout = timeout; } - public String getThreadName() { return threadName; } + public String getThreadName() { return thread.getName(); } public String getLockPath() { return lockPath; } public int getThreadHoldCountOnAcquire() { return threadHoldCountOnAcquire; } public Instant getTimeAcquiredWasInvoked() { return acquireInstant; } @@ -64,28 +65,17 @@ public class LockInfo { return terminalStateInstant.map(instant -> Duration.between(acquireInstant, instant)).orElse(Duration.ZERO); } - void timedOut() { setTerminalState(LockState.TIMED_OUT); } - void failedToAcquireReentrantLock() { setTerminalState(LockState.FAILED_TO_REENTER); } - void released() { setTerminalState(LockState.RELEASED); } - - void lockAcquired() { - lockState = LockState.ACQUIRED; - lockAcquiredInstant = Optional.of(Instant.now()); - } - - void setTerminalState(LockState terminalState) { - lockState = terminalState; - terminalStateInstant = Optional.of(Instant.now()); - } - /** Fill in the stack trace starting at the caller's stack frame. */ - void fillStackTrace() { - final int elementsToIgnore = 1; + public void fillStackTrace() { + // This method is public. If invoked concurrently, the this.stackTrace may be updated twice, + // which is fine. + + if (this.stackTrace.isPresent()) return; var stackTrace = new StringBuilder(); - StackTraceElement[] elements = Thread.currentThread().getStackTrace(); - for (int i = elementsToIgnore; i < elements.length; ++i) { + StackTraceElement[] elements = thread.getStackTrace(); + for (int i = 0; i < elements.length && i < 20; ++i) { Stream.of(elements).forEach(element -> stackTrace.append(element.getClassName()) .append('(') @@ -97,4 +87,18 @@ public class LockInfo { this.stackTrace = Optional.of(stackTrace.toString()); } + + void timedOut() { setTerminalState(LockState.TIMED_OUT); } + void failedToAcquireReentrantLock() { setTerminalState(LockState.FAILED_TO_REENTER); } + void released() { setTerminalState(LockState.RELEASED); } + + void lockAcquired() { + lockState = LockState.ACQUIRED; + lockAcquiredInstant = Optional.of(Instant.now()); + } + + void setTerminalState(LockState terminalState) { + lockState = terminalState; + terminalStateInstant = Optional.of(Instant.now()); + } } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java index 5e77ec76290..2e4672841bf 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java @@ -77,7 +77,7 @@ public class ThreadLockInfo { public void invokingAcquire(Duration timeout) { lockCountersForPath.invokeAcquireCount.incrementAndGet(); lockCountersForPath.inCriticalRegionCount.incrementAndGet(); - lockInfos.add(LockInfo.invokingAcquire(getThreadName(), lockPath, lock.getHoldCount(), timeout)); + lockInfos.add(LockInfo.invokingAcquire(thread, lockPath, lock.getHoldCount(), timeout)); } /** Mutable method (see class doc) */ -- cgit v1.2.3 From 3c04df69105ea3b48b405e3943b082f5714dbac1 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Thu, 24 Sep 2020 18:42:51 +0200 Subject: Revert "Revert "Revert "Balder/group operations to tls and commit in batches""" --- .../documentmetastore/lid_reuse_delayer_config.cpp | 2 - .../documentmetastore/lid_reuse_delayer_config.h | 2 - .../proton/documentmetastore/lidreusedelayer.cpp | 1 - .../proton/documentmetastore/lidreusedelayer.h | 4 +- .../proton/server/document_db_maintenance_config.h | 1 - .../vespa/searchcore/proton/server/feedhandler.cpp | 32 +---------- .../vespa/searchcore/proton/server/feedhandler.h | 5 -- .../searchcore/proton/server/i_operation_storer.h | 4 -- .../proton/server/lid_space_compaction_job.cpp | 2 +- .../proton/server/lid_space_compaction_job.h | 6 +- .../proton/server/maintenance_jobs_injector.cpp | 2 +- .../searchcore/proton/server/storeonlyfeedview.cpp | 26 ++++----- .../searchcore/proton/server/storeonlyfeedview.h | 2 +- .../src/vespa/searchlib/config/translogserver.def | 2 +- .../src/vespa/searchlib/transactionlog/domain.cpp | 66 ++-------------------- .../src/vespa/searchlib/transactionlog/domain.h | 5 -- .../vespa/searchlib/transactionlog/domainpart.cpp | 12 +--- .../searchlib/transactionlog/translogserver.cpp | 7 +-- 18 files changed, 32 insertions(+), 149 deletions(-) diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp index 1f979d1566c..b04bac5ef26 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp @@ -7,7 +7,6 @@ namespace proton::documentmetastore { LidReuseDelayerConfig::LidReuseDelayerConfig(const DocumentDBConfig & configSnapshot) : _visibilityDelay(configSnapshot.getMaintenanceConfigSP()->getVisibilityDelay()), - _allowEarlyAck(configSnapshot.getMaintenanceConfigSP()->allowEarlyAck()), _hasIndexedOrAttributeFields(configSnapshot.getSchemaSP()->getNumIndexFields() > 0 || configSnapshot.getSchemaSP()->getNumAttributeFields() > 0) { @@ -19,7 +18,6 @@ LidReuseDelayerConfig::LidReuseDelayerConfig() LidReuseDelayerConfig::LidReuseDelayerConfig(vespalib::duration visibilityDelay, bool hasIndexedOrAttributeFields_in) : _visibilityDelay(visibilityDelay), - _allowEarlyAck(visibilityDelay > 1ms), _hasIndexedOrAttributeFields(hasIndexedOrAttributeFields_in) { } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h index c81a2ff399f..82dab433a22 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h @@ -15,7 +15,6 @@ class LidReuseDelayerConfig { private: vespalib::duration _visibilityDelay; - bool _allowEarlyAck; bool _hasIndexedOrAttributeFields; public: LidReuseDelayerConfig(); @@ -23,7 +22,6 @@ public: explicit LidReuseDelayerConfig(const DocumentDBConfig &configSnapshot); vespalib::duration visibilityDelay() const { return _visibilityDelay; } bool hasIndexedOrAttributeFields() const { return _hasIndexedOrAttributeFields; } - bool allowEarlyAck() const { return _allowEarlyAck; } }; } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp index ed2202c830b..03dfd83a132 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp @@ -16,7 +16,6 @@ LidReuseDelayer::LidReuseDelayer(IThreadingService &writeService, IStore &docume : _writeService(writeService), _documentMetaStore(documentMetaStore), _immediateCommit(config.visibilityDelay() == vespalib::duration::zero()), - _allowEarlyAck(config.allowEarlyAck()), _config(config), _pendingLids() { diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h index ba407ab57f8..5f1de878b4a 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h @@ -27,7 +27,6 @@ class LidReuseDelayer searchcorespi::index::IThreadingService &_writeService; IStore &_documentMetaStore; const bool _immediateCommit; - const bool _allowEarlyAck; LidReuseDelayerConfig _config; std::vector _pendingLids; // lids waiting for commit @@ -39,8 +38,7 @@ public: bool delayReuse(const std::vector &lids); std::vector getReuseLids(); - bool needImmediateCommit() const { return _immediateCommit; } - bool allowEarlyAck() const { return _allowEarlyAck; } + bool getImmediateCommit() const { return _immediateCommit; } const LidReuseDelayerConfig & getConfig() const { return _config; } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h index 5ed0ad7492c..4c0485baec6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h @@ -135,7 +135,6 @@ public: } vespalib::duration getVisibilityDelay() const { return _visibilityDelay; } bool hasVisibilityDelay() const { return _visibilityDelay > vespalib::duration::zero(); } - bool allowEarlyAck() const { return _visibilityDelay > 1ms; } const DocumentDBLidSpaceCompactionConfig &getLidSpaceCompactionConfig() const { return _lidSpaceCompaction; } diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 209a35ce4a2..37dfddf0c2c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -402,8 +402,6 @@ FeedHandler::FeedHandler(IThreadingService &writeService, _tlsReplayProgress(), _serialNum(0), _prunedSerialNum(0), - _numPendingCommit(0), - _numCommitsCompleted(0), _delayedPrune(false), _feedLock(), _feedState(make_shared(getDocTypeName())), @@ -496,40 +494,12 @@ FeedHandler::getTransactionLogReplayDone() const { return _tlsMgr.getReplayDone(); } -void -FeedHandler::onCommitDone(uint64_t numPendingAtStart) { - assert(numPendingAtStart <= _numPendingCommit); - _numPendingCommit -= numPendingAtStart; - if (_numPendingCommit > 0) { - enqueCommitTask(); - } -} - -void FeedHandler::enqueCommitTask() { - _writeService.master().execute(makeLambdaTask([this]() { initiateCommit(); })); -} - -void -FeedHandler::initiateCommit() { - auto commitResult = _tlsWriter->startCommit(std::make_shared( - _writeService.master(), - makeLambdaTask([this, numPendingAtStart=_numPendingCommit]() { - onCommitDone(numPendingAtStart); - }))); - if (_activeFeedView) { - _activeFeedView->forceCommit(_serialNum, std::make_shared>(std::move(commitResult))); - } -} - void FeedHandler::appendOperation(const FeedOperation &op, TlsWriter::DoneCallback onDone) { if (!op.getSerialNum()) { const_cast(op).setSerialNum(incSerialNum()); } _tlsWriter->appendOperation(op, std::move(onDone)); - if (++_numPendingCommit == 1) { - enqueCommitTask(); - } } FeedHandler::CommitResult @@ -540,7 +510,7 @@ FeedHandler::startCommit(DoneCallback onDone) { void FeedHandler::storeOperationSync(const FeedOperation &op) { vespalib::Gate gate; - appendAndCommitOperation(op, make_shared(gate)); + appendOperation(op, make_shared(gate)); gate.await(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index 97629bfc018..4807c596130 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -76,8 +76,6 @@ private: // the serial num of the last message in the transaction log SerialNum _serialNum; SerialNum _prunedSerialNum; - size_t _numPendingCommit; - size_t _numCommitsCompleted; bool _delayedPrune; mutable std::shared_mutex _feedLock; FeedStateSP _feedState; @@ -127,9 +125,6 @@ private: FeedStateSP getFeedState() const; void changeFeedState(FeedStateSP newState); void doChangeFeedState(FeedStateSP newState); - void onCommitDone(uint64_t numPendingAtStart); - void initiateCommit(); - void enqueCommitTask(); public: FeedHandler(const FeedHandler &) = delete; FeedHandler & operator = (const FeedHandler &) = delete; diff --git a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h index c3b76a9db75..b276779c2ee 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h @@ -22,10 +22,6 @@ struct IOperationStorer */ virtual void appendOperation(const FeedOperation &op, DoneCallback onDone) = 0; [[nodiscard]] virtual CommitResult startCommit(DoneCallback onDone) = 0; - void appendAndCommitOperation(const FeedOperation &op, DoneCallback onDone) { - appendOperation(op, onDone); - (void) startCommit(std::move(onDone)); - } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp index 468850b4409..d423e095ad9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp @@ -82,7 +82,7 @@ LidSpaceCompactionJob::compactLidSpace(const LidUsageStats &stats) uint32_t wantedLidLimit = stats.getHighestUsedLid() + 1; CompactLidSpaceOperation op(_handler.getSubDbId(), wantedLidLimit); vespalib::Gate gate; - _opStorer.appendAndCommitOperation(op, std::make_shared(gate)); + _opStorer.appendOperation(op, std::make_shared(gate)); gate.await(); _handler.handleCompactLidSpace(op); EventLogger::lidSpaceCompactionComplete(_handler.getName(), wantedLidLimit); diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h index 35549f21471..37497eaa998 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h @@ -63,13 +63,13 @@ public: ~LidSpaceCompactionJob(); // Implements IDiskMemUsageListener - void notifyDiskMemUsage(DiskMemUsageState state) override; + virtual void notifyDiskMemUsage(DiskMemUsageState state) override; // Implements IClusterStateChangedNofifier - void notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) override; + virtual void notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) override; // Implements IMaintenanceJob - bool run() override; + virtual bool run() override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp index d4b542a0af8..d94bb2e3d03 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp @@ -105,7 +105,7 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller, AttributeUsageFilter &attributeUsageFilter) { controller.registerJobInMasterThread(std::make_unique(hbHandler, config.getHeartBeatConfig())); controller.registerJobInDefaultPool(std::make_unique(scPruner, config.getSessionCachePruneInterval())); - if (config.hasVisibilityDelay() && config.allowEarlyAck()) { + if (config.hasVisibilityDelay()) { controller.registerJobInMasterThread(std::make_unique(commit, config.getVisibilityDelay())); } const MaintenanceDocumentSubDB &mRemSubDB(controller.getRemSubDB()); diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index 9927a2d2ce0..217a3bb24d3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -210,11 +210,11 @@ moveMetaData(documentmetastore::IStore &meta_store, const DocumentId & doc_id, c } std::unique_ptr -createUncommitedLidTracker(bool allowEarlyAck) { - if (allowEarlyAck) { - return std::make_unique(); - } else { +createUncommitedLidTracker(bool needImmediateCommit) { + if (needImmediateCommit) { return std::make_unique(); + } else { + return std::make_unique(); } } @@ -229,7 +229,7 @@ StoreOnlyFeedView::StoreOnlyFeedView(const Context &ctx, const PersistentParams _docType(nullptr), _lidReuseDelayer(ctx._writeService, _documentMetaStoreContext->get(), ctx._lidReuseDelayerConfig), _pendingLidsForDocStore(), - _pendingLidsForCommit(createUncommitedLidTracker(_lidReuseDelayer.allowEarlyAck())), + _pendingLidsForCommit(createUncommitedLidTracker(_lidReuseDelayer.getImmediateCommit())), _schema(ctx._schema), _writeService(ctx._writeService), _params(params), @@ -275,7 +275,7 @@ StoreOnlyFeedView::internalForceCommit(SerialNum serialNum, OnForceCommitDoneTyp void StoreOnlyFeedView::considerEarlyAck(FeedToken & token) { - if ( _lidReuseDelayer.allowEarlyAck() && token) { + if ( ! needCommit() && token) { token.reset(); } } @@ -327,7 +327,7 @@ StoreOnlyFeedView::internalPut(FeedToken token, const PutOperation &putOp) bool docAlreadyExists = putOp.getValidPrevDbdId(_params._subDbId); if (putOp.getValidDbdId(_params._subDbId)) { - bool immediateCommit = needImmediateCommit(); + bool immediateCommit = needCommit(); const document::GlobalId &gid = docId.getGlobalId(); std::shared_ptr onWriteDone = createPutDoneContext(std::move(token), std::move(uncommitted), @@ -345,8 +345,8 @@ StoreOnlyFeedView::internalPut(FeedToken token, const PutOperation &putOp) } bool -StoreOnlyFeedView::needImmediateCommit() const { - return _lidReuseDelayer.needImmediateCommit(); +StoreOnlyFeedView::needCommit() const { + return _lidReuseDelayer.getImmediateCommit(); } void @@ -483,7 +483,7 @@ StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp) auto uncommitted = _pendingLidsForCommit->produce(updOp.getLid()); considerEarlyAck(token); - bool immediateCommit = needImmediateCommit(); + bool immediateCommit = needCommit(); auto onWriteDone = createUpdateDoneContext(std::move(token), std::move(uncommitted), updOp.getUpdate()); UpdateScope updateScope(*_schema, upd); updateAttributes(serialNum, lid, upd, immediateCommit, onWriteDone, updateScope); @@ -657,7 +657,7 @@ StoreOnlyFeedView::internalRemove(FeedToken token, IPendingLidTracker::Token unc std::move(pendingNotifyRemoveDone), (explicitReuseLid ? lid : 0u), std::move(moveDoneCtx)); removeSummary(serialNum, lid, onWriteDone); - bool immediateCommit = needImmediateCommit(); + bool immediateCommit = needCommit(); removeAttributes(serialNum, lid, immediateCommit, onWriteDone); removeIndexedFields(serialNum, lid, immediateCommit, onWriteDone); } @@ -770,7 +770,7 @@ StoreOnlyFeedView::handleDeleteBucket(const DeleteBucketOperation &delOp) void StoreOnlyFeedView::internalDeleteBucket(const DeleteBucketOperation &delOp) { - bool immediateCommit = needImmediateCommit(); + bool immediateCommit = needCommit(); size_t rm_count = removeDocuments(delOp, true, immediateCommit); LOG(debug, "internalDeleteBucket(): docType(%s), bucket(%s), lidsToRemove(%zu)", _params._docTypeName.toString().c_str(), delOp.getBucketId().toString().c_str(), rm_count); @@ -809,7 +809,7 @@ StoreOnlyFeedView::handleMove(const MoveOperation &moveOp, IDestructorCallback:: PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(moveOp, docId.getGlobalId(), docId); bool docAlreadyExists = moveOp.getValidPrevDbdId(_params._subDbId); if (moveOp.getValidDbdId(_params._subDbId)) { - bool immediateCommit = needImmediateCommit(); + bool immediateCommit = needCommit(); const document::GlobalId &gid = docId.getGlobalId(); std::shared_ptr onWriteDone = createPutDoneContext(FeedToken(), _pendingLidsForCommit->produce(moveOp.getLid()), diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h index 4569e01f9fd..7d91ea86a22 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h @@ -161,7 +161,7 @@ private: void putSummary(SerialNum serialNum, Lid lid, DocumentSP doc, OnOperationDoneType onDone); void removeSummary(SerialNum serialNum, Lid lid, OnWriteDoneType onDone); void heartBeatSummary(SerialNum serialNum); - bool needImmediateCommit() const; + bool needCommit() const; bool useDocumentStore(SerialNum replaySerialNum) const { diff --git a/searchlib/src/vespa/searchlib/config/translogserver.def b/searchlib/src/vespa/searchlib/config/translogserver.def index defce8c3421..540895b2404 100644 --- a/searchlib/src/vespa/searchlib/config/translogserver.def +++ b/searchlib/src/vespa/searchlib/config/translogserver.def @@ -15,7 +15,7 @@ basedir string default="tmp" restart ## Use fsync after each commit. ## If not the below interval is used. -usefsync bool default=false +usefsync bool default=false restart ##Number of threads available for visiting/subscription. maxthreads int default=4 restart diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index bd7feec0598..9e0f1a8a1aa 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -113,12 +113,7 @@ Domain::addPart(SerialNum partId, bool isLastPart) { } } -Domain::~Domain() { - MonitorGuard guard(_currentChunkMonitor); - guard.broadcast(); - commitChunk(grabCurrentChunk(guard), guard); - _singleCommitter->shutdown().sync(); -} +Domain::~Domain() { } DomainInfo Domain::getDomainInfo() const @@ -323,73 +318,22 @@ Domain::optionallyRotateFile(SerialNum serialNum) { return dp; } -void -Domain::append(const Packet & packet, Writer::DoneCallback onDone) { - vespalib::MonitorGuard guard(_currentChunkMonitor); - if (_lastSerial >= packet.range().from()) { - throw runtime_error(fmt("Incoming serial number(%" PRIu64 ") must be bigger than the last one (%" PRIu64 ").", - packet.range().from(), _lastSerial)); - } else { - _lastSerial = packet.range().to(); - } - _currentChunk->add(packet, std::move(onDone)); - commitIfFull(guard); -} - Domain::CommitResult Domain::startCommit(DoneCallback onDone) { - vespalib::MonitorGuard guard(_currentChunkMonitor); - if ( !_currentChunk->empty() ) { - auto completed = grabCurrentChunk(guard); - completed->setCommitDoneCallback(std::move(onDone)); - CommitResult result(completed->createCommitResult()); - commitChunk(std::move(completed), guard); - return result; - } + (void) onDone; return CommitResult(); } void -Domain::commitIfFull(const vespalib::MonitorGuard &guard) { - if (_currentChunk->sizeBytes() > _config.getChunkSizeLimit()) { - auto completed = std::move(_currentChunk); - _currentChunk = std::make_unique(_config.getChunkSizeLimit(), completed->stealCallbacks()); - commitChunk(std::move(completed), guard); - } -} - -std::unique_ptr -Domain::grabCurrentChunk(const vespalib::MonitorGuard & guard) { - assert(guard.monitors(_currentChunkMonitor)); - auto chunk = std::move(_currentChunk); - _currentChunk = createCommitChunk(_config); - return chunk; -} - -void -Domain::commitChunk(std::unique_ptr chunk, const vespalib::MonitorGuard & chunkOrderGuard) { - assert(chunkOrderGuard.monitors(_currentChunkMonitor)); - _singleCommitter->execute( makeLambdaTask([this, chunk = std::move(chunk)]() mutable { - doCommit(std::move(chunk)); - })); -} - -void -Domain::doCommit(std::unique_ptr chunk) { - const Packet & packet = chunk->getPacket(); - if (packet.empty()) return; - +Domain::append(const Packet & packet, Writer::DoneCallback onDone) +{ + (void) onDone; vespalib::nbostream_longlivedbuf is(packet.getHandle().data(), packet.getHandle().size()); Packet::Entry entry; entry.deserialize(is); DomainPart::SP dp = optionallyRotateFile(entry.serial()); dp->commit(entry.serial(), packet); - if (_config.getFSyncOnCommit()) { - dp->sync(); - } cleanSessions(); - LOG(debug, "Releasing %zu acks and %zu entries and %zu bytes.", - chunk->getNumCallBacks(), chunk->getPacket().size(), chunk->sizeBytes()); } bool diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h index 041ec27cf23..7e77e6ef0ef 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h @@ -56,11 +56,6 @@ public: uint64_t size() const; Domain & setConfig(const DomainConfig & cfg); private: - void commitIfFull(const vespalib::MonitorGuard & guard); - - std::unique_ptr grabCurrentChunk(const vespalib::MonitorGuard & guard); - void commitChunk(std::unique_ptr chunk, const vespalib::MonitorGuard & chunkOrderGuard); - void doCommit(std::unique_ptr chunk); SerialNum begin(const vespalib::LockGuard & guard) const; SerialNum end(const vespalib::LockGuard & guard) const; size_t byteSize(const vespalib::LockGuard & guard) const; diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp index b7e02894e6b..8855183226d 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp @@ -252,7 +252,7 @@ DomainPart::buildPacketMapping(bool allowTruncate) DomainPart::DomainPart(const string & name, const string & baseDir, SerialNum s, Encoding encoding, uint8_t compressionLevel, const FileHeaderContext &fileHeaderContext, bool allowTruncate) - : _encoding(encoding), + : _encoding(encoding.getCrc(), Encoding::Compression::none), //TODO We do not yet support compression _compressionLevel(compressionLevel), _lock(), _fileLock(), @@ -396,19 +396,16 @@ DomainPart::commit(SerialNum firstSerial, const Packet &packet) if (_range.from() == 0) { _range.from(firstSerial); } - IChunk::UP chunk = IChunk::create(_encoding, _compressionLevel); for (size_t i(0); h.size() > 0; i++) { //LOG(spam, //"Pos(%d) Len(%d), Lim(%d), Remaining(%d)", //h.getPos(), h.getLength(), h.getLimit(), h.getRemaining()); + IChunk::UP chunk = IChunk::create(_encoding, _compressionLevel); Packet::Entry entry; entry.deserialize(h); if (_range.to() < entry.serial()) { chunk->add(entry); - if (_encoding.getCompression() == Encoding::Compression::none) { - write(*_transLog, *chunk); - chunk = IChunk::create(_encoding, _compressionLevel); - } + write(*_transLog, *chunk); _sz++; _range.to(entry.serial()); } else { @@ -416,9 +413,6 @@ DomainPart::commit(SerialNum firstSerial, const Packet &packet) entry.serial(), _range.to())); } } - if ( ! chunk->getEntries().empty()) { - write(*_transLog, *chunk); - } bool merged(false); LockGuard guard(_lock); diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index 0c0c9186e12..7be3dd708a5 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -572,11 +572,8 @@ TransLogServer::domainCommit(FRT_RPCRequest *req) Packet packet(params[1]._data._buf, params[1]._data._len); try { vespalib::Gate gate; - { - // Need to scope in order to drain out all the callbacks. - domain->append(packet, make_shared(gate)); - auto keep = domain->startCommit(make_shared()); - } + domain->append(packet, make_shared(gate)); + auto keep = domain->startCommit(make_shared()); gate.await(); ret.AddInt32(0); ret.AddString("ok"); -- cgit v1.2.3 From 93d1fb64807a77db4a0280af877abb76be3fcd25 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Thu, 24 Sep 2020 19:17:43 +0200 Subject: Use MessageResponse --- .../vespa/config/server/http/v2/TenantCreateResponse.java | 13 +++---------- .../vespa/config/server/http/v2/TenantDeleteResponse.java | 13 +++---------- .../vespa/config/server/http/v2/TenantGetResponse.java | 13 +++---------- 3 files changed, 9 insertions(+), 30 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantCreateResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantCreateResponse.java index 2850698ea87..6ff2b30075d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantCreateResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantCreateResponse.java @@ -2,8 +2,7 @@ package com.yahoo.vespa.config.server.http.v2; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.http.HttpConfigResponse; -import com.yahoo.vespa.config.server.http.SessionResponse; +import com.yahoo.restapi.MessageResponse; /** * Response for tenant create @@ -11,16 +10,10 @@ import com.yahoo.vespa.config.server.http.SessionResponse; * @author vegardh * */ -public class TenantCreateResponse extends SessionResponse { +public class TenantCreateResponse extends MessageResponse { public TenantCreateResponse(TenantName tenant) { - super(); - this.root.setString("message", "Tenant "+tenant+" created."); - } - - @Override - public String getContentType() { - return HttpConfigResponse.JSON_CONTENT_TYPE; + super("Tenant " + tenant + " created."); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantDeleteResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantDeleteResponse.java index 3ba61f84270..d21584c8cdc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantDeleteResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantDeleteResponse.java @@ -1,8 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.http.HttpConfigResponse; -import com.yahoo.vespa.config.server.http.SessionResponse; +import com.yahoo.restapi.MessageResponse; /** * Response for tenant delete @@ -10,16 +9,10 @@ import com.yahoo.vespa.config.server.http.SessionResponse; * @author vegardh * */ -public class TenantDeleteResponse extends SessionResponse { +public class TenantDeleteResponse extends MessageResponse { public TenantDeleteResponse(TenantName tenant) { - super(); - this.root.setString("message", "Tenant "+tenant+" deleted."); - } - - @Override - public String getContentType() { - return HttpConfigResponse.JSON_CONTENT_TYPE; + super("Tenant " + tenant + " deleted."); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantGetResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantGetResponse.java index 183fcd9b3c1..b918cab7828 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantGetResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantGetResponse.java @@ -2,24 +2,17 @@ package com.yahoo.vespa.config.server.http.v2; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.http.HttpConfigResponse; -import com.yahoo.vespa.config.server.http.SessionResponse; +import com.yahoo.restapi.MessageResponse; /** * Response for tenant create * * @author hmusum */ -public class TenantGetResponse extends SessionResponse { +public class TenantGetResponse extends MessageResponse { public TenantGetResponse(TenantName tenant) { - super(); - this.root.setString("message", "Tenant '" + tenant + "' exists."); - } - - @Override - public String getContentType() { - return HttpConfigResponse.JSON_CONTENT_TYPE; + super("Tenant '" + tenant + "' exists."); } } -- cgit v1.2.3 From a62b109c312a00f4ec33229d1f54d515ca9e75c4 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Thu, 24 Sep 2020 19:38:40 +0200 Subject: Use SlimeJsonResponse --- .../vespa/config/server/http/ContentHandler.java | 3 +- .../server/http/SessionContentListResponse.java | 13 +----- .../http/SessionContentStatusListResponse.java | 16 ++------ .../server/http/SessionContentStatusResponse.java | 36 ++++------------- .../vespa/config/server/http/SessionResponse.java | 46 ---------------------- .../config/server/http/v2/ListTenantsResponse.java | 13 ++---- .../server/http/v2/SessionActiveResponse.java | 9 +++-- .../server/http/v2/SessionCreateHandler.java | 7 +--- .../server/http/v2/SessionCreateResponse.java | 26 +++++------- .../http/v2/SessionPrepareAndActivateResponse.java | 12 ++++-- .../server/http/v2/SessionPrepareHandler.java | 2 +- .../server/http/v2/SessionPrepareResponse.java | 23 ++++++----- .../config/server/http/v2/TenantHandlerTest.java | 5 +-- .../java/com/yahoo/restapi/SlimeJsonResponse.java | 6 ++- 14 files changed, 63 insertions(+), 154 deletions(-) delete mode 100644 configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionResponse.java diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentHandler.java index 7112c7d3e23..bbbc8764122 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentHandler.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; @@ -102,6 +103,6 @@ public class ContentHandler { Slime slime = new Slime(); Cursor root = slime.setObject(); root.setString("prepared", request.getUrlBase("/prepared")); - return new SessionResponse(slime, root); + return new SlimeJsonResponse(slime); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentListResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentListResponse.java index f761f5f6b6e..57095772b26 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentListResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentListResponse.java @@ -2,12 +2,9 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; -import java.io.IOException; -import java.io.OutputStream; import java.util.List; /** @@ -16,19 +13,13 @@ import java.util.List; * @author Ulf Lilleengen * @since 5.1 */ -class SessionContentListResponse extends SessionResponse { - private final Slime slime = new Slime(); +class SessionContentListResponse extends SlimeJsonResponse { public SessionContentListResponse(String urlBase, List files) { - super(); Cursor array = slime.setArray(); for (ApplicationFile file : files) { array.addString(urlBase + file.getPath() + (file.isDirectory() ? "/" : "")); } } - @Override - public void render(OutputStream outputStream) throws IOException { - new JsonFormat(true).encode(outputStream, slime); - } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java index 075b4bc329b..08cda869111 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java @@ -3,12 +3,10 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.application.api.ApplicationFile; import java.util.logging.Level; + +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; -import java.io.IOException; -import java.io.OutputStream; import java.util.*; /** @@ -16,14 +14,11 @@ import java.util.*; * * @author hmusum */ -class SessionContentStatusListResponse extends SessionResponse { +class SessionContentStatusListResponse extends SlimeJsonResponse { private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger("SessionContentStatusListResponse"); - private final Slime slime = new Slime(); - public SessionContentStatusListResponse(String urlBase, List files) { - super(); Cursor array = slime.setArray(); for (ApplicationFile f : files) { Cursor element = array.addObject(); @@ -34,9 +29,4 @@ class SessionContentStatusListResponse extends SessionResponse { } } - @Override - public void render(OutputStream outputStream) throws IOException { - new JsonFormat(true).encode(outputStream, slime); - } - } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusResponse.java index bd182093e99..e6909e32985 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusResponse.java @@ -1,28 +1,19 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http; -import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.slime.Cursor; -import java.io.*; /** * Represents a response for a request to show the status and md5sum of a file in the application package. * * @author hmusum */ -public class SessionContentStatusResponse extends SessionResponse { - - private final ApplicationFile file; - private final String urlBase; - private final ApplicationFile.MetaData metaData; - private final ObjectMapper mapper = new ObjectMapper(); +public class SessionContentStatusResponse extends SlimeJsonResponse { public SessionContentStatusResponse(ApplicationFile file, String urlBase) { - super(); - this.file = file; - this.urlBase = urlBase; - ApplicationFile.MetaData metaData; if (file == null) { metaData = new ApplicationFile.MetaData(ApplicationFile.ContentStatusDeleted, ""); @@ -32,24 +23,11 @@ public class SessionContentStatusResponse extends SessionResponse { if (metaData == null) { throw new IllegalArgumentException("Could not find status for '" + file.getPath() + "'"); } - this.metaData = metaData; - } - @Override - public void render(OutputStream outputStream) throws IOException { - mapper.writeValue(outputStream, new ResponseData(metaData.status, metaData.md5, urlBase + file.getPath())); - } - - private static class ResponseData { - public final String status; - public final String md5; - public final String name; - - private ResponseData(String status, String md5, String name) { - this.status = status; - this.md5 = md5; - this.name = name; - } + Cursor element = slime.setObject(); + element.setString("status", metaData.getStatus()); + element.setString("md5", metaData.getMd5()); + element.setString("name", urlBase + file.getPath()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionResponse.java deleted file mode 100644 index ad658e3848a..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionResponse.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http; - -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; - -import java.io.IOException; -import java.io.OutputStream; - -import static com.yahoo.jdisc.http.HttpResponse.Status.OK; - -/** - * Superclass for responses from session HTTP handlers. Implements the - * render method. - * - * @author hmusum - * @since 5.1.14 - */ -public class SessionResponse extends HttpResponse { - private final Slime slime; - protected final Cursor root; - - public SessionResponse() { - super(OK); - slime = new Slime(); - root = slime.setObject(); - } - - public SessionResponse(Slime slime, Cursor root) { - super(OK); - this.slime = slime; - this.root = root; - } - - @Override - public void render(OutputStream outputStream) throws IOException { - new JsonFormat(true).encode(outputStream, slime); - } - - @Override - public String getContentType() { - return HttpConfigResponse.JSON_CONTENT_TYPE; - } -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java index 95a71881b47..3789939429c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java @@ -3,26 +3,19 @@ package com.yahoo.vespa.config.server.http.v2; import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.TenantName; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; -import com.yahoo.vespa.config.server.http.HttpConfigResponse; -import com.yahoo.vespa.config.server.http.SessionResponse; /** * Tenant list response * * @author vegardh */ -public class ListTenantsResponse extends SessionResponse { +public class ListTenantsResponse extends SlimeJsonResponse { ListTenantsResponse(ImmutableSet tenants) { - super(); - Cursor tenantArray = this.root.setArray("tenants"); + Cursor tenantArray = slime.setObject().setArray("tenants"); tenants.forEach(tenantName -> tenantArray.addString(tenantName.value())); } - @Override - public String getContentType() { - return HttpConfigResponse.JSON_CONTENT_TYPE; - } - } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveResponse.java index 334dbe88614..9c0fbdf2613 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveResponse.java @@ -4,16 +4,19 @@ package com.yahoo.vespa.config.server.http.v2; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.config.server.http.SessionResponse; -public class SessionActiveResponse extends SessionResponse { +public class SessionActiveResponse extends SlimeJsonResponse { public SessionActiveResponse(Slime metaData, HttpRequest request, ApplicationId applicationId, long sessionId, Zone zone) { - super(metaData, metaData.get()); + super(metaData); TenantName tenantName = applicationId.tenant(); String message = "Session " + sessionId + " for tenant '" + tenantName.value() + "' activated."; + Cursor root = metaData.get(); + root.setString("tenant", tenantName.value()); root.setString("message", message); root.setString("url", "http://" + request.getHost() + ":" + request.getPort() + diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java index 5aee711b379..de8b2e63aeb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java @@ -60,7 +60,7 @@ public class SessionCreateHandler extends SessionHandler { ApplicationId applicationId = ApplicationId.from(tenantName, ApplicationName.defaultName(), InstanceName.defaultName()); sessionId = applicationRepository.createSession(applicationId, timeoutBudget, request.getData(), request.getHeader(ApplicationApiHandler.contentTypeHeader)); } - return createResponse(request, tenantName, deployLog, sessionId); + return new SessionCreateResponse(deployLog, tenantName, request.getHost(), request.getPort(), sessionId); } static ApplicationId getFromApplicationId(HttpRequest request) { @@ -99,9 +99,4 @@ public class SessionCreateHandler extends SessionHandler { ApplicationApiHandler.APPLICATION_X_GZIP + "' and '" + ApplicationApiHandler.APPLICATION_ZIP + "' are supported"); } } - - private HttpResponse createResponse(HttpRequest request, TenantName tenantName, Slime deployLog, long sessionId) { - return new SessionCreateResponse(tenantName, deployLog, deployLog.get()) - .createResponse(request.getHost(), request.getPort(), sessionId); - } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateResponse.java index 7d08ea94ce6..33c8f54b1f6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateResponse.java @@ -2,10 +2,9 @@ package com.yahoo.vespa.config.server.http.v2; import com.yahoo.config.provision.TenantName; -import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.server.http.SessionResponse; /** * Creates a response for SessionCreateHandler. @@ -13,22 +12,17 @@ import com.yahoo.vespa.config.server.http.SessionResponse; * @author hmusum * @since 5.1.27 */ -public class SessionCreateResponse extends SessionResponse { - private final TenantName tenantName; +public class SessionCreateResponse extends SlimeJsonResponse { - public SessionCreateResponse(TenantName tenantName, Slime deployLog, Cursor root) { - super(deployLog, root); - this.tenantName = tenantName; - } - - public HttpResponse createResponse(String hostName, int port, long sessionId) { + public SessionCreateResponse(Slime deployLog, TenantName tenantName, String hostName, int port, long sessionId) { + super(deployLog); String path = "http://" + hostName + ":" + port + "/application/v2/tenant/" + tenantName.value() + "/session/" + sessionId; + Cursor root = deployLog.get(); - this.root.setString("tenant", tenantName.value()); - this.root.setString("session-id", Long.toString(sessionId)); - this.root.setString("prepared", path + "/prepared"); - this.root.setString("content", path + "/content/"); - this.root.setString("message", "Session " + sessionId + " for tenant '" + tenantName.value() + "' created."); - return this; + root.setString("tenant", tenantName.value()); + root.setString("session-id", Long.toString(sessionId)); + root.setString("prepared", path + "/prepared"); + root.setString("content", path + "/content/"); + root.setString("message", "Session " + sessionId + " for tenant '" + tenantName.value() + "' created."); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareAndActivateResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareAndActivateResponse.java index 7d9a0b11c28..7bace4749a8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareAndActivateResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareAndActivateResponse.java @@ -5,21 +5,25 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.slime.Cursor; import com.yahoo.vespa.config.server.configchange.ConfigChangeActionsSlimeConverter; -import com.yahoo.vespa.config.server.http.SessionResponse; /** * Creates a response for SessionPrepareHandler. * * @author hmusum */ -class SessionPrepareAndActivateResponse extends SessionResponse { +class SessionPrepareAndActivateResponse extends SlimeJsonResponse { SessionPrepareAndActivateResponse(PrepareResult result, HttpRequest request, ApplicationId applicationId, Zone zone) { - super(result.deployLog(), result.deployLog().get()); + super(result.deployLogger().slime()); + TenantName tenantName = applicationId.tenant(); String message = "Session " + result.sessionId() + " for tenant '" + tenantName.value() + "' prepared and activated."; - this.root.setString("tenant", tenantName.value()); + Cursor root = slime.get(); + + root.setString("tenant", tenantName.value()); root.setString("url", "http://" + request.getHost() + ":" + request.getPort() + "/application/v2/tenant/" + tenantName + "/application/" + applicationId.application().value() + diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java index c0789a9c828..4cb07e37f28 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java @@ -51,7 +51,7 @@ public class SessionPrepareHandler extends SessionHandler { long sessionId = getSessionIdV2(request); applicationRepository.validateThatSessionIsNotActive(tenant, sessionId); applicationRepository.validateThatSessionIsPrepared(tenant, sessionId); - return new SessionPrepareResponse(applicationRepository.createDeployLog(), tenant.getName(), request, sessionId); + return new SessionPrepareResponse(tenant.getName(), request, sessionId); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareResponse.java index 6d2aa426036..a97cd37d3b4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareResponse.java @@ -3,33 +3,36 @@ package com.yahoo.vespa.config.server.http.v2; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; +import com.yahoo.slime.Type; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; import com.yahoo.vespa.config.server.configchange.ConfigChangeActionsSlimeConverter; -import com.yahoo.vespa.config.server.http.SessionResponse; /** * Creates a response for SessionPrepareHandler. * * @author hmusum */ -class SessionPrepareResponse extends SessionResponse { +class SessionPrepareResponse extends SlimeJsonResponse { - SessionPrepareResponse(Slime deployLog, TenantName tenantName, HttpRequest request, long sessionId) { - this(deployLog, tenantName, request, sessionId, new ConfigChangeActions()); + SessionPrepareResponse(TenantName tenantName, HttpRequest request, long sessionId) { + this(new Slime(), tenantName, request, sessionId, new ConfigChangeActions()); } SessionPrepareResponse(PrepareResult result, TenantName tenantName, HttpRequest request) { - this(result.deployLog(), tenantName, request, result.sessionId(), result.configChangeActions()); + this(result.deployLogger().slime(), tenantName, request, result.sessionId(), result.configChangeActions()); } private SessionPrepareResponse(Slime deployLog, TenantName tenantName, HttpRequest request, long sessionId, ConfigChangeActions actions) { - super(deployLog, deployLog.get()); - String message = "Session " + sessionId + " for tenant '" + tenantName.value() + "' prepared."; - this.root.setString("tenant", tenantName.value()); - this.root.setString("activate", "http://" + request.getHost() + ":" + request.getPort() + + super(deployLog); + + Cursor root = deployLog.get().type() != Type.NIX ? deployLog.get() : deployLog.setObject(); + root.setString("tenant", tenantName.value()); + root.setString("activate", "http://" + request.getHost() + ":" + request.getPort() + "/application/v2/tenant/" + tenantName.value() + "/session/" + sessionId + "/active"); - root.setString("message", message); + root.setString("message", "Session " + sessionId + " for tenant '" + tenantName.value() + "' prepared."); new ConfigChangeActionsSlimeConverter(actions).toSlime(root); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java index e2eeb68a565..748c43bafeb 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java @@ -8,18 +8,17 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.time.Clock; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.SessionHandlerTest; -import com.yahoo.vespa.config.server.http.SessionResponse; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; @@ -157,7 +156,7 @@ public class TenantHandlerTest { return (TenantCreateResponse) handler.handlePUT(testRequest); } - private void assertResponseEquals(SessionResponse response, String payload) throws IOException { + private void assertResponseEquals(HttpResponse response, String payload) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); response.render(baos); assertEquals(baos.toString(StandardCharsets.UTF_8), payload); diff --git a/container-core/src/main/java/com/yahoo/restapi/SlimeJsonResponse.java b/container-core/src/main/java/com/yahoo/restapi/SlimeJsonResponse.java index 2473da3578d..a0f8b58fb6a 100644 --- a/container-core/src/main/java/com/yahoo/restapi/SlimeJsonResponse.java +++ b/container-core/src/main/java/com/yahoo/restapi/SlimeJsonResponse.java @@ -15,7 +15,11 @@ import java.io.OutputStream; */ public class SlimeJsonResponse extends HttpResponse { - private final Slime slime; + protected final Slime slime; + + public SlimeJsonResponse() { + this(new Slime()); + } public SlimeJsonResponse(Slime slime) { super(200); -- cgit v1.2.3 From 266288b03b51d763ea57f2a4e08aec7b529430a2 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Thu, 24 Sep 2020 19:56:22 +0200 Subject: Store DeployHandlerLogger in PrepareResult --- .../vespa/config/server/ApplicationRepository.java | 13 ++------- .../config/server/deploy/DeployHandlerLogger.java | 33 ++++++++++++++-------- .../vespa/config/server/http/SessionHandler.java | 11 -------- .../vespa/config/server/http/v2/PrepareResult.java | 12 ++++---- .../server/http/v2/SessionCreateHandler.java | 16 ++++------- .../server/deploy/DeployHandlerLoggerTest.java | 10 ++----- .../config/server/session/SessionPreparerTest.java | 5 ++-- 7 files changed, 42 insertions(+), 58 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index fb981eb1c67..d3d95302f57 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -285,15 +285,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye LocalSession session = getLocalSession(tenant, sessionId); ApplicationId applicationId = prepareParams.getApplicationId(); Optional currentActiveApplicationSet = getCurrentActiveApplicationSet(tenant, applicationId); - Slime deployLog = createDeployLog(); - DeployLogger logger = new DeployHandlerLogger(deployLog.get().setArray("log"), prepareParams.isVerbose(), applicationId); + DeployHandlerLogger logger = DeployHandlerLogger.forApplication(applicationId, prepareParams.isVerbose()); try (ActionTimer timer = timerFor(applicationId, "deployment.prepareMillis")) { SessionRepository sessionRepository = tenant.getSessionRepository(); ConfigChangeActions actions = sessionRepository.prepareLocalSession(session, logger, prepareParams, currentActiveApplicationSet, tenant.getPath(), now); logConfigChangeActions(actions, logger); log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " prepared successfully. "); - return new PrepareResult(sessionId, actions, deployLog); + return new PrepareResult(sessionId, actions, logger); } } @@ -330,7 +329,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye hostProvisioner.get().restart(applicationId, HostFilter.from(hostnames, Set.of(), Set.of(), Set.of())); ConfigChangeActions newActions = new ConfigChangeActions(new RestartActions(), result.configChangeActions().getRefeedActions()); - return new PrepareResult(result.sessionId(), newActions, result.deployLog()); + return new PrepareResult(result.sessionId(), newActions, result.deployLogger()); } return result; @@ -1072,12 +1071,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye .getPort(); } - public Slime createDeployLog() { - Slime deployLog = new Slime(); - deployLog.setObject(); - return deployLog; - } - public Zone zone() { return new Zone(SystemName.from(configserverConfig.system()), Environment.from(configserverConfig.environment()), diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java index c33c5ff9f57..b6cd22d78b4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.deploy; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; import com.yahoo.log.LogLevel; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; @@ -22,25 +23,24 @@ public class DeployHandlerLogger implements DeployLogger { private static final Logger log = Logger.getLogger(DeployHandlerLogger.class.getName()); - private final Cursor logroot; + private final String prefix; private final boolean verbose; - private final ApplicationId app; + private final Slime slime; + private final Cursor logroot; - public DeployHandlerLogger(Cursor root, boolean verbose, ApplicationId app) { - logroot = root; + private DeployHandlerLogger(String prefix, boolean verbose) { + this.prefix = prefix; this.verbose = verbose; - this.app = app; + this.slime = new Slime(); + this.logroot = slime.setObject().setArray("log"); } @Override public void log(Level level, String message) { - if ((level == Level.FINE || - level == LogLevel.DEBUG || - level == LogLevel.SPAM) && - !verbose) { + if ((level == Level.FINE || level == LogLevel.DEBUG || level == LogLevel.SPAM) && !verbose) return; - } - String fullMsg = TenantRepository.logPre(app) + message; + + String fullMsg = prefix + message; Cursor entry = logroot.addObject(); entry.setLong("time", System.currentTimeMillis()); entry.setString("level", level.getName()); @@ -49,4 +49,15 @@ public class DeployHandlerLogger implements DeployLogger { log.log(Level.FINE, fullMsg); } + public Slime slime() { + return slime; + } + + public static DeployHandlerLogger forApplication(ApplicationId app, boolean verbose) { + return new DeployHandlerLogger(TenantRepository.logPre(app), verbose); + } + + public static DeployHandlerLogger forTenant(TenantName tenantName, boolean verbose) { + return new DeployHandlerLogger(TenantRepository.logPre(tenantName), verbose); + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java index 59d12e037e9..fcac023eec3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java @@ -1,12 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.jdisc.application.BindingMatch; -import com.yahoo.slime.Slime; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; import com.yahoo.vespa.config.server.TimeoutBudget; import java.time.Clock; @@ -67,14 +64,6 @@ public class SessionHandler extends HttpHandler { return new TimeoutBudget(Clock.systemUTC(), getRequestTimeout(request, defaultTimeout)); } - public static DeployHandlerLogger createLogger(Slime deployLog, HttpRequest request, ApplicationId app) { - return createLogger(deployLog, request.getBooleanProperty("verbose"), app); - } - - public static DeployHandlerLogger createLogger(Slime deployLog, boolean verbose, ApplicationId app) { - return new DeployHandlerLogger(deployLog.get().setArray("log"), verbose, app); - } - /** * True if this request should ignore activation failure because the session was made from an active session that is not active now * @param request a {@link com.yahoo.container.jdisc.HttpRequest} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/PrepareResult.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/PrepareResult.java index bb2b57ba45c..24bdfd81f1c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/PrepareResult.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/PrepareResult.java @@ -1,8 +1,8 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import com.yahoo.slime.Slime; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; +import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; /** * Encapsulates the result from preparing an application @@ -13,12 +13,12 @@ public class PrepareResult { private final long sessionId; private final ConfigChangeActions configChangeActions; - private final Slime deployLog; + private final DeployHandlerLogger logger; - public PrepareResult(long sessionId, ConfigChangeActions configChangeActions, Slime deployLog) { + public PrepareResult(long sessionId, ConfigChangeActions configChangeActions, DeployHandlerLogger logger) { this.sessionId = sessionId; this.configChangeActions = configChangeActions; - this.deployLog = deployLog; + this.logger = logger; } public long sessionId() { @@ -29,8 +29,8 @@ public class PrepareResult { return configChangeActions; } - public Slime deployLog() { - return deployLog; + public DeployHandlerLogger deployLogger() { + return logger; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java index de8b2e63aeb..b0468f5e608 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.config.server.http.v2; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; @@ -11,7 +10,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.application.UriPattern; -import com.yahoo.slime.Slime; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; import com.yahoo.vespa.config.server.TimeoutBudget; @@ -45,22 +43,25 @@ public class SessionCreateHandler extends SessionHandler { @Override protected HttpResponse handlePOST(HttpRequest request) { - Slime deployLog = applicationRepository.createDeployLog(); final TenantName tenantName = Utils.getTenantNameFromSessionRequest(request); Utils.checkThatTenantExists(applicationRepository.tenantRepository(), tenantName); TimeoutBudget timeoutBudget = SessionHandler.getTimeoutBudget(request, zookeeperBarrierTimeout); - DeployLogger logger = createLogger(request, deployLog, tenantName); + boolean verbose = request.getBooleanProperty("verbose"); + + DeployHandlerLogger logger; long sessionId; if (request.hasProperty("from")) { ApplicationId applicationId = getFromApplicationId(request); + logger = DeployHandlerLogger.forApplication(applicationId, verbose); sessionId = applicationRepository.createSessionFromExisting(applicationId, logger, false, timeoutBudget); } else { validateDataAndHeader(request); + logger = DeployHandlerLogger.forTenant(tenantName, verbose); // TODO: Avoid using application id here at all ApplicationId applicationId = ApplicationId.from(tenantName, ApplicationName.defaultName(), InstanceName.defaultName()); sessionId = applicationRepository.createSession(applicationId, timeoutBudget, request.getData(), request.getHeader(ApplicationApiHandler.contentTypeHeader)); } - return new SessionCreateResponse(deployLog, tenantName, request.getHost(), request.getPort(), sessionId); + return new SessionCreateResponse(logger.slime(), tenantName, request.getHost(), request.getPort(), sessionId); } static ApplicationId getFromApplicationId(HttpRequest request) { @@ -82,11 +83,6 @@ public class SessionCreateHandler extends SessionHandler { .instanceName(match.group(6)).build(); } - private static DeployHandlerLogger createLogger(HttpRequest request, Slime deployLog, TenantName tenant) { - return SessionHandler.createLogger(deployLog, request, - new ApplicationId.Builder().tenant(tenant).applicationName("-").build()); - } - static void validateDataAndHeader(HttpRequest request) { if (request.getData() == null) { throw new BadRequestException("Request contains no data"); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java index 6aa72e3e672..3ac9e681604 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java @@ -6,9 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import java.util.logging.Level; import com.yahoo.log.LogLevel; -import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; import org.junit.Test; @@ -33,13 +31,11 @@ public class DeployHandlerLoggerTest { } private void testLogging(boolean verbose, String expectedPattern) throws IOException { - Slime slime = new Slime(); - Cursor array = slime.setArray(); - DeployLogger logger = new DeployHandlerLogger(array, verbose, new ApplicationId.Builder() - .tenant("testtenant").applicationName("testapp").build()); + DeployHandlerLogger logger = DeployHandlerLogger.forApplication( + new ApplicationId.Builder().tenant("testtenant").applicationName("testapp").build(), verbose); logMessages(logger); ByteArrayOutputStream baos = new ByteArrayOutputStream(); - new JsonFormat(true).encode(baos, slime); + new JsonFormat(true).encode(baos, logger.slime()); assertTrue(Pattern.matches(expectedPattern, baos.toString())); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index c1698718ad1..95cdccf4cb8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -28,7 +28,6 @@ import com.yahoo.security.KeyUtils; import com.yahoo.security.SignatureAlgorithm; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.X509CertificateUtils; -import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.MockSecretStore; import com.yahoo.vespa.config.server.TestComponentRegistry; @@ -388,8 +387,8 @@ public class SessionPreparerTest { } private DeployHandlerLogger getLogger() { - return new DeployHandlerLogger(new Slime().get(), false /*verbose */, - new ApplicationId.Builder().tenant("testtenant").applicationName("testapp").build()); + return DeployHandlerLogger.forApplication( + new ApplicationId.Builder().tenant("testtenant").applicationName("testapp").build(), false /*verbose */); } -- cgit v1.2.3 From 432dffc30c93e2de1fd2ac037ca1ed1b00a91da3 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Thu, 24 Sep 2020 19:58:26 +0200 Subject: Log internal restarts --- .../java/com/yahoo/vespa/config/server/ApplicationRepository.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index d3d95302f57..5fe4d64871a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -322,12 +322,16 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye activate(tenant, sessionId, prepareParams.getTimeoutBudget(), prepareParams.force()); if (prepareParams.internalRestart() && !result.configChangeActions().getRestartActions().isEmpty()) { + result.deployLogger().log(Level.INFO, result.configChangeActions().getRestartActions().format()); Set hostnames = result.configChangeActions().getRestartActions().getEntries().stream() .flatMap(entry -> entry.getServices().stream()) .map(ServiceInfo::getHostName) .collect(Collectors.toUnmodifiableSet()); hostProvisioner.get().restart(applicationId, HostFilter.from(hostnames, Set.of(), Set.of(), Set.of())); + result.deployLogger().log(Level.INFO, String.format("Scheduled service restart of %d nodes: %s", + hostnames.size(), hostnames.stream().sorted().collect(Collectors.joining(", ")))); + ConfigChangeActions newActions = new ConfigChangeActions(new RestartActions(), result.configChangeActions().getRefeedActions()); return new PrepareResult(result.sessionId(), newActions, result.deployLogger()); } -- cgit v1.2.3 From c648df7a83081440f21fb6af2edcdcfe99a18124 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Thu, 24 Sep 2020 19:59:01 +0200 Subject: Revert "Support multiple levels of directories for flags" --- .../api/systemflags/v1/SystemFlagsDataArchive.java | 18 +----------------- .../api/systemflags/v1/SystemFlagsDataArchiveTest.java | 13 ------------- .../flags/group-1/my-test-flag/default.json | 8 -------- .../flags/group-2/my-test-flag/default.json | 8 -------- .../flags/group-1/my-test-flag/default.json | 8 -------- .../flags/group-2/my-other-test-flag/default.json | 8 -------- 6 files changed, 1 insertion(+), 62 deletions(-) delete mode 100644 controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-1/my-test-flag/default.json delete mode 100644 controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-2/my-test-flag/default.json delete mode 100644 controller-api/src/test/resources/system-flags-multi-level/flags/group-1/my-test-flag/default.json delete mode 100644 controller-api/src/test/resources/system-flags-multi-level/flags/group-2/my-other-test-flag/default.json diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java index a00992da815..e6310cc6432 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java @@ -45,11 +45,6 @@ import static com.yahoo.yolean.Exceptions.uncheck; * The flag files must reside in a 'flags/' root directory containing a directory for each flag name: * {@code ./flags//*.json} * - * Optionally, there can be an arbitrary number of directories "between" 'flags/' root directory and - * the flag name directory: - * {@code ./flags/onelevel//*.json} - * {@code ./flags/onelevel/anotherlevel//*.json} - * * @author bjorncs */ public class SystemFlagsDataArchive { @@ -160,7 +155,7 @@ public class SystemFlagsDataArchive { if (!filename.endsWith(".json")) { throw new IllegalArgumentException(String.format("Only JSON files are allowed in 'flags/' directory (found '%s')", filePath.toString())); } - FlagId directoryDeducedFlagId = new FlagId(filePath.getName(filePath.getNameCount()-2).toString()); + FlagId directoryDeducedFlagId = new FlagId(filePath.getName(1).toString()); FlagData flagData; if (rawData.isBlank()) { flagData = new FlagData(directoryDeducedFlagId); @@ -183,13 +178,6 @@ public class SystemFlagsDataArchive { "\nSee https://git.ouroath.com/vespa/hosted-feature-flags for more info on the JSON syntax"); } } - - if (builder.hasFile(filename, flagData)) { - throw new IllegalArgumentException( - String.format("Flag data file in '%s' contains redundant flag data for id '%s' already set in another directory!", - filePath, flagData.id())); - } - builder.addFile(filename, flagData); } @@ -248,10 +236,6 @@ public class SystemFlagsDataArchive { return this; } - public boolean hasFile(String filename, FlagData data) { - return files.containsKey(data.id()) && files.get(data.id()).containsKey(filename); - } - public SystemFlagsDataArchive build() { Map> copy = new TreeMap<>(); files.forEach((flagId, map) -> copy.put(flagId, new TreeMap<>(map))); diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java index 21e89cb5ea3..4cdbe5241bc 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java @@ -83,19 +83,6 @@ public class SystemFlagsDataArchiveTest { assertArchiveReturnsCorrectTestFlagDataForTarget(archive); } - @Test - public void supports_multi_level_flags_directory() { - var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags-multi-level/")); - assertFlagDataHasValue(archive, MY_TEST_FLAG, mainControllerTarget, "default"); - } - - @Test - public void duplicated_flagdata_is_detected() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("Flag data file in 'flags/group-1/my-test-flag/default.json' contains redundant flag data for id 'my-test-flag' already set in another directory!"); - var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags-multi-level-with-duplicated-flagdata/")); - } - @Test public void empty_files_are_handled_as_no_flag_data_for_target() { var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags/")); diff --git a/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-1/my-test-flag/default.json b/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-1/my-test-flag/default.json deleted file mode 100644 index 5924eb860c0..00000000000 --- a/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-1/my-test-flag/default.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "id" : "my-test-flag", - "rules" : [ - { - "value" : "default" - } - ] -} \ No newline at end of file diff --git a/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-2/my-test-flag/default.json b/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-2/my-test-flag/default.json deleted file mode 100644 index 5924eb860c0..00000000000 --- a/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-2/my-test-flag/default.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "id" : "my-test-flag", - "rules" : [ - { - "value" : "default" - } - ] -} \ No newline at end of file diff --git a/controller-api/src/test/resources/system-flags-multi-level/flags/group-1/my-test-flag/default.json b/controller-api/src/test/resources/system-flags-multi-level/flags/group-1/my-test-flag/default.json deleted file mode 100644 index 5924eb860c0..00000000000 --- a/controller-api/src/test/resources/system-flags-multi-level/flags/group-1/my-test-flag/default.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "id" : "my-test-flag", - "rules" : [ - { - "value" : "default" - } - ] -} \ No newline at end of file diff --git a/controller-api/src/test/resources/system-flags-multi-level/flags/group-2/my-other-test-flag/default.json b/controller-api/src/test/resources/system-flags-multi-level/flags/group-2/my-other-test-flag/default.json deleted file mode 100644 index e30485b755c..00000000000 --- a/controller-api/src/test/resources/system-flags-multi-level/flags/group-2/my-other-test-flag/default.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "id" : "my-other-test-flag", - "rules" : [ - { - "value" : "default" - } - ] -} \ No newline at end of file -- cgit v1.2.3 From 8309eba80f9e2869e2d095588b0f0e2c1e930f82 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Thu, 24 Sep 2020 20:23:17 +0200 Subject: Do not log restart actions, logged in prepare() --- .../main/java/com/yahoo/vespa/config/server/ApplicationRepository.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 5fe4d64871a..bd704611bcb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -24,7 +24,6 @@ import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; import com.yahoo.path.Path; -import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.application.Application; @@ -322,7 +321,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye activate(tenant, sessionId, prepareParams.getTimeoutBudget(), prepareParams.force()); if (prepareParams.internalRestart() && !result.configChangeActions().getRestartActions().isEmpty()) { - result.deployLogger().log(Level.INFO, result.configChangeActions().getRestartActions().format()); Set hostnames = result.configChangeActions().getRestartActions().getEntries().stream() .flatMap(entry -> entry.getServices().stream()) .map(ServiceInfo::getHostName) -- cgit v1.2.3 From b2659214ba1810ade0cb490c87835f1025240b33 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Thu, 24 Sep 2020 21:03:45 +0200 Subject: Revert "Revert "Revert "Revert "Balder/group operations to tls and commit in batches"""" --- .../documentmetastore/lid_reuse_delayer_config.cpp | 2 + .../documentmetastore/lid_reuse_delayer_config.h | 2 + .../proton/documentmetastore/lidreusedelayer.cpp | 1 + .../proton/documentmetastore/lidreusedelayer.h | 4 +- .../proton/server/document_db_maintenance_config.h | 1 + .../vespa/searchcore/proton/server/feedhandler.cpp | 32 ++++++++++- .../vespa/searchcore/proton/server/feedhandler.h | 5 ++ .../searchcore/proton/server/i_operation_storer.h | 4 ++ .../proton/server/lid_space_compaction_job.cpp | 2 +- .../proton/server/lid_space_compaction_job.h | 6 +- .../proton/server/maintenance_jobs_injector.cpp | 2 +- .../searchcore/proton/server/storeonlyfeedview.cpp | 26 ++++----- .../searchcore/proton/server/storeonlyfeedview.h | 2 +- .../src/vespa/searchlib/config/translogserver.def | 2 +- .../src/vespa/searchlib/transactionlog/domain.cpp | 66 ++++++++++++++++++++-- .../src/vespa/searchlib/transactionlog/domain.h | 5 ++ .../vespa/searchlib/transactionlog/domainpart.cpp | 12 +++- .../searchlib/transactionlog/translogserver.cpp | 7 ++- 18 files changed, 149 insertions(+), 32 deletions(-) diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp index b04bac5ef26..1f979d1566c 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp @@ -7,6 +7,7 @@ namespace proton::documentmetastore { LidReuseDelayerConfig::LidReuseDelayerConfig(const DocumentDBConfig & configSnapshot) : _visibilityDelay(configSnapshot.getMaintenanceConfigSP()->getVisibilityDelay()), + _allowEarlyAck(configSnapshot.getMaintenanceConfigSP()->allowEarlyAck()), _hasIndexedOrAttributeFields(configSnapshot.getSchemaSP()->getNumIndexFields() > 0 || configSnapshot.getSchemaSP()->getNumAttributeFields() > 0) { @@ -18,6 +19,7 @@ LidReuseDelayerConfig::LidReuseDelayerConfig() LidReuseDelayerConfig::LidReuseDelayerConfig(vespalib::duration visibilityDelay, bool hasIndexedOrAttributeFields_in) : _visibilityDelay(visibilityDelay), + _allowEarlyAck(visibilityDelay > 1ms), _hasIndexedOrAttributeFields(hasIndexedOrAttributeFields_in) { } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h index 82dab433a22..c81a2ff399f 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h @@ -15,6 +15,7 @@ class LidReuseDelayerConfig { private: vespalib::duration _visibilityDelay; + bool _allowEarlyAck; bool _hasIndexedOrAttributeFields; public: LidReuseDelayerConfig(); @@ -22,6 +23,7 @@ public: explicit LidReuseDelayerConfig(const DocumentDBConfig &configSnapshot); vespalib::duration visibilityDelay() const { return _visibilityDelay; } bool hasIndexedOrAttributeFields() const { return _hasIndexedOrAttributeFields; } + bool allowEarlyAck() const { return _allowEarlyAck; } }; } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp index 03dfd83a132..ed2202c830b 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp @@ -16,6 +16,7 @@ LidReuseDelayer::LidReuseDelayer(IThreadingService &writeService, IStore &docume : _writeService(writeService), _documentMetaStore(documentMetaStore), _immediateCommit(config.visibilityDelay() == vespalib::duration::zero()), + _allowEarlyAck(config.allowEarlyAck()), _config(config), _pendingLids() { diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h index 5f1de878b4a..ba407ab57f8 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h @@ -27,6 +27,7 @@ class LidReuseDelayer searchcorespi::index::IThreadingService &_writeService; IStore &_documentMetaStore; const bool _immediateCommit; + const bool _allowEarlyAck; LidReuseDelayerConfig _config; std::vector _pendingLids; // lids waiting for commit @@ -38,7 +39,8 @@ public: bool delayReuse(const std::vector &lids); std::vector getReuseLids(); - bool getImmediateCommit() const { return _immediateCommit; } + bool needImmediateCommit() const { return _immediateCommit; } + bool allowEarlyAck() const { return _allowEarlyAck; } const LidReuseDelayerConfig & getConfig() const { return _config; } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h index 4c0485baec6..5ed0ad7492c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h @@ -135,6 +135,7 @@ public: } vespalib::duration getVisibilityDelay() const { return _visibilityDelay; } bool hasVisibilityDelay() const { return _visibilityDelay > vespalib::duration::zero(); } + bool allowEarlyAck() const { return _visibilityDelay > 1ms; } const DocumentDBLidSpaceCompactionConfig &getLidSpaceCompactionConfig() const { return _lidSpaceCompaction; } diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 37dfddf0c2c..209a35ce4a2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -402,6 +402,8 @@ FeedHandler::FeedHandler(IThreadingService &writeService, _tlsReplayProgress(), _serialNum(0), _prunedSerialNum(0), + _numPendingCommit(0), + _numCommitsCompleted(0), _delayedPrune(false), _feedLock(), _feedState(make_shared(getDocTypeName())), @@ -494,12 +496,40 @@ FeedHandler::getTransactionLogReplayDone() const { return _tlsMgr.getReplayDone(); } +void +FeedHandler::onCommitDone(uint64_t numPendingAtStart) { + assert(numPendingAtStart <= _numPendingCommit); + _numPendingCommit -= numPendingAtStart; + if (_numPendingCommit > 0) { + enqueCommitTask(); + } +} + +void FeedHandler::enqueCommitTask() { + _writeService.master().execute(makeLambdaTask([this]() { initiateCommit(); })); +} + +void +FeedHandler::initiateCommit() { + auto commitResult = _tlsWriter->startCommit(std::make_shared( + _writeService.master(), + makeLambdaTask([this, numPendingAtStart=_numPendingCommit]() { + onCommitDone(numPendingAtStart); + }))); + if (_activeFeedView) { + _activeFeedView->forceCommit(_serialNum, std::make_shared>(std::move(commitResult))); + } +} + void FeedHandler::appendOperation(const FeedOperation &op, TlsWriter::DoneCallback onDone) { if (!op.getSerialNum()) { const_cast(op).setSerialNum(incSerialNum()); } _tlsWriter->appendOperation(op, std::move(onDone)); + if (++_numPendingCommit == 1) { + enqueCommitTask(); + } } FeedHandler::CommitResult @@ -510,7 +540,7 @@ FeedHandler::startCommit(DoneCallback onDone) { void FeedHandler::storeOperationSync(const FeedOperation &op) { vespalib::Gate gate; - appendOperation(op, make_shared(gate)); + appendAndCommitOperation(op, make_shared(gate)); gate.await(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index 4807c596130..97629bfc018 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -76,6 +76,8 @@ private: // the serial num of the last message in the transaction log SerialNum _serialNum; SerialNum _prunedSerialNum; + size_t _numPendingCommit; + size_t _numCommitsCompleted; bool _delayedPrune; mutable std::shared_mutex _feedLock; FeedStateSP _feedState; @@ -125,6 +127,9 @@ private: FeedStateSP getFeedState() const; void changeFeedState(FeedStateSP newState); void doChangeFeedState(FeedStateSP newState); + void onCommitDone(uint64_t numPendingAtStart); + void initiateCommit(); + void enqueCommitTask(); public: FeedHandler(const FeedHandler &) = delete; FeedHandler & operator = (const FeedHandler &) = delete; diff --git a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h index b276779c2ee..c3b76a9db75 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h @@ -22,6 +22,10 @@ struct IOperationStorer */ virtual void appendOperation(const FeedOperation &op, DoneCallback onDone) = 0; [[nodiscard]] virtual CommitResult startCommit(DoneCallback onDone) = 0; + void appendAndCommitOperation(const FeedOperation &op, DoneCallback onDone) { + appendOperation(op, onDone); + (void) startCommit(std::move(onDone)); + } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp index d423e095ad9..468850b4409 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp @@ -82,7 +82,7 @@ LidSpaceCompactionJob::compactLidSpace(const LidUsageStats &stats) uint32_t wantedLidLimit = stats.getHighestUsedLid() + 1; CompactLidSpaceOperation op(_handler.getSubDbId(), wantedLidLimit); vespalib::Gate gate; - _opStorer.appendOperation(op, std::make_shared(gate)); + _opStorer.appendAndCommitOperation(op, std::make_shared(gate)); gate.await(); _handler.handleCompactLidSpace(op); EventLogger::lidSpaceCompactionComplete(_handler.getName(), wantedLidLimit); diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h index 37497eaa998..35549f21471 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h @@ -63,13 +63,13 @@ public: ~LidSpaceCompactionJob(); // Implements IDiskMemUsageListener - virtual void notifyDiskMemUsage(DiskMemUsageState state) override; + void notifyDiskMemUsage(DiskMemUsageState state) override; // Implements IClusterStateChangedNofifier - virtual void notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) override; + void notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) override; // Implements IMaintenanceJob - virtual bool run() override; + bool run() override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp index d94bb2e3d03..d4b542a0af8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp @@ -105,7 +105,7 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller, AttributeUsageFilter &attributeUsageFilter) { controller.registerJobInMasterThread(std::make_unique(hbHandler, config.getHeartBeatConfig())); controller.registerJobInDefaultPool(std::make_unique(scPruner, config.getSessionCachePruneInterval())); - if (config.hasVisibilityDelay()) { + if (config.hasVisibilityDelay() && config.allowEarlyAck()) { controller.registerJobInMasterThread(std::make_unique(commit, config.getVisibilityDelay())); } const MaintenanceDocumentSubDB &mRemSubDB(controller.getRemSubDB()); diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index 217a3bb24d3..9927a2d2ce0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -210,11 +210,11 @@ moveMetaData(documentmetastore::IStore &meta_store, const DocumentId & doc_id, c } std::unique_ptr -createUncommitedLidTracker(bool needImmediateCommit) { - if (needImmediateCommit) { - return std::make_unique(); - } else { +createUncommitedLidTracker(bool allowEarlyAck) { + if (allowEarlyAck) { return std::make_unique(); + } else { + return std::make_unique(); } } @@ -229,7 +229,7 @@ StoreOnlyFeedView::StoreOnlyFeedView(const Context &ctx, const PersistentParams _docType(nullptr), _lidReuseDelayer(ctx._writeService, _documentMetaStoreContext->get(), ctx._lidReuseDelayerConfig), _pendingLidsForDocStore(), - _pendingLidsForCommit(createUncommitedLidTracker(_lidReuseDelayer.getImmediateCommit())), + _pendingLidsForCommit(createUncommitedLidTracker(_lidReuseDelayer.allowEarlyAck())), _schema(ctx._schema), _writeService(ctx._writeService), _params(params), @@ -275,7 +275,7 @@ StoreOnlyFeedView::internalForceCommit(SerialNum serialNum, OnForceCommitDoneTyp void StoreOnlyFeedView::considerEarlyAck(FeedToken & token) { - if ( ! needCommit() && token) { + if ( _lidReuseDelayer.allowEarlyAck() && token) { token.reset(); } } @@ -327,7 +327,7 @@ StoreOnlyFeedView::internalPut(FeedToken token, const PutOperation &putOp) bool docAlreadyExists = putOp.getValidPrevDbdId(_params._subDbId); if (putOp.getValidDbdId(_params._subDbId)) { - bool immediateCommit = needCommit(); + bool immediateCommit = needImmediateCommit(); const document::GlobalId &gid = docId.getGlobalId(); std::shared_ptr onWriteDone = createPutDoneContext(std::move(token), std::move(uncommitted), @@ -345,8 +345,8 @@ StoreOnlyFeedView::internalPut(FeedToken token, const PutOperation &putOp) } bool -StoreOnlyFeedView::needCommit() const { - return _lidReuseDelayer.getImmediateCommit(); +StoreOnlyFeedView::needImmediateCommit() const { + return _lidReuseDelayer.needImmediateCommit(); } void @@ -483,7 +483,7 @@ StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp) auto uncommitted = _pendingLidsForCommit->produce(updOp.getLid()); considerEarlyAck(token); - bool immediateCommit = needCommit(); + bool immediateCommit = needImmediateCommit(); auto onWriteDone = createUpdateDoneContext(std::move(token), std::move(uncommitted), updOp.getUpdate()); UpdateScope updateScope(*_schema, upd); updateAttributes(serialNum, lid, upd, immediateCommit, onWriteDone, updateScope); @@ -657,7 +657,7 @@ StoreOnlyFeedView::internalRemove(FeedToken token, IPendingLidTracker::Token unc std::move(pendingNotifyRemoveDone), (explicitReuseLid ? lid : 0u), std::move(moveDoneCtx)); removeSummary(serialNum, lid, onWriteDone); - bool immediateCommit = needCommit(); + bool immediateCommit = needImmediateCommit(); removeAttributes(serialNum, lid, immediateCommit, onWriteDone); removeIndexedFields(serialNum, lid, immediateCommit, onWriteDone); } @@ -770,7 +770,7 @@ StoreOnlyFeedView::handleDeleteBucket(const DeleteBucketOperation &delOp) void StoreOnlyFeedView::internalDeleteBucket(const DeleteBucketOperation &delOp) { - bool immediateCommit = needCommit(); + bool immediateCommit = needImmediateCommit(); size_t rm_count = removeDocuments(delOp, true, immediateCommit); LOG(debug, "internalDeleteBucket(): docType(%s), bucket(%s), lidsToRemove(%zu)", _params._docTypeName.toString().c_str(), delOp.getBucketId().toString().c_str(), rm_count); @@ -809,7 +809,7 @@ StoreOnlyFeedView::handleMove(const MoveOperation &moveOp, IDestructorCallback:: PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(moveOp, docId.getGlobalId(), docId); bool docAlreadyExists = moveOp.getValidPrevDbdId(_params._subDbId); if (moveOp.getValidDbdId(_params._subDbId)) { - bool immediateCommit = needCommit(); + bool immediateCommit = needImmediateCommit(); const document::GlobalId &gid = docId.getGlobalId(); std::shared_ptr onWriteDone = createPutDoneContext(FeedToken(), _pendingLidsForCommit->produce(moveOp.getLid()), diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h index 7d91ea86a22..4569e01f9fd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h @@ -161,7 +161,7 @@ private: void putSummary(SerialNum serialNum, Lid lid, DocumentSP doc, OnOperationDoneType onDone); void removeSummary(SerialNum serialNum, Lid lid, OnWriteDoneType onDone); void heartBeatSummary(SerialNum serialNum); - bool needCommit() const; + bool needImmediateCommit() const; bool useDocumentStore(SerialNum replaySerialNum) const { diff --git a/searchlib/src/vespa/searchlib/config/translogserver.def b/searchlib/src/vespa/searchlib/config/translogserver.def index 540895b2404..defce8c3421 100644 --- a/searchlib/src/vespa/searchlib/config/translogserver.def +++ b/searchlib/src/vespa/searchlib/config/translogserver.def @@ -15,7 +15,7 @@ basedir string default="tmp" restart ## Use fsync after each commit. ## If not the below interval is used. -usefsync bool default=false restart +usefsync bool default=false ##Number of threads available for visiting/subscription. maxthreads int default=4 restart diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index 9e0f1a8a1aa..bd7feec0598 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -113,7 +113,12 @@ Domain::addPart(SerialNum partId, bool isLastPart) { } } -Domain::~Domain() { } +Domain::~Domain() { + MonitorGuard guard(_currentChunkMonitor); + guard.broadcast(); + commitChunk(grabCurrentChunk(guard), guard); + _singleCommitter->shutdown().sync(); +} DomainInfo Domain::getDomainInfo() const @@ -318,22 +323,73 @@ Domain::optionallyRotateFile(SerialNum serialNum) { return dp; } +void +Domain::append(const Packet & packet, Writer::DoneCallback onDone) { + vespalib::MonitorGuard guard(_currentChunkMonitor); + if (_lastSerial >= packet.range().from()) { + throw runtime_error(fmt("Incoming serial number(%" PRIu64 ") must be bigger than the last one (%" PRIu64 ").", + packet.range().from(), _lastSerial)); + } else { + _lastSerial = packet.range().to(); + } + _currentChunk->add(packet, std::move(onDone)); + commitIfFull(guard); +} + Domain::CommitResult Domain::startCommit(DoneCallback onDone) { - (void) onDone; + vespalib::MonitorGuard guard(_currentChunkMonitor); + if ( !_currentChunk->empty() ) { + auto completed = grabCurrentChunk(guard); + completed->setCommitDoneCallback(std::move(onDone)); + CommitResult result(completed->createCommitResult()); + commitChunk(std::move(completed), guard); + return result; + } return CommitResult(); } void -Domain::append(const Packet & packet, Writer::DoneCallback onDone) -{ - (void) onDone; +Domain::commitIfFull(const vespalib::MonitorGuard &guard) { + if (_currentChunk->sizeBytes() > _config.getChunkSizeLimit()) { + auto completed = std::move(_currentChunk); + _currentChunk = std::make_unique(_config.getChunkSizeLimit(), completed->stealCallbacks()); + commitChunk(std::move(completed), guard); + } +} + +std::unique_ptr +Domain::grabCurrentChunk(const vespalib::MonitorGuard & guard) { + assert(guard.monitors(_currentChunkMonitor)); + auto chunk = std::move(_currentChunk); + _currentChunk = createCommitChunk(_config); + return chunk; +} + +void +Domain::commitChunk(std::unique_ptr chunk, const vespalib::MonitorGuard & chunkOrderGuard) { + assert(chunkOrderGuard.monitors(_currentChunkMonitor)); + _singleCommitter->execute( makeLambdaTask([this, chunk = std::move(chunk)]() mutable { + doCommit(std::move(chunk)); + })); +} + +void +Domain::doCommit(std::unique_ptr chunk) { + const Packet & packet = chunk->getPacket(); + if (packet.empty()) return; + vespalib::nbostream_longlivedbuf is(packet.getHandle().data(), packet.getHandle().size()); Packet::Entry entry; entry.deserialize(is); DomainPart::SP dp = optionallyRotateFile(entry.serial()); dp->commit(entry.serial(), packet); + if (_config.getFSyncOnCommit()) { + dp->sync(); + } cleanSessions(); + LOG(debug, "Releasing %zu acks and %zu entries and %zu bytes.", + chunk->getNumCallBacks(), chunk->getPacket().size(), chunk->sizeBytes()); } bool diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h index 7e77e6ef0ef..041ec27cf23 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h @@ -56,6 +56,11 @@ public: uint64_t size() const; Domain & setConfig(const DomainConfig & cfg); private: + void commitIfFull(const vespalib::MonitorGuard & guard); + + std::unique_ptr grabCurrentChunk(const vespalib::MonitorGuard & guard); + void commitChunk(std::unique_ptr chunk, const vespalib::MonitorGuard & chunkOrderGuard); + void doCommit(std::unique_ptr chunk); SerialNum begin(const vespalib::LockGuard & guard) const; SerialNum end(const vespalib::LockGuard & guard) const; size_t byteSize(const vespalib::LockGuard & guard) const; diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp index 8855183226d..b7e02894e6b 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp @@ -252,7 +252,7 @@ DomainPart::buildPacketMapping(bool allowTruncate) DomainPart::DomainPart(const string & name, const string & baseDir, SerialNum s, Encoding encoding, uint8_t compressionLevel, const FileHeaderContext &fileHeaderContext, bool allowTruncate) - : _encoding(encoding.getCrc(), Encoding::Compression::none), //TODO We do not yet support compression + : _encoding(encoding), _compressionLevel(compressionLevel), _lock(), _fileLock(), @@ -396,16 +396,19 @@ DomainPart::commit(SerialNum firstSerial, const Packet &packet) if (_range.from() == 0) { _range.from(firstSerial); } + IChunk::UP chunk = IChunk::create(_encoding, _compressionLevel); for (size_t i(0); h.size() > 0; i++) { //LOG(spam, //"Pos(%d) Len(%d), Lim(%d), Remaining(%d)", //h.getPos(), h.getLength(), h.getLimit(), h.getRemaining()); - IChunk::UP chunk = IChunk::create(_encoding, _compressionLevel); Packet::Entry entry; entry.deserialize(h); if (_range.to() < entry.serial()) { chunk->add(entry); - write(*_transLog, *chunk); + if (_encoding.getCompression() == Encoding::Compression::none) { + write(*_transLog, *chunk); + chunk = IChunk::create(_encoding, _compressionLevel); + } _sz++; _range.to(entry.serial()); } else { @@ -413,6 +416,9 @@ DomainPart::commit(SerialNum firstSerial, const Packet &packet) entry.serial(), _range.to())); } } + if ( ! chunk->getEntries().empty()) { + write(*_transLog, *chunk); + } bool merged(false); LockGuard guard(_lock); diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index 7be3dd708a5..0c0c9186e12 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -572,8 +572,11 @@ TransLogServer::domainCommit(FRT_RPCRequest *req) Packet packet(params[1]._data._buf, params[1]._data._len); try { vespalib::Gate gate; - domain->append(packet, make_shared(gate)); - auto keep = domain->startCommit(make_shared()); + { + // Need to scope in order to drain out all the callbacks. + domain->append(packet, make_shared(gate)); + auto keep = domain->startCommit(make_shared()); + } gate.await(); ret.AddInt32(0); ret.AddString("ok"); -- cgit v1.2.3 From 8a0846a5cd4d4306abe46f61d544c473f30b4411 Mon Sep 17 00:00:00 2001 From: Håvard Pettersen Date: Thu, 24 Sep 2020 15:37:31 +0000 Subject: instruction benchmark --- eval/CMakeLists.txt | 1 + .../tests/tensor/instruction_benchmark/.gitignore | 1 + .../tensor/instruction_benchmark/CMakeLists.txt | 8 + .../instruction_benchmark.cpp | 320 +++++++++++++++++++++ 4 files changed, 330 insertions(+) create mode 100644 eval/src/tests/tensor/instruction_benchmark/.gitignore create mode 100644 eval/src/tests/tensor/instruction_benchmark/CMakeLists.txt create mode 100644 eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 66ee9916c83..33bd098975a 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -56,6 +56,7 @@ vespa_define_module( src/tests/tensor/direct_dense_tensor_builder src/tests/tensor/direct_sparse_tensor_builder src/tests/tensor/index_lookup_table + src/tests/tensor/instruction_benchmark src/tests/tensor/onnx_wrapper src/tests/tensor/packed_mappings src/tests/tensor/tensor_add_operation diff --git a/eval/src/tests/tensor/instruction_benchmark/.gitignore b/eval/src/tests/tensor/instruction_benchmark/.gitignore new file mode 100644 index 00000000000..31b087883e0 --- /dev/null +++ b/eval/src/tests/tensor/instruction_benchmark/.gitignore @@ -0,0 +1 @@ +/eval_instruction_benchmark_app diff --git a/eval/src/tests/tensor/instruction_benchmark/CMakeLists.txt b/eval/src/tests/tensor/instruction_benchmark/CMakeLists.txt new file mode 100644 index 00000000000..d2384eaf129 --- /dev/null +++ b/eval/src/tests/tensor/instruction_benchmark/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_instruction_benchmark_app TEST + SOURCES + instruction_benchmark.cpp + DEPENDS + vespaeval + GTest::GTest +) diff --git a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp new file mode 100644 index 00000000000..8bb227a7e85 --- /dev/null +++ b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp @@ -0,0 +1,320 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +// Microbenchmark exploring performance differences between +// interpreted function instructions. + +// This benchmark was initially written to measure the difference in +// performance between (old) instructions using the TensorEngine +// immediate API and (new) instructions using the Value API +// directly. Note that all previous optimizations for dense tensors +// are trivially transformed to use the Value API, and thus only the +// generic cases need to be compared. Specifically; we want to make +// sure join performance for sparse tensors with full dimensional +// overlap does not suffer too much. Also, we want to showcase an +// improvement in generic dense join and possibly also in sparse join +// with partial dimensional overlap. Benchmarks are done using float +// cells since this is what gives best overall performance in +// production. Also, we use the multiply operation since it is the +// most optimized operations across all implementations. When +// benchmarking different implementations against each other, a smoke +// test is performed by verifying that all implementations produce the +// same result. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::tensor; +using namespace vespalib::eval::instruction; +using vespalib::make_string_short::fmt; + +using Instruction = InterpretedFunction::Instruction; +using EvalSingle = InterpretedFunction::EvalSingle; + +template using CREF = std::reference_wrapper; + +//----------------------------------------------------------------------------- + +struct Impl { + virtual const vespalib::string &name() const = 0; + virtual Value::UP create_value(const TensorSpec &spec) const = 0; + virtual TensorSpec create_spec(const Value &value) const = 0; + virtual Instruction create_join(const ValueType &lhs, const ValueType &rhs, operation::op2_t function, Stash &stash) const = 0; + virtual const TensorEngine &engine() const { return SimpleTensorEngine::ref(); } // engine used by EvalSingle + virtual ~Impl() {} +}; + +struct ValueImpl : Impl { + vespalib::string my_name; + const ValueBuilderFactory &my_factory; + ValueImpl(const vespalib::string &name_in, const ValueBuilderFactory &factory) + : my_name(name_in), my_factory(factory) {} + const vespalib::string &name() const override { return my_name; } + Value::UP create_value(const TensorSpec &spec) const override { return value_from_spec(spec, my_factory); } + TensorSpec create_spec(const Value &value) const override { return spec_from_value(value); } + Instruction create_join(const ValueType &lhs, const ValueType &rhs, operation::op2_t function, Stash &stash) const override { + return GenericJoin::make_instruction(lhs, rhs, function, my_factory, stash); + } +}; + +struct EngineImpl : Impl { + vespalib::string my_name; + const TensorEngine &my_engine; + EngineImpl(const vespalib::string &name_in, const TensorEngine &engine_in) + : my_name(name_in), my_engine(engine_in) {} + const vespalib::string &name() const override { return my_name; } + Value::UP create_value(const TensorSpec &spec) const override { return my_engine.from_spec(spec); } + TensorSpec create_spec(const Value &value) const override { return my_engine.to_spec(value); } + Instruction create_join(const ValueType &lhs, const ValueType &rhs, operation::op2_t function, Stash &stash) const override { + // create a complete tensor function joining two parameters, but only compile the join instruction itself + const auto &lhs_node = tensor_function::inject(lhs, 0, stash); + const auto &rhs_node = tensor_function::inject(rhs, 1, stash); + const auto &join_node = tensor_function::join(lhs_node, rhs_node, function, stash); + return join_node.compile_self(my_engine, stash); + } + const TensorEngine &engine() const override { return my_engine; } +}; + +//----------------------------------------------------------------------------- + +EngineImpl simple_tensor_engine_impl(" [SimpleTensorEngine]", SimpleTensorEngine::ref()); +EngineImpl default_tensor_engine_impl("[DefaultTensorEngine]", DefaultTensorEngine::ref()); +ValueImpl simple_value_impl(" [SimpleValue]", SimpleValueBuilderFactory::get()); +ValueImpl packed_mixed_tensor_impl(" [PackedMixedTensor]", PackedMixedTensorBuilderFactory::get()); + +double budget = 5.0; +std::vector> impl_list = {simple_tensor_engine_impl, + default_tensor_engine_impl, + simple_value_impl, + packed_mixed_tensor_impl}; + +//----------------------------------------------------------------------------- + +struct EvalOp { + using UP = std::unique_ptr; + const Impl &impl; + std::vector values; + std::vector stack; + EvalSingle single; + EvalOp(const EvalOp &) = delete; + EvalOp &operator=(const EvalOp &) = delete; + EvalOp(Instruction op, const std::vector> &stack_spec, const Impl &impl_in) + : impl(impl_in), values(), stack(), single(impl.engine(), op) + { + for (const TensorSpec &spec: stack_spec) { + values.push_back(impl.create_value(spec)); + } + for (const auto &value: values) { + stack.push_back(*value.get()); + } + } + TensorSpec result() { return impl.create_spec(single.eval(stack)); } + double estimate_cost_us() { + auto actual = [&](){ single.eval(stack); }; + return BenchmarkTimer::benchmark(actual, budget) * 1000.0 * 1000.0; + } +}; + +//----------------------------------------------------------------------------- + +void benchmark(const vespalib::string &desc, const std::vector &list) { + fprintf(stderr, "--------------------------------------------------------\n"); + fprintf(stderr, "Benchmark Case: [%s]\n", desc.c_str()); + std::optional expect = std::nullopt; + for (const auto &eval: list) { + if (expect.has_value()) { + ASSERT_EQ(eval->result(), expect.value()); + } else { + expect = eval->result(); + } + } + for (const auto &eval: list) { + fprintf(stderr, " %s: %10.3f us\n", eval->impl.name().c_str(), eval->estimate_cost_us()); + } + fprintf(stderr, "--------------------------------------------------------\n"); +} + +//----------------------------------------------------------------------------- + +void benchmark_join(const vespalib::string &desc, const TensorSpec &lhs, + const TensorSpec &rhs, operation::op2_t function) +{ + Stash stash; + ValueType lhs_type = ValueType::from_spec(lhs.type()); + ValueType rhs_type = ValueType::from_spec(rhs.type()); + ValueType res_type = ValueType::join(lhs_type, rhs_type); + ASSERT_FALSE(lhs_type.is_error()); + ASSERT_FALSE(rhs_type.is_error()); + ASSERT_FALSE(res_type.is_error()); + std::vector list; + for (const Impl &impl: impl_list) { + auto op = impl.create_join(lhs_type, rhs_type, function, stash); + std::vector> stack_spec({lhs, rhs}); + list.push_back(std::make_unique(op, stack_spec, impl)); + } + benchmark(desc, list); +} + +//----------------------------------------------------------------------------- + +struct D { + vespalib::string name; + bool mapped; + size_t size; + size_t stride; + static D map(const vespalib::string &name_in, size_t size_in, size_t stride_in) { return D{name_in, true, size_in, stride_in}; } + static D idx(const vespalib::string &name_in, size_t size_in) { return D{name_in, false, size_in, 1}; } + operator ValueType::Dimension() const { + if (mapped) { + return ValueType::Dimension(name); + } else { + return ValueType::Dimension(name, size); + } + } + std::pair operator()(size_t idx) const { + if (mapped) { + return std::make_pair(name, TensorSpec::Label(fmt("label_%zu", idx))); + } else { + return std::make_pair(name, TensorSpec::Label(idx)); + } + } +}; + +TensorSpec make_vector(const D &d1, double seq) { + auto type = ValueType::tensor_type({d1}, ValueType::CellType::FLOAT); + TensorSpec spec(type.to_spec()); + for (size_t i = 0, idx1 = 0; i < d1.size; ++i, idx1 += d1.stride, seq += 1.0) { + spec.add({d1(idx1)}, seq); + } + return spec; +} + +TensorSpec make_cube(const D &d1, const D &d2, const D &d3, double seq) { + auto type = ValueType::tensor_type({d1, d2, d3}, ValueType::CellType::FLOAT); + TensorSpec spec(type.to_spec()); + for (size_t i = 0, idx1 = 0; i < d1.size; ++i, idx1 += d1.stride) { + for (size_t j = 0, idx2 = 0; j < d2.size; ++j, idx2 += d2.stride) { + for (size_t k = 0, idx3 = 0; k < d3.size; ++k, idx3 += d3.stride, seq += 1.0) { + spec.add({d1(idx1), d2(idx2), d3(idx3)}, seq); + } + } + } + return spec; +} + +//----------------------------------------------------------------------------- + +TEST(MakeInputTest, print_some_test_input) { + auto sparse = make_vector(D::map("x", 5, 3), 1.0); + auto dense = make_vector(D::idx("x", 5), 10.0); + auto mixed = make_cube(D::map("x", 3, 7), D::idx("y", 2), D::idx("z", 2), 100.0); + fprintf(stderr, "--------------------------------------------------------\n"); + fprintf(stderr, "sparse vector: %s\n", sparse.to_string().c_str()); + fprintf(stderr, "dense vector: %s\n", dense.to_string().c_str()); + fprintf(stderr, "mixed cube: %s\n", mixed.to_string().c_str()); + fprintf(stderr, "--------------------------------------------------------\n"); +} + +//----------------------------------------------------------------------------- + +TEST(NumberJoin, plain_op2) { + auto lhs = TensorSpec("double").add({}, 2.0); + auto rhs = TensorSpec("double").add({}, 3.0); + benchmark_join("simple numbers multiply", lhs, rhs, operation::Mul::f); +} + +//----------------------------------------------------------------------------- + +TEST(DenseJoin, small_vectors) { + auto lhs = make_vector(D::idx("x", 10), 1.0); + auto rhs = make_vector(D::idx("x", 10), 2.0); + benchmark_join("small dense vector multiply", lhs, rhs, operation::Mul::f); +} + +TEST(DenseJoin, full_overlap) { + auto lhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::idx("c", 16), 1.0); + auto rhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::idx("c", 16), 2.0); + benchmark_join("dense full overlap multiply", lhs, rhs, operation::Mul::f); +} + +TEST(DenseJoin, partial_overlap) { + auto lhs = make_cube(D::idx("a", 8), D::idx("c", 8), D::idx("d", 8), 1.0); + auto rhs = make_cube(D::idx("b", 8), D::idx("c", 8), D::idx("d", 8), 2.0); + benchmark_join("dense partial overlap multiply", lhs, rhs, operation::Mul::f); +} + +TEST(DenseJoin, no_overlap) { + auto lhs = make_cube(D::idx("a", 4), D::idx("e", 4), D::idx("f", 4), 1.0); + auto rhs = make_cube(D::idx("b", 4), D::idx("c", 4), D::idx("d", 4), 2.0); + benchmark_join("dense no overlap multiply", lhs, rhs, operation::Mul::f); +} + +//----------------------------------------------------------------------------- + +TEST(SparseJoin, small_vectors) { + auto lhs = make_vector(D::map("x", 10, 1), 1.0); + auto rhs = make_vector(D::map("x", 10, 2), 2.0); + benchmark_join("small sparse vector multiply", lhs, rhs, operation::Mul::f); +} + +TEST(SparseJoin, full_overlap) { + auto lhs = make_cube(D::map("a", 16, 1), D::map("b", 16, 1), D::map("c", 16, 1), 1.0); + auto rhs = make_cube(D::map("a", 16, 2), D::map("b", 16, 2), D::map("c", 16, 2), 2.0); + benchmark_join("sparse full overlap multiply", lhs, rhs, operation::Mul::f); +} + +TEST(SparseJoin, full_overlap_big_vs_small) { + auto lhs = make_cube(D::map("a", 16, 1), D::map("b", 16, 1), D::map("c", 16, 1), 1.0); + auto rhs = make_cube(D::map("a", 2, 1), D::map("b", 2, 1), D::map("c", 2, 1), 2.0); + benchmark_join("sparse full overlap big vs small multiply", lhs, rhs, operation::Mul::f); +} + +TEST(SparseJoin, partial_overlap) { + auto lhs = make_cube(D::map("a", 8, 1), D::map("c", 8, 1), D::map("d", 8, 1), 1.0); + auto rhs = make_cube(D::map("b", 8, 2), D::map("c", 8, 2), D::map("d", 8, 2), 2.0); + benchmark_join("sparse partial overlap multiply", lhs, rhs, operation::Mul::f); +} + +TEST(SparseJoin, no_overlap) { + auto lhs = make_cube(D::map("a", 4, 1), D::map("e", 4, 1), D::map("f", 4, 1), 1.0); + auto rhs = make_cube(D::map("b", 4, 1), D::map("c", 4, 1), D::map("d", 4, 1), 2.0); + benchmark_join("sparse no overlap multiply", lhs, rhs, operation::Mul::f); +} + +//----------------------------------------------------------------------------- + +TEST(MixedJoin, full_overlap) { + auto lhs = make_cube(D::map("a", 16, 1), D::map("b", 16, 1), D::idx("c", 16), 1.0); + auto rhs = make_cube(D::map("a", 16, 2), D::map("b", 16, 2), D::idx("c", 16), 2.0); + benchmark_join("mixed full overlap multiply", lhs, rhs, operation::Mul::f); +} + +TEST(MixedJoin, partial_sparse_overlap) { + auto lhs = make_cube(D::map("a", 8, 1), D::map("c", 8, 1), D::idx("d", 8), 1.0); + auto rhs = make_cube(D::map("b", 8, 2), D::map("c", 8, 2), D::idx("d", 8), 2.0); + benchmark_join("mixed partial sparse overlap multiply", lhs, rhs, operation::Mul::f); +} + +TEST(MixedJoin, no_overlap) { + auto lhs = make_cube(D::map("a", 4, 1), D::map("e", 4, 1), D::idx("f", 4), 1.0); + auto rhs = make_cube(D::map("b", 4, 1), D::map("c", 4, 1), D::idx("d", 4), 2.0); + benchmark_join("mixed no overlap multiply", lhs, rhs, operation::Mul::f); +} + +//----------------------------------------------------------------------------- + +GTEST_MAIN_RUN_ALL_TESTS() -- cgit v1.2.3 From bea49fe5bd3f4d02b003617dc8a7b4091ec3a632 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 24 Sep 2020 17:30:32 +0200 Subject: Add DocumentOperationParameters to document session API --- .../java/com/yahoo/documentapi/AsyncSession.java | 77 +++++++++++++++++++++- .../documentapi/DocumentOperationParameters.java | 66 +++++++++++++++++++ .../java/com/yahoo/documentapi/SyncSession.java | 55 ++++++++++++++++ .../messagebus/MessageBusAsyncSession.java | 55 +++++++++++++--- .../messagebus/MessageBusSyncSession.java | 51 ++++++++------ .../com/yahoo/messagebus/routing/RoutingSpec.java | 4 +- .../src/main/java/com/yahoo/feedapi/XMLFeeder.java | 2 +- 7 files changed, 274 insertions(+), 36 deletions(-) create mode 100644 documentapi/src/main/java/com/yahoo/documentapi/DocumentOperationParameters.java diff --git a/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java index 9f4ceaad37f..721faf281f7 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java @@ -7,6 +7,8 @@ import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentUpdate; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import static com.yahoo.documentapi.DocumentOperationParameters.parameters; + /** *

A session for asynchronous access to a document repository. * This class provides document repository writes and random access with high @@ -42,10 +44,11 @@ public interface AsyncSession extends Session { * If it was not a success, this method has no further effects.

* * @param document the Document to put + * @param priority the priority with which to send the operation * @return the synchronous result of this operation */ default Result put(Document document, DocumentProtocol.Priority priority) { - return put(new DocumentPut(document), priority); + return put(new DocumentPut(document), parameters().withPriority(priority)); } /** @@ -60,7 +63,7 @@ public interface AsyncSession extends Session { * @return the synchronous result of this operation */ default Result put(DocumentPut documentPut) { - return put(documentPut, DocumentProtocol.Priority.NORMAL_3); + return put(documentPut, parameters()); } /** @@ -72,10 +75,26 @@ public interface AsyncSession extends Session { * If it was not a success, this method has no further effects.

* * @param documentPut the DocumentPut to perform + * @param priority the priority with which to send the operation * @return the synchronous result of this operation */ - // TODO Vespa 8: Make this the one to implement. default Result put(DocumentPut documentPut, DocumentProtocol.Priority priority) { + return put(documentPut, parameters().withPriority(priority)); + } + + /** + *

Puts a document, with optional conditions on the operation. This method returns immediately.

+ * + *

If this result is a success, this + * call will cause one or more {@link DocumentResponse} objects to appear within the timeout time of this session. + * The response returned later will either be a success, or contain the document submitted here. + * If it was not a success, this method has no further effects.

+ * + * @param documentPut the DocumentPut to perform + * @param parameters parameters for the operation + * @return the synchronous result of this operation + */ + default Result put(DocumentPut documentPut, DocumentOperationParameters parameters) { return put(documentPut.getDocument()); } @@ -126,6 +145,23 @@ public interface AsyncSession extends Session { * @throws UnsupportedOperationException if this access implementation does not support retrieving */ default Result get(DocumentId id, DocumentProtocol.Priority priority) { + return get(id, parameters().withPriority(priority)); + } + + /** + *

Gets a document. This method returns immediately.

+ * + *

If this result is a success, this + * call will cause one or more {@link DocumentResponse} objects to appear within the timeout time of this session. + * The response returned later will contain the requested document if it is a success. + * If it was not a success, this method has no further effects.

+ * + * @param id the id of the document to get + * @param parameters parameters for the operation + * @return the synchronous result of this operation + * @throws UnsupportedOperationException if this access implementation does not support retrieving + */ + default Result get(DocumentId id, DocumentOperationParameters parameters) { return get(id); } @@ -158,6 +194,23 @@ public interface AsyncSession extends Session { * @throws UnsupportedOperationException if this access implementation does not support removal */ default Result remove(DocumentId id, DocumentProtocol.Priority priority) { + return remove(id, parameters().withPriority(priority)); + } + + /** + *

Removes a document if it is present. This method returns immediately.

+ * + *

If this result is a success, this + * call will cause one or more {@link DocumentIdResponse} objects to apprear within the timeout time of this session. + * The response returned later will either be a success, or contain the document id submitted here. + * If it was not a success, this method has no further effects.

+ * + * @param id the id of the document to remove + * @param parameters parameters for the operation + * @return the synchronous result of this operation + * @throws UnsupportedOperationException if this access implementation does not support removal + */ + default Result remove(DocumentId id, DocumentOperationParameters parameters) { return remove(id); } @@ -189,6 +242,23 @@ public interface AsyncSession extends Session { * @throws UnsupportedOperationException if this access implementation does not support update */ default Result update(DocumentUpdate update, DocumentProtocol.Priority priority) { + return update(update, parameters().withPriority(priority)); + } + + /** + *

Updates a document. This method returns immediately.

+ * + *

If this result is a success, this + * call will cause one or more {@link DocumentUpdateResponse} within the timeout time of this session. + * The returned response returned later will either be a success or contain the update submitted here. + * If it was not a success, this method has no further effects.

+ * + * @param update the updates to perform + * @param parameters parameters for the operation + * @return the synchronous result of this operation + * @throws UnsupportedOperationException if this access implementation does not support update + */ + default Result update(DocumentUpdate update, DocumentOperationParameters parameters) { return update(update); } @@ -199,4 +269,5 @@ public interface AsyncSession extends Session { */ double getCurrentWindowSize(); + } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentOperationParameters.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentOperationParameters.java new file mode 100644 index 00000000000..cfb134ffa4b --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentOperationParameters.java @@ -0,0 +1,66 @@ +package com.yahoo.documentapi; + +import com.yahoo.document.fieldset.FieldSet; +import com.yahoo.document.fieldset.FieldSetRepo; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; + +import java.util.Optional; +import java.util.OptionalInt; + +import static java.util.Objects.requireNonNull; + +/** Optional parameters for a document operation. Immutable class. */ +public class DocumentOperationParameters { + + private static final DocumentOperationParameters empty = new DocumentOperationParameters(null, null, null, -1); + + private final DocumentProtocol.Priority priority; + private final String fieldSet; + private final String route; + private final int traceLevel; + + private DocumentOperationParameters(DocumentProtocol.Priority priority, String fieldSet, String route, int traceLevel) { + this.priority = priority; + this.fieldSet = fieldSet; + this.route = route; + this.traceLevel = traceLevel; + } + + public static DocumentOperationParameters parameters() { + return empty; + } + + /** Sets the priority with which to perform an operation. */ + public DocumentOperationParameters withPriority(DocumentProtocol.Priority priority) { + return new DocumentOperationParameters(requireNonNull(priority), fieldSet, route, traceLevel); + } + + /** Sets the field set used for retrieval. */ + public DocumentOperationParameters withFieldSet(FieldSet fieldSet) { + return new DocumentOperationParameters(priority, new FieldSetRepo().serialize(fieldSet), route, traceLevel); + } + + /** Sets the field set used for retrieval. */ + public DocumentOperationParameters withFieldSet(String fieldSet) { + return new DocumentOperationParameters(priority, requireNonNull(fieldSet), route, traceLevel); + } + + /** Sets the route along which to send the operation. */ + public DocumentOperationParameters withRoute(String route) { + return new DocumentOperationParameters(priority, fieldSet, requireNonNull(route), traceLevel); + } + + /** Sets the trace level for an operation. */ + public DocumentOperationParameters withTraceLevel(int traceLevel) { + if (traceLevel < 0 || traceLevel > 9) + throw new IllegalArgumentException("Trace level must be from 0 (no tracing) to 9 (maximum)"); + + return new DocumentOperationParameters(priority, fieldSet, route, traceLevel); + } + + public Optional priority() { return Optional.ofNullable(priority); } + public Optional fieldSet() { return Optional.ofNullable(fieldSet); } + public Optional route() { return Optional.ofNullable(route); } + public OptionalInt traceLevel() { return traceLevel >= 0 ? OptionalInt.of(traceLevel) : OptionalInt.empty(); } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java index 24fd47ed12c..1cee3249032 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java @@ -11,6 +11,8 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import java.time.Duration; +import static com.yahoo.documentapi.DocumentOperationParameters.parameters; + /** * A session for synchronous access to a document repository. This class * provides simple document access where throughput is not a concern. @@ -35,6 +37,18 @@ public interface SyncSession extends Session { * @param priority the priority with which to perform this operation */ default void put(DocumentPut documentPut, DocumentProtocol.Priority priority) { + put(documentPut, parameters().withPriority(priority)); + } + + /** + * Puts a document. When this method returns, the document is safely received. + * + * @param documentPut the DocumentPut operation + * @param parameters parameters for the operation + * + * @param documentPut the DocumentPut operation + */ + default void put(DocumentPut documentPut, DocumentOperationParameters parameters) { put(documentPut); } @@ -84,6 +98,20 @@ public interface SyncSession extends Session { */ Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority priority, Duration timeout); + /** + * Gets a document with timeout. + * + * @param id the id of the document to get + * @param parameters parameters for the operation + * @param timeout timeout. If timeout is null, an unspecified default will be used + * @return the known document having this id, or null if there is no document having this id + * @throws UnsupportedOperationException thrown if this access does not support retrieving + * @throws DocumentAccessException on any messagebus error, including timeout ({@link com.yahoo.messagebus.ErrorCode#TIMEOUT}) + */ + default Document get(DocumentId id, DocumentOperationParameters parameters, Duration timeout) { + return get(id, timeout); + } + /** * Removes a document if it is present and condition is fulfilled. * @@ -102,6 +130,18 @@ public interface SyncSession extends Session { */ boolean remove(DocumentRemove documentRemove, DocumentProtocol.Priority priority); + /** + * Removes a document if it is present. + * + * @param documentRemove document remove operation + * @param parameters parameters for the operation + * @return true if the document with this id was removed, false otherwise. + * @throws UnsupportedOperationException thrown if this access does not support removal + */ + default boolean remove(DocumentRemove documentRemove, DocumentOperationParameters parameters) { + return remove(documentRemove); + } + /** * Updates a document. * @@ -127,4 +167,19 @@ public interface SyncSession extends Session { */ boolean update(DocumentUpdate update, DocumentProtocol.Priority priority); + /** + * Updates a document. + * + * @param update the updates to perform. + * @param parameters parameters for the operation + * @return false if the updates could not be applied as the document does not exist and + * {@link DocumentUpdate#setCreateIfNonExistent(boolean) create-if-non-existent} is not set. + * @throws DocumentAccessException on update error, including but not limited to: 1. timeouts, + * 2. the document exists but the {@link DocumentUpdate#setCondition(TestAndSetCondition) condition} + * is not met. + */ + default boolean update(DocumentUpdate update, DocumentOperationParameters parameters) { + return update(update); + } + } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java index 7471d285db1..98e7aec7b11 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java @@ -9,6 +9,7 @@ import com.yahoo.document.fieldset.AllFields; import com.yahoo.documentapi.AsyncParameters; import com.yahoo.documentapi.AsyncSession; import com.yahoo.documentapi.DocumentIdResponse; +import com.yahoo.documentapi.DocumentOperationParameters; import com.yahoo.documentapi.DocumentResponse; import com.yahoo.documentapi.DocumentUpdateResponse; import com.yahoo.documentapi.RemoveResponse; @@ -16,6 +17,7 @@ import com.yahoo.documentapi.Response; import com.yahoo.documentapi.ResponseHandler; import com.yahoo.documentapi.Result; import com.yahoo.documentapi.UpdateResponse; +import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; @@ -33,6 +35,7 @@ import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.SourceSession; import com.yahoo.messagebus.StaticThrottlePolicy; import com.yahoo.messagebus.ThrottlePolicy; +import com.yahoo.messagebus.routing.Route; import java.util.Queue; import java.util.concurrent.BlockingQueue; @@ -41,6 +44,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Logger; +import static com.yahoo.documentapi.DocumentOperationParameters.parameters; import static com.yahoo.documentapi.Response.Outcome.CONDITION_FAILED; import static com.yahoo.documentapi.Response.Outcome.ERROR; import static com.yahoo.documentapi.Response.Outcome.NOT_FOUND; @@ -96,19 +100,24 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { @Override public Result put(Document document) { - return put(new DocumentPut(document), DocumentProtocol.Priority.NORMAL_3); + return put(new DocumentPut(document), parameters()); } @Override public Result put(DocumentPut documentPut, DocumentProtocol.Priority pri) { + return put(documentPut, parameters().withPriority(pri)); + } + + @Override + public Result put(DocumentPut documentPut, DocumentOperationParameters parameters) { PutDocumentMessage msg = new PutDocumentMessage(documentPut); - msg.setPriority(pri); + setParameters(msg, parameters, DocumentProtocol.Priority.NORMAL_3); return send(msg); } @Override public Result get(DocumentId id) { - return get(id, DocumentProtocol.Priority.NORMAL_1); + return get(id, parameters()); } @Override @@ -119,35 +128,52 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { @Override public Result get(DocumentId id, DocumentProtocol.Priority pri) { - GetDocumentMessage msg = new GetDocumentMessage(id, AllFields.NAME); - msg.setPriority(pri); + return get(id, parameters().withPriority(pri)); + } + + @Override + public Result get(DocumentId id, DocumentOperationParameters parameters) { + GetDocumentMessage msg = new GetDocumentMessage(id, parameters.fieldSet().orElse(AllFields.NAME)); + setParameters(msg, parameters, DocumentProtocol.Priority.NORMAL_1); return send(msg); } @Override public Result remove(DocumentId id) { - return remove(id, DocumentProtocol.Priority.NORMAL_2); + return remove(id, parameters()); } @Override public Result remove(DocumentId id, DocumentProtocol.Priority pri) { + return remove(id, parameters().withPriority(pri)); + } + + @Override + public Result remove(DocumentId id, DocumentOperationParameters parameters) { RemoveDocumentMessage msg = new RemoveDocumentMessage(id); - msg.setPriority(pri); + setParameters(msg, parameters, DocumentProtocol.Priority.NORMAL_2); return send(msg); } @Override public Result update(DocumentUpdate update) { - return update(update, DocumentProtocol.Priority.NORMAL_2); + return update(update, parameters()); } @Override public Result update(DocumentUpdate update, DocumentProtocol.Priority pri) { + return update(update, parameters().withPriority(pri)); + } + + @Override + public Result update(DocumentUpdate update, DocumentOperationParameters parameters) { UpdateDocumentMessage msg = new UpdateDocumentMessage(update); - msg.setPriority(pri); + setParameters(msg, parameters, DocumentProtocol.Priority.NORMAL_2); return send(msg); } + // TODO jonmv: Was this done to remedy get route no longer being possible to set through doc/v1 after default-get was added? + // TODO jonmv: If so, this is no longer needed with doc/v1.1 and later. private boolean mayOverrideWithGetOnlyRoute(Message msg) { // Only allow implicitly overriding the default Get route if the message is attempted sent // with the default route originally. Otherwise it's reasonable to assume that the caller @@ -168,7 +194,9 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { long reqId = requestId.incrementAndGet(); msg.setContext(reqId); msg.getTrace().setLevel(traceLevel); - String toRoute = (mayOverrideWithGetOnlyRoute(msg) ? routeForGet : route); + // Use message route if set, or session route if non-default, or finally, defaults for get and non-get, if set. Phew! + String toRoute = msg.getRoute() != null ? msg.getRoute().toString() + : (mayOverrideWithGetOnlyRoute(msg) ? routeForGet : route); if (toRoute != null) { return toResult(reqId, session.send(msg, toRoute, true)); } else { @@ -336,4 +364,11 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { } } + static void setParameters(DocumentMessage message, DocumentOperationParameters parameters, + DocumentProtocol.Priority defaultPriority) { + message.setPriority(parameters.priority().orElse(defaultPriority)); + parameters.route().map(Route::parse).ifPresent(message::setRoute); + parameters.traceLevel().ifPresent(message.getTrace()::setLevel); + } + } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java index a9621950b88..7bf0976f249 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java @@ -9,6 +9,7 @@ import com.yahoo.document.DocumentUpdate; import com.yahoo.document.fieldset.AllFields; import com.yahoo.documentapi.AsyncParameters; import com.yahoo.documentapi.DocumentAccessException; +import com.yahoo.documentapi.DocumentOperationParameters; import com.yahoo.documentapi.Response; import com.yahoo.documentapi.Result; import com.yahoo.documentapi.SyncParameters; @@ -25,9 +26,13 @@ import com.yahoo.messagebus.Message; import com.yahoo.messagebus.MessageBus; import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.ReplyHandler; +import com.yahoo.messagebus.routing.Route; import java.time.Duration; +import static com.yahoo.documentapi.DocumentOperationParameters.parameters; +import static com.yahoo.documentapi.messagebus.MessageBusAsyncSession.setParameters; + /** * An implementation of the SyncSession interface running over message bus. * @@ -114,35 +119,35 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re @Override public void put(DocumentPut documentPut) { - put(documentPut, DocumentProtocol.Priority.NORMAL_3); + put(documentPut, parameters()); } @Override public void put(DocumentPut documentPut, DocumentProtocol.Priority priority) { - PutDocumentMessage msg = new PutDocumentMessage(documentPut); - msg.setPriority(priority); - syncSendPutDocumentMessage(msg); + put(documentPut, parameters().withPriority(priority)); } @Override - public Document get(DocumentId id) { - return get(id, null); + public void put(DocumentPut documentPut, DocumentOperationParameters parameters) { + PutDocumentMessage msg = new PutDocumentMessage(documentPut); + setParameters(msg, parameters, DocumentProtocol.Priority.NORMAL_3); + syncSendPutDocumentMessage(msg); } @Override - public Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority pri) { - return get(id, fieldSet, pri, null); + public Document get(DocumentId id, Duration timeout) { + return get(id, parameters(), timeout); } @Override - public Document get(DocumentId id, Duration timeout) { - return get(id, AllFields.NAME, DocumentProtocol.Priority.NORMAL_1, timeout); + public Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority pri, Duration timeout) { + return get(id, parameters().withFieldSet(fieldSet).withPriority(pri), timeout); } @Override - public Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority pri, Duration timeout) { - GetDocumentMessage msg = new GetDocumentMessage(id, fieldSet); - msg.setPriority(pri); + public Document get(DocumentId id, DocumentOperationParameters parameters, Duration timeout) { + GetDocumentMessage msg = new GetDocumentMessage(id, parameters.fieldSet().orElse(AllFields.NAME)); + setParameters(msg, parameters, DocumentProtocol.Priority.NORMAL_1); Reply reply = syncSend(msg, timeout != null ? timeout : defaultTimeout); if (reply.hasErrors()) { @@ -161,15 +166,18 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re @Override public boolean remove(DocumentRemove documentRemove) { - RemoveDocumentMessage msg = new RemoveDocumentMessage(documentRemove.getId()); - msg.setCondition(documentRemove.getCondition()); - return remove(msg); + return remove(documentRemove, parameters()); } @Override public boolean remove(DocumentRemove documentRemove, DocumentProtocol.Priority pri) { + return remove(documentRemove, parameters().withPriority(pri)); + } + + @Override + public boolean remove(DocumentRemove documentRemove, DocumentOperationParameters parameters) { RemoveDocumentMessage msg = new RemoveDocumentMessage(documentRemove.getId()); - msg.setPriority(pri); + setParameters(msg, parameters, DocumentProtocol.Priority.NORMAL_2); msg.setCondition(documentRemove.getCondition()); return remove(msg); } @@ -187,13 +195,18 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re @Override public boolean update(DocumentUpdate update) { - return update(update, DocumentProtocol.Priority.NORMAL_2); + return update(update, parameters()); } @Override public boolean update(DocumentUpdate update, DocumentProtocol.Priority pri) { + return update(update, parameters().withPriority(pri)); + } + + @Override + public boolean update(DocumentUpdate update, DocumentOperationParameters parameters) { UpdateDocumentMessage msg = new UpdateDocumentMessage(update); - msg.setPriority(pri); + setParameters(msg, parameters, DocumentProtocol.Priority.NORMAL_2); Reply reply = syncSend(msg); if (reply.hasErrors()) { throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply), reply.getErrorCodes()); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingSpec.java b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingSpec.java index 032f13a008a..b5cbfd8224e 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingSpec.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingSpec.java @@ -195,7 +195,6 @@ public class RoutingSpec { return ret.toString(); } - // Overrides Object. @Override public String toString() { StringBuilder ret = new StringBuilder(); @@ -203,7 +202,6 @@ public class RoutingSpec { return ret.toString(); } - // Overrides Object. @Override public boolean equals(Object obj) { if (!(obj instanceof RoutingSpec)) { @@ -218,7 +216,7 @@ public class RoutingSpec { @Override public int hashCode() { - int result = tables != null ? tables.hashCode() : 0; + int result = tables.hashCode(); result = 31 * result + (verify ? 1 : 0); return result; } diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/XMLFeeder.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/XMLFeeder.java index 85b714b3b42..9bd2d66fa91 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/XMLFeeder.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/XMLFeeder.java @@ -12,7 +12,7 @@ import java.io.InputStream; * access point. * * @author Thomas Gundersen - * @author steinar + * @author Steinar Knutsen */ public class XMLFeeder extends Feeder { public XMLFeeder(DocumentTypeManager docMan, SimpleFeedAccess sender, InputStream stream) { -- cgit v1.2.3 From ba878c027385ac2db61dd849d39705c704786f62 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 24 Sep 2020 21:18:25 +0200 Subject: Use trace level from parameters, and pass on to Response --- .../com/yahoo/documentapi/DocumentIdResponse.java | 15 +++- .../com/yahoo/documentapi/DocumentResponse.java | 15 +++- .../yahoo/documentapi/DocumentUpdateResponse.java | 15 +++- .../java/com/yahoo/documentapi/RemoveResponse.java | 8 ++- .../main/java/com/yahoo/documentapi/Response.java | 17 +++++ .../java/com/yahoo/documentapi/UpdateResponse.java | 8 ++- .../messagebus/MessageBusAsyncSession.java | 81 ++++++++++------------ .../messagebus/MessageBusSyncSession.java | 41 +++++------ .../yahoo/document/restapi/resource/RestApi.java | 1 + 9 files changed, 130 insertions(+), 71 deletions(-) diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentIdResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentIdResponse.java index 34ab47571cf..e4a44fb88cd 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentIdResponse.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentIdResponse.java @@ -2,6 +2,7 @@ package com.yahoo.documentapi; import com.yahoo.document.DocumentId; +import com.yahoo.messagebus.Trace; /** * The asynchronous response to a document remove operation. @@ -61,7 +62,19 @@ public class DocumentIdResponse extends Response { * @param outcome the outcome of the operation */ public DocumentIdResponse(long requestId, DocumentId documentId, String textMessage, Outcome outcome) { - super(requestId, textMessage, outcome); + this(requestId, documentId, textMessage, outcome, null); + } + + + /** + * Creates a response containing a textual message and/or a document id + * + * @param documentId the DocumentId to encapsulate in the Response + * @param textMessage the message to encapsulate in the Response + * @param outcome the outcome of the operation + */ + public DocumentIdResponse(long requestId, DocumentId documentId, String textMessage, Outcome outcome, Trace trace) { + super(requestId, textMessage, outcome, null); this.documentId = documentId; } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java index 172e5fd11c0..a4d81a5e145 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java @@ -2,6 +2,7 @@ package com.yahoo.documentapi; import com.yahoo.document.Document; +import com.yahoo.messagebus.Trace; /** * The asynchronous response to a document put or get operation. @@ -72,7 +73,19 @@ public class DocumentResponse extends Response { * @param outcome the outcome of this operation */ public DocumentResponse(long requestId, Document document, String textMessage, Outcome outcome) { - super(requestId, textMessage, outcome); + this(requestId, document, textMessage, outcome, null); + } + + + /** + * Creates a response containing a textual message and/or a document + * + * @param document the Document to encapsulate in the Response + * @param textMessage the message to encapsulate in the Response + * @param outcome the outcome of this operation + */ + public DocumentResponse(long requestId, Document document, String textMessage, Outcome outcome, Trace trace) { + super(requestId, textMessage, outcome, trace); this.document = document; } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentUpdateResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentUpdateResponse.java index 3294c216d96..d34873aeaa6 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentUpdateResponse.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentUpdateResponse.java @@ -2,6 +2,7 @@ package com.yahoo.documentapi; import com.yahoo.document.DocumentUpdate; +import com.yahoo.messagebus.Trace; /** * The asynchronous response to a document update operation. @@ -71,7 +72,19 @@ public class DocumentUpdateResponse extends Response { * @param outcome the outcome of this operation */ public DocumentUpdateResponse(long requestId, DocumentUpdate documentUpdate, String textMessage, Outcome outcome) { - super(requestId, textMessage, outcome); + this(requestId, documentUpdate, textMessage, outcome, null); + } + + + /** + * Creates a response containing a textual message and/or a document update + * + * @param documentUpdate the DocumentUpdate to encapsulate in the Response + * @param textMessage the message to encapsulate in the Response + * @param outcome the outcome of this operation + */ + public DocumentUpdateResponse(long requestId, DocumentUpdate documentUpdate, String textMessage, Outcome outcome, Trace trace) { + super(requestId, textMessage, outcome, trace); this.documentUpdate = documentUpdate; } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/RemoveResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/RemoveResponse.java index 2a7c6f45d95..2d3f2934890 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/RemoveResponse.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/RemoveResponse.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi; +import com.yahoo.messagebus.Trace; + /** * This response is provided for successful document remove operations. Use the * wasFound() method to check whether or not the document was actually found. @@ -12,7 +14,11 @@ public class RemoveResponse extends Response { private final boolean wasFound; public RemoveResponse(long requestId, boolean wasFound) { - super(requestId, null, wasFound ? Outcome.SUCCESS : Outcome.NOT_FOUND); + this(requestId, wasFound, null); + } + + public RemoveResponse(long requestId, boolean wasFound, Trace trace) { + super(requestId, null, wasFound ? Outcome.SUCCESS : Outcome.NOT_FOUND, trace); this.wasFound = wasFound; } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/Response.java b/documentapi/src/main/java/com/yahoo/documentapi/Response.java index 4c95a648949..0a541a92c6a 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/Response.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/Response.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi; +import com.yahoo.messagebus.Trace; + import java.util.Objects; import static com.yahoo.documentapi.Response.Outcome.ERROR; @@ -19,6 +21,7 @@ public class Response { private final long requestId; private final String textMessage; private final Outcome outcome; + private final Trace trace; /** Creates a successful response containing no information */ public Response(long requestId) { @@ -52,9 +55,20 @@ public class Response { * @param outcome the outcome of the operation */ public Response(long requestId, String textMessage, Outcome outcome) { + this(requestId, textMessage, outcome, null); + } + + /** + * Creates a response containing a textual message + * + * @param textMessage the message to encapsulate in the Response + * @param outcome the outcome of the operation + */ + public Response(long requestId, String textMessage, Outcome outcome, Trace trace) { this.requestId = requestId; this.textMessage = textMessage; this.outcome = outcome; + this.trace = trace; } /** @@ -76,6 +90,9 @@ public class Response { public long getRequestId() { return requestId; } + /** Returns the trace of this operation, or null if there is none. */ + public Trace getTrace() { return trace; } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/UpdateResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/UpdateResponse.java index aca34a92a30..54c338dc6c9 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/UpdateResponse.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/UpdateResponse.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi; +import com.yahoo.messagebus.Trace; + /** * This response is provided for successful document update operations. Use the * wasFound() method to check whether or not the document was actually found. @@ -12,7 +14,11 @@ public class UpdateResponse extends Response { private final boolean wasFound; public UpdateResponse(long requestId, boolean wasFound) { - super(requestId, null, wasFound ? Outcome.SUCCESS : Outcome.NOT_FOUND); + this(requestId, wasFound, null); + } + + public UpdateResponse(long requestId, boolean wasFound, Trace trace) { + super(requestId, null, wasFound ? Outcome.SUCCESS : Outcome.NOT_FOUND, trace); this.wasFound = wasFound; } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java index 98e7aec7b11..d1da3c7dc03 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java @@ -17,7 +17,6 @@ import com.yahoo.documentapi.Response; import com.yahoo.documentapi.ResponseHandler; import com.yahoo.documentapi.Result; import com.yahoo.documentapi.UpdateResponse; -import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; @@ -35,7 +34,6 @@ import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.SourceSession; import com.yahoo.messagebus.StaticThrottlePolicy; import com.yahoo.messagebus.ThrottlePolicy; -import com.yahoo.messagebus.routing.Route; import java.util.Queue; import java.util.concurrent.BlockingQueue; @@ -48,6 +46,7 @@ import static com.yahoo.documentapi.DocumentOperationParameters.parameters; import static com.yahoo.documentapi.Response.Outcome.CONDITION_FAILED; import static com.yahoo.documentapi.Response.Outcome.ERROR; import static com.yahoo.documentapi.Response.Outcome.NOT_FOUND; +import static com.yahoo.documentapi.Response.Outcome.SUCCESS; /** * An access session which wraps a messagebus source session sending document messages. @@ -111,8 +110,8 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { @Override public Result put(DocumentPut documentPut, DocumentOperationParameters parameters) { PutDocumentMessage msg = new PutDocumentMessage(documentPut); - setParameters(msg, parameters, DocumentProtocol.Priority.NORMAL_3); - return send(msg); + msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_3)); + return send(msg, parameters); } @Override @@ -134,8 +133,8 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { @Override public Result get(DocumentId id, DocumentOperationParameters parameters) { GetDocumentMessage msg = new GetDocumentMessage(id, parameters.fieldSet().orElse(AllFields.NAME)); - setParameters(msg, parameters, DocumentProtocol.Priority.NORMAL_1); - return send(msg); + msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_1)); + return send(msg, parameters); } @Override @@ -151,8 +150,8 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { @Override public Result remove(DocumentId id, DocumentOperationParameters parameters) { RemoveDocumentMessage msg = new RemoveDocumentMessage(id); - setParameters(msg, parameters, DocumentProtocol.Priority.NORMAL_2); - return send(msg); + msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_2)); + return send(msg, parameters); } @Override @@ -168,8 +167,8 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { @Override public Result update(DocumentUpdate update, DocumentOperationParameters parameters) { UpdateDocumentMessage msg = new UpdateDocumentMessage(update); - setParameters(msg, parameters, DocumentProtocol.Priority.NORMAL_2); - return send(msg); + msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_2)); + return send(msg, parameters); } // TODO jonmv: Was this done to remedy get route no longer being possible to set through doc/v1 after default-get was added? @@ -182,21 +181,13 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { && ("default".equals(route) || "route:default".equals(route))); } - /** - * A convenience method for assigning the internal trace level and route string to a message before sending it - * through the internal mbus session object. - * - * @param msg the message to send. - * @return the document api result object. - */ - public Result send(Message msg) { + Result send(Message msg, DocumentOperationParameters parameters) { try { long reqId = requestId.incrementAndGet(); msg.setContext(reqId); - msg.getTrace().setLevel(traceLevel); - // Use message route if set, or session route if non-default, or finally, defaults for get and non-get, if set. Phew! - String toRoute = msg.getRoute() != null ? msg.getRoute().toString() - : (mayOverrideWithGetOnlyRoute(msg) ? routeForGet : route); + msg.getTrace().setLevel(parameters.traceLevel().orElse(traceLevel)); + // Use route from parameters, or session route if non-default, or finally, defaults for get and non-get, if set. Phew! + String toRoute = parameters.route().orElse(mayOverrideWithGetOnlyRoute(msg) ? routeForGet : route); if (toRoute != null) { return toResult(reqId, session.send(msg, toRoute, true)); } else { @@ -207,6 +198,17 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { } } + /** + * A convenience method for assigning the internal trace level and route string to a message before sending it + * through the internal mbus session object. + * + * @param msg the message to send. + * @return the document api result object. + */ + public Result send(Message msg) { + return send(msg, null); + } + @Override public Response getNext() { return responses.poll(); @@ -284,7 +286,7 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { } private static Response toResponse(Reply reply) { - long reqId = (Long)reply.getContext(); + long reqId = (Long) reply.getContext(); return reply.hasErrors() ? toError(reply, reqId) : toSuccess(reply, reqId); } @@ -297,15 +299,15 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { String err = getErrorMessage(reply); switch (msg.getType()) { case DocumentProtocol.MESSAGE_PUTDOCUMENT: - return new DocumentResponse(reqId, ((PutDocumentMessage)msg).getDocumentPut().getDocument(), err, outcome); + return new DocumentResponse(reqId, ((PutDocumentMessage)msg).getDocumentPut().getDocument(), err, outcome, reply.getTrace()); case DocumentProtocol.MESSAGE_UPDATEDOCUMENT: - return new DocumentUpdateResponse(reqId, ((UpdateDocumentMessage)msg).getDocumentUpdate(), err, outcome); + return new DocumentUpdateResponse(reqId, ((UpdateDocumentMessage)msg).getDocumentUpdate(), err, outcome, reply.getTrace()); case DocumentProtocol.MESSAGE_REMOVEDOCUMENT: - return new DocumentIdResponse(reqId, ((RemoveDocumentMessage)msg).getDocumentId(), err, outcome); + return new DocumentIdResponse(reqId, ((RemoveDocumentMessage)msg).getDocumentId(), err, outcome, reply.getTrace()); case DocumentProtocol.MESSAGE_GETDOCUMENT: - return new DocumentIdResponse(reqId, ((GetDocumentMessage)msg).getDocumentId(), err, outcome); + return new DocumentIdResponse(reqId, ((GetDocumentMessage)msg).getDocumentId(), err, outcome, reply.getTrace()); default: - return new Response(reqId, err, outcome); + return new Response(reqId, err, outcome, reply.getTrace()); } } @@ -317,26 +319,26 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { if (getDoc != null) { getDoc.setLastModified(docReply.getLastModified()); } - return new DocumentResponse(reqId, getDoc); + return new DocumentResponse(reqId, getDoc, reply.getTrace()); case DocumentProtocol.REPLY_REMOVEDOCUMENT: - return new RemoveResponse(reqId, ((RemoveDocumentReply)reply).wasFound()); + return new RemoveResponse(reqId, ((RemoveDocumentReply)reply).wasFound(), reply.getTrace()); case DocumentProtocol.REPLY_UPDATEDOCUMENT: - return new UpdateResponse(reqId, ((UpdateDocumentReply)reply).wasFound()); + return new UpdateResponse(reqId, ((UpdateDocumentReply)reply).wasFound(), reply.getTrace()); case DocumentProtocol.REPLY_PUTDOCUMENT: break; default: - return new Response(reqId); + return new Response(reqId, null, SUCCESS, reply.getTrace()); } Message msg = reply.getMessage(); switch (msg.getType()) { case DocumentProtocol.MESSAGE_PUTDOCUMENT: - return new DocumentResponse(reqId, ((PutDocumentMessage)msg).getDocumentPut().getDocument()); + return new DocumentResponse(reqId, ((PutDocumentMessage)msg).getDocumentPut().getDocument(), null, SUCCESS, reply.getTrace()); case DocumentProtocol.MESSAGE_REMOVEDOCUMENT: - return new DocumentIdResponse(reqId, ((RemoveDocumentMessage)msg).getDocumentId()); + return new DocumentIdResponse(reqId, ((RemoveDocumentMessage)msg).getDocumentId(), null, SUCCESS, reply.getTrace()); case DocumentProtocol.MESSAGE_UPDATEDOCUMENT: - return new DocumentUpdateResponse(reqId, ((UpdateDocumentMessage)msg).getDocumentUpdate()); + return new DocumentUpdateResponse(reqId, ((UpdateDocumentMessage)msg).getDocumentUpdate(), null, SUCCESS, reply.getTrace()); default: - return new Response(reqId); + return new Response(reqId, null, SUCCESS, reply.getTrace()); } } @@ -364,11 +366,4 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { } } - static void setParameters(DocumentMessage message, DocumentOperationParameters parameters, - DocumentProtocol.Priority defaultPriority) { - message.setPriority(parameters.priority().orElse(defaultPriority)); - parameters.route().map(Route::parse).ifPresent(message::setRoute); - parameters.traceLevel().ifPresent(message.getTrace()::setLevel); - } - } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java index 7bf0976f249..c7ab8a23e11 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java @@ -26,12 +26,10 @@ import com.yahoo.messagebus.Message; import com.yahoo.messagebus.MessageBus; import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.ReplyHandler; -import com.yahoo.messagebus.routing.Route; import java.time.Duration; import static com.yahoo.documentapi.DocumentOperationParameters.parameters; -import static com.yahoo.documentapi.messagebus.MessageBusAsyncSession.setParameters; /** * An implementation of the SyncSession interface running over message bus. @@ -89,10 +87,14 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re * @return The reply received. */ public Reply syncSend(Message msg) { - return syncSend(msg, defaultTimeout); + return syncSend(msg, parameters()); } - private Reply syncSend(Message msg, Duration timeout) { + private Reply syncSend(Message msg, DocumentOperationParameters parameters) { + return syncSend(msg, defaultTimeout, parameters()); + } + + private Reply syncSend(Message msg, Duration timeout, DocumentOperationParameters parameters) { if (timeout != null) { msg.setTimeRemaining(timeout.toMillis()); } @@ -102,7 +104,7 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re msg.pushHandler(this); // store monitor Result result = null; while (result == null || result.type() == Result.ResultType.TRANSIENT_ERROR) { - result = session.send(msg); + result = session.send(msg, parameters); if (result != null && result.isSuccess()) { break; } @@ -130,8 +132,11 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re @Override public void put(DocumentPut documentPut, DocumentOperationParameters parameters) { PutDocumentMessage msg = new PutDocumentMessage(documentPut); - setParameters(msg, parameters, DocumentProtocol.Priority.NORMAL_3); - syncSendPutDocumentMessage(msg); + msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_3)); + Reply reply = syncSend(msg, parameters); + if (reply.hasErrors()) { + throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply), reply.getErrorCodes()); + } } @Override @@ -147,9 +152,9 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re @Override public Document get(DocumentId id, DocumentOperationParameters parameters, Duration timeout) { GetDocumentMessage msg = new GetDocumentMessage(id, parameters.fieldSet().orElse(AllFields.NAME)); - setParameters(msg, parameters, DocumentProtocol.Priority.NORMAL_1); + msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_1)); - Reply reply = syncSend(msg, timeout != null ? timeout : defaultTimeout); + Reply reply = syncSend(msg, timeout != null ? timeout : defaultTimeout, parameters); if (reply.hasErrors()) { throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply)); } @@ -177,13 +182,9 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re @Override public boolean remove(DocumentRemove documentRemove, DocumentOperationParameters parameters) { RemoveDocumentMessage msg = new RemoveDocumentMessage(documentRemove.getId()); - setParameters(msg, parameters, DocumentProtocol.Priority.NORMAL_2); + msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_2)); msg.setCondition(documentRemove.getCondition()); - return remove(msg); - } - - private boolean remove(RemoveDocumentMessage msg) { - Reply reply = syncSend(msg); + Reply reply = syncSend(msg, parameters); if (reply.hasErrors()) { throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply)); } @@ -206,8 +207,8 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re @Override public boolean update(DocumentUpdate update, DocumentOperationParameters parameters) { UpdateDocumentMessage msg = new UpdateDocumentMessage(update); - setParameters(msg, parameters, DocumentProtocol.Priority.NORMAL_2); - Reply reply = syncSend(msg); + msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_2)); + Reply reply = syncSend(msg, parameters); if (reply.hasErrors()) { throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply), reply.getErrorCodes()); } @@ -256,10 +257,4 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re } } - private void syncSendPutDocumentMessage(PutDocumentMessage putDocumentMessage) { - Reply reply = syncSend(putDocumentMessage); - if (reply.hasErrors()) { - throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply), reply.getErrorCodes()); - } - } } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java index 6000026580a..e603a150b34 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java @@ -23,6 +23,7 @@ import com.yahoo.document.restapi.RestApiException; import com.yahoo.document.restapi.RestUri; import com.yahoo.document.select.DocumentSelector; import com.yahoo.document.select.parser.ParseException; +import com.yahoo.documentapi.DocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; -- cgit v1.2.3 From 7d280adf22a83d5446153820a42258d352418269 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 22 Sep 2020 16:38:43 +0200 Subject: Use NOT_FOUND in DocumentResponse as well — as success MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- documentapi/abi-spec.json | 2 +- .../com/yahoo/documentapi/DocumentResponse.java | 21 ++++++++------------- .../yahoo/documentapi/local/LocalAsyncSession.java | 4 ++-- .../messagebus/MessageBusAsyncSession.java | 1 + .../src/main/java/com/yahoo/jdisc/Response.java | 1 + .../http/server/util/ByteLimitedInputStream.java | 2 +- .../src/main/java/com/yahoo/io/ByteWriter.java | 2 +- .../src/main/java/com/yahoo/slime/SlimeUtils.java | 6 +++++- 8 files changed, 20 insertions(+), 19 deletions(-) diff --git a/documentapi/abi-spec.json b/documentapi/abi-spec.json index 7a0a5fa0dd7..23c3bd3c1b8 100644 --- a/documentapi/abi-spec.json +++ b/documentapi/abi-spec.json @@ -159,10 +159,10 @@ "public void (long)", "public void (long, com.yahoo.document.Document)", "public void (long, java.lang.String, boolean)", - "public void (long, java.lang.String, com.yahoo.documentapi.Response$Outcome)", "public void (long, com.yahoo.document.Document, java.lang.String, boolean)", "public void (long, com.yahoo.document.Document, java.lang.String, com.yahoo.documentapi.Response$Outcome)", "public com.yahoo.document.Document getDocument()", + "public boolean isSuccess()", "public int hashCode()", "public boolean equals(java.lang.Object)", "public java.lang.String toString()" diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java index a4d81a5e145..7fa9707f6ed 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java @@ -26,8 +26,7 @@ public class DocumentResponse extends Response { * @param document the Document to encapsulate in the Response */ public DocumentResponse(long requestId, Document document) { - super(requestId); - this.document = document; + this(requestId, document, null, document != null ? Outcome.SUCCESS : Outcome.NOT_FOUND); } /** @@ -38,20 +37,10 @@ public class DocumentResponse extends Response { */ @Deprecated(since = "7") // TODO: Remove on Vespa 8 public DocumentResponse(long requestId, String textMessage, boolean success) { - super(requestId, textMessage, success); + super(requestId, textMessage, success ? Outcome.NOT_FOUND : Outcome.ERROR); document = null; } - /** - * Creates a response containing a textual message - * - * @param textMessage the message to encapsulate in the Response - * @param outcome the outcome of this operation - */ - public DocumentResponse(long requestId, String textMessage, Outcome outcome) { - this(requestId, null, textMessage, outcome); - } - /** * Creates a response containing a textual message and/or a document * @@ -97,6 +86,12 @@ public class DocumentResponse extends Response { */ public Document getDocument() { return document; } + @Override + public boolean isSuccess() { + // TODO: is it right that Get operations are successful without a result, in this API? + return super.isSuccess() || outcome() == Outcome.NOT_FOUND; + } + public int hashCode() { return super.hashCode() + (document == null ? 0 : document.hashCode()); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java index 398675e594e..40f26a82a89 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java @@ -61,7 +61,7 @@ public class LocalAsyncSession implements AsyncSession { long req = getNextRequestId(); try { syncSession.put(documentPut, pri); - addResponse(new DocumentResponse(req)); + addResponse(new DocumentResponse(req, documentPut.getDocument())); } catch (Exception e) { addResponse(new DocumentResponse(req, documentPut.getDocument(), e.getMessage(), Response.Outcome.ERROR)); } @@ -85,7 +85,7 @@ public class LocalAsyncSession implements AsyncSession { try { addResponse(new DocumentResponse(req, syncSession.get(id))); } catch (Exception e) { - addResponse(new DocumentResponse(req, e.getMessage(), Response.Outcome.ERROR)); + addResponse(new DocumentResponse(req, null, e.getMessage(), Response.Outcome.ERROR)); } return new Result(req); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java index d1da3c7dc03..fc7d7fe6b9f 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java @@ -329,6 +329,7 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { default: return new Response(reqId, null, SUCCESS, reply.getTrace()); } + // TODO jonmv: Why on earth is this relevant!? Message msg = reply.getMessage(); switch (msg.getType()) { case DocumentProtocol.MESSAGE_PUTDOCUMENT: diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/Response.java b/jdisc_core/src/main/java/com/yahoo/jdisc/Response.java index c3d07a70e14..c3e386b020c 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/Response.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/Response.java @@ -87,6 +87,7 @@ public class Response { int UNPROCESSABLE_ENTITY = 422; int LOCKED = 423; int FAILED_DEPENDENCY = 424; + int TOO_MANY_REQUESTS = 429; /** 5xx: Server Error - The server failed to fulfill an apparently valid request. */ int INTERNAL_SERVER_ERROR = 500; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java index c8ae79deebd..260b57b2112 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java @@ -5,7 +5,7 @@ import java.io.IOException; import java.io.InputStream; /** - * @author Einar M R Rosenvinge + * @author Einar M R Rosenvinge * * @since 5.1.23 */ diff --git a/vespajlib/src/main/java/com/yahoo/io/ByteWriter.java b/vespajlib/src/main/java/com/yahoo/io/ByteWriter.java index 05f7cec9189..de9c88a713d 100644 --- a/vespajlib/src/main/java/com/yahoo/io/ByteWriter.java +++ b/vespajlib/src/main/java/com/yahoo/io/ByteWriter.java @@ -11,7 +11,7 @@ import java.nio.charset.CharsetEncoder; /** * A buffered writer which accepts byte arrays in addition to character arrays. * - * @author Steinar Knutsen + * @author Steinar Knutsen */ public class ByteWriter extends AbstractByteWriter { private final OutputStream stream; diff --git a/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java index 51a4fc167c7..144a9a585f6 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java +++ b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java @@ -94,8 +94,12 @@ public class SlimeUtils { } public static byte[] toJsonBytes(Slime slime) throws IOException { + return toJsonBytes(slime.get()); + } + + public static byte[] toJsonBytes(Inspector inspector) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - new JsonFormat(true).encode(baos, slime); + new JsonFormat(true).encode(baos, inspector); return baos.toByteArray(); } -- cgit v1.2.3 From 0001e35d02bd576355507d78a44e08514dcc7f0c Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 24 Sep 2020 21:25:48 +0200 Subject: Remove _probably_ irrelevant case handling --- .../main/java/com/yahoo/documentapi/DocumentResponse.java | 11 ++++++++++- .../documentapi/messagebus/MessageBusAsyncSession.java | 14 +------------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java index 7fa9707f6ed..2a02b70d4fc 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java @@ -26,7 +26,16 @@ public class DocumentResponse extends Response { * @param document the Document to encapsulate in the Response */ public DocumentResponse(long requestId, Document document) { - this(requestId, document, null, document != null ? Outcome.SUCCESS : Outcome.NOT_FOUND); + this(requestId, document, null); + } + + /** + * Creates a successful response containing a document + * + * @param document the Document to encapsulate in the Response + */ + public DocumentResponse(long requestId, Document document, Trace trace) { + this(requestId, document, null, document != null ? Outcome.SUCCESS : Outcome.NOT_FOUND, trace); } /** diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java index fc7d7fe6b9f..7a71089c180 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java @@ -325,19 +325,7 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { case DocumentProtocol.REPLY_UPDATEDOCUMENT: return new UpdateResponse(reqId, ((UpdateDocumentReply)reply).wasFound(), reply.getTrace()); case DocumentProtocol.REPLY_PUTDOCUMENT: - break; - default: - return new Response(reqId, null, SUCCESS, reply.getTrace()); - } - // TODO jonmv: Why on earth is this relevant!? - Message msg = reply.getMessage(); - switch (msg.getType()) { - case DocumentProtocol.MESSAGE_PUTDOCUMENT: - return new DocumentResponse(reqId, ((PutDocumentMessage)msg).getDocumentPut().getDocument(), null, SUCCESS, reply.getTrace()); - case DocumentProtocol.MESSAGE_REMOVEDOCUMENT: - return new DocumentIdResponse(reqId, ((RemoveDocumentMessage)msg).getDocumentId(), null, SUCCESS, reply.getTrace()); - case DocumentProtocol.MESSAGE_UPDATEDOCUMENT: - return new DocumentUpdateResponse(reqId, ((UpdateDocumentMessage)msg).getDocumentUpdate(), null, SUCCESS, reply.getTrace()); + return new DocumentResponse(reqId, ((PutDocumentMessage)reply.getMessage()).getDocumentPut().getDocument(), reply.getTrace()); default: return new Response(reqId, null, SUCCESS, reply.getTrace()); } -- cgit v1.2.3 From 08bca580ba37998cc826629e0a11532c51e35131 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 24 Sep 2020 21:27:34 +0200 Subject: Update abi spec --- documentapi/abi-spec.json | 48 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/documentapi/abi-spec.json b/documentapi/abi-spec.json index 23c3bd3c1b8..f5f2a7c1845 100644 --- a/documentapi/abi-spec.json +++ b/documentapi/abi-spec.json @@ -40,13 +40,17 @@ "public com.yahoo.documentapi.Result put(com.yahoo.document.Document, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public com.yahoo.documentapi.Result put(com.yahoo.document.DocumentPut)", "public com.yahoo.documentapi.Result put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.Result put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.DocumentOperationParameters)", "public abstract com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId)", "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, boolean, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, com.yahoo.documentapi.DocumentOperationParameters)", "public abstract com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentId)", "public com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentId, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentId, com.yahoo.documentapi.DocumentOperationParameters)", "public abstract com.yahoo.documentapi.Result update(com.yahoo.document.DocumentUpdate)", "public com.yahoo.documentapi.Result update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.Result update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.DocumentOperationParameters)", "public abstract double getCurrentWindowSize()" ], "fields": [] @@ -130,6 +134,7 @@ "public void (long, java.lang.String, boolean)", "public void (long, com.yahoo.document.DocumentId, java.lang.String, boolean)", "public void (long, com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.Response$Outcome)", + "public void (long, com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.Response$Outcome, com.yahoo.messagebus.Trace)", "public com.yahoo.document.DocumentId getDocumentId()", "public int hashCode()", "public boolean equals(java.lang.Object)", @@ -149,6 +154,26 @@ ], "fields": [] }, + "com.yahoo.documentapi.DocumentOperationParameters": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public static com.yahoo.documentapi.DocumentOperationParameters parameters()", + "public com.yahoo.documentapi.DocumentOperationParameters withPriority(com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.DocumentOperationParameters withFieldSet(com.yahoo.document.fieldset.FieldSet)", + "public com.yahoo.documentapi.DocumentOperationParameters withFieldSet(java.lang.String)", + "public com.yahoo.documentapi.DocumentOperationParameters withRoute(java.lang.String)", + "public com.yahoo.documentapi.DocumentOperationParameters withTraceLevel(int)", + "public java.util.Optional priority()", + "public java.util.Optional fieldSet()", + "public java.util.Optional route()", + "public java.util.OptionalInt traceLevel()" + ], + "fields": [] + }, "com.yahoo.documentapi.DocumentResponse": { "superClass": "com.yahoo.documentapi.Response", "interfaces": [], @@ -158,9 +183,11 @@ "methods": [ "public void (long)", "public void (long, com.yahoo.document.Document)", + "public void (long, com.yahoo.document.Document, com.yahoo.messagebus.Trace)", "public void (long, java.lang.String, boolean)", "public void (long, com.yahoo.document.Document, java.lang.String, boolean)", "public void (long, com.yahoo.document.Document, java.lang.String, com.yahoo.documentapi.Response$Outcome)", + "public void (long, com.yahoo.document.Document, java.lang.String, com.yahoo.documentapi.Response$Outcome, com.yahoo.messagebus.Trace)", "public com.yahoo.document.Document getDocument()", "public boolean isSuccess()", "public int hashCode()", @@ -182,6 +209,7 @@ "public void (long, java.lang.String, com.yahoo.documentapi.Response$Outcome)", "public void (long, com.yahoo.document.DocumentUpdate, java.lang.String, boolean)", "public void (long, com.yahoo.document.DocumentUpdate, java.lang.String, com.yahoo.documentapi.Response$Outcome)", + "public void (long, com.yahoo.document.DocumentUpdate, java.lang.String, com.yahoo.documentapi.Response$Outcome, com.yahoo.messagebus.Trace)", "public com.yahoo.document.DocumentUpdate getDocumentUpdate()", "public int hashCode()", "public boolean equals(java.lang.Object)", @@ -354,6 +382,7 @@ ], "methods": [ "public void (long, boolean)", + "public void (long, boolean, com.yahoo.messagebus.Trace)", "public boolean wasFound()", "public boolean isSuccess()", "public int hashCode()", @@ -392,10 +421,12 @@ "public void (long, java.lang.String)", "public void (long, java.lang.String, boolean)", "public void (long, java.lang.String, com.yahoo.documentapi.Response$Outcome)", + "public void (long, java.lang.String, com.yahoo.documentapi.Response$Outcome, com.yahoo.messagebus.Trace)", "public java.lang.String getTextMessage()", "public boolean isSuccess()", "public com.yahoo.documentapi.Response$Outcome outcome()", "public long getRequestId()", + "public com.yahoo.messagebus.Trace getTrace()", "public boolean equals(java.lang.Object)", "public int hashCode()", "public java.lang.String toString()" @@ -543,14 +574,18 @@ "methods": [ "public abstract void put(com.yahoo.document.DocumentPut)", "public void put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public void put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.DocumentOperationParameters)", "public com.yahoo.document.Document get(com.yahoo.document.DocumentId)", "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public abstract com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.time.Duration)", "public abstract com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority, java.time.Duration)", + "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, com.yahoo.documentapi.DocumentOperationParameters, java.time.Duration)", "public abstract boolean remove(com.yahoo.document.DocumentRemove)", "public abstract boolean remove(com.yahoo.document.DocumentRemove, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public boolean remove(com.yahoo.document.DocumentRemove, com.yahoo.documentapi.DocumentOperationParameters)", "public abstract boolean update(com.yahoo.document.DocumentUpdate)", - "public abstract boolean update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)" + "public abstract boolean update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public boolean update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.DocumentOperationParameters)" ], "fields": [] }, @@ -562,6 +597,7 @@ ], "methods": [ "public void (long, boolean)", + "public void (long, boolean, com.yahoo.messagebus.Trace)", "public boolean wasFound()", "public boolean isSuccess()", "public int hashCode()", @@ -1022,13 +1058,17 @@ "methods": [ "public com.yahoo.documentapi.Result put(com.yahoo.document.Document)", "public com.yahoo.documentapi.Result put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.Result put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.DocumentOperationParameters)", "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId)", "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, boolean, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, com.yahoo.documentapi.DocumentOperationParameters)", "public com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentId)", "public com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentId, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentId, com.yahoo.documentapi.DocumentOperationParameters)", "public com.yahoo.documentapi.Result update(com.yahoo.document.DocumentUpdate)", "public com.yahoo.documentapi.Result update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.Result update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.DocumentOperationParameters)", "public com.yahoo.documentapi.Result send(com.yahoo.messagebus.Message)", "public com.yahoo.documentapi.Response getNext()", "public com.yahoo.documentapi.Response getNext(int)", @@ -1132,14 +1172,16 @@ "public com.yahoo.messagebus.Reply syncSend(com.yahoo.messagebus.Message)", "public void put(com.yahoo.document.DocumentPut)", "public void put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", - "public com.yahoo.document.Document get(com.yahoo.document.DocumentId)", - "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public void put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.DocumentOperationParameters)", "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.time.Duration)", "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority, java.time.Duration)", + "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, com.yahoo.documentapi.DocumentOperationParameters, java.time.Duration)", "public boolean remove(com.yahoo.document.DocumentRemove)", "public boolean remove(com.yahoo.document.DocumentRemove, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public boolean remove(com.yahoo.document.DocumentRemove, com.yahoo.documentapi.DocumentOperationParameters)", "public boolean update(com.yahoo.document.DocumentUpdate)", "public boolean update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public boolean update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.DocumentOperationParameters)", "public java.lang.String getRoute()", "public void setRoute(java.lang.String)", "public int getTraceLevel()", -- cgit v1.2.3 From 8371f2aa81f4ca58d4c8fca5a5a9cc18055e8422 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Fri, 25 Sep 2020 07:06:24 +0200 Subject: Remove topleveldispatch remnants --- .../model/builder/xml/dom/ContentBuilderTest.java | 1 - .../com/yahoo/vespa/model/content/IndexedTest.java | 2 +- configutil/src/lib/configstatus.cpp | 1 - configutil/src/tests/model_inspect/model.cfg | 50 ++++++++-------------- .../service/ConfigSentinelClientTest.java | 3 +- .../metricsproxy/service/ConfigSentinelDummy.java | 4 -- 6 files changed, 21 insertions(+), 40 deletions(-) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 443bd2433c1..10049daa541 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -229,7 +229,6 @@ public class ContentBuilderTest extends DomBuilderTest { /* Not yet assertNotNull(h.getService("qrserver")); - assertNotNull(h.getService("topleveldisptach")); assertNotNull(h.getService("docproc")); */ diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java index 504c3d9ba9c..5d4756c430d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java @@ -143,7 +143,7 @@ public class IndexedTest extends ContentBaseTest { // HostResource h = model.getHostSystem().getHosts().get(0); // String [] expectedServices = {"logserver", "configserver", "adminserver", "slobrok", // "logd", "configproxy","config-sentinel", - // "qrserver", "fleetcontroller", "topleveldispatch", + // "qrserver", "fleetcontroller", // "storagenode", "searchnode", "distributor", "transactionlogserver"}; // DomContentBuilderTest.assertServices(h, expectedServices); Routing routing = model.getRouting(); diff --git a/configutil/src/lib/configstatus.cpp b/configutil/src/lib/configstatus.cpp index 415edf6defc..254fa94a8ec 100644 --- a/configutil/src/lib/configstatus.cpp +++ b/configutil/src/lib/configstatus.cpp @@ -165,7 +165,6 @@ ConfigStatus::action() if (!upToDate) { if (svc.type == "searchnode" || - svc.type == "topleveldispatch" || svc.type == "logd") { std::cerr << "[generation not up-to-date ignored]" << std::endl; diff --git a/configutil/src/tests/model_inspect/model.cfg b/configutil/src/tests/model_inspect/model.cfg index 388bae2256d..bc29226e494 100644 --- a/configutil/src/tests/model_inspect/model.cfg +++ b/configutil/src/tests/model_inspect/model.cfg @@ -115,35 +115,23 @@ hosts[0].services[11].ports[0].number 19106 hosts[0].services[11].ports[0].tags "status admin rpc rtx" hosts[0].services[11].ports[1].number 19107 hosts[0].services[11].ports[1].tags "unused" -hosts[0].services[12].name "topleveldispatch" -hosts[0].services[12].type "topleveldispatch" -hosts[0].services[12].configid "search/cluster.music/tlds/tld.0" -hosts[0].services[12].clustertype "search" -hosts[0].services[12].clustername "music" +hosts[0].services[12].name "docprocservice" +hosts[0].services[12].type "container" +hosts[0].services[12].configid "docproc/cluster.music.indexing/0" +hosts[0].services[12].clustertype "" +hosts[0].services[12].clustername "music.indexing" hosts[0].services[12].index 0 -hosts[0].services[12].ports[0].number 19108 -hosts[0].services[12].ports[0].tags "admin rpc" -hosts[0].services[12].ports[1].number 19109 -hosts[0].services[12].ports[1].tags "fs4" -hosts[0].services[12].ports[2].number 19110 -hosts[0].services[12].ports[2].tags "health json http" -hosts[0].services[13].name "docprocservice" -hosts[0].services[13].type "container" -hosts[0].services[13].configid "docproc/cluster.music.indexing/0" -hosts[0].services[13].clustertype "" -hosts[0].services[13].clustername "music.indexing" -hosts[0].services[13].index 0 -hosts[0].services[13].ports[0].number 19111 -hosts[0].services[13].ports[0].tags "state external query http" -hosts[0].services[13].ports[1].number 19112 -hosts[0].services[13].ports[1].tags "external status http" -hosts[0].services[13].ports[2].number 19113 -hosts[0].services[13].ports[2].tags "messaging rpc" -hosts[0].services[13].ports[3].number 19114 -hosts[0].services[13].ports[3].tags "external fileserver http" -hosts[0].services[13].ports[4].number 19115 -hosts[0].services[13].ports[4].tags "admin rpc" -hosts[0].services[13].ports[5].number 19116 -hosts[0].services[13].ports[5].tags "rmiregistry rmi" -hosts[0].services[13].ports[6].number 19117 -hosts[0].services[13].ports[6].tags "jmx rmi" +hosts[0].services[12].ports[0].number 19111 +hosts[0].services[12].ports[0].tags "state external query http" +hosts[0].services[12].ports[1].number 19112 +hosts[0].services[12].ports[1].tags "external status http" +hosts[0].services[12].ports[2].number 19113 +hosts[0].services[12].ports[2].tags "messaging rpc" +hosts[0].services[12].ports[3].number 19114 +hosts[0].services[12].ports[3].tags "external fileserver http" +hosts[0].services[12].ports[4].number 19115 +hosts[0].services[12].ports[4].tags "admin rpc" +hosts[0].services[12].ports[5].number 19116 +hosts[0].services[12].ports[5].tags "rmiregistry rmi" +hosts[0].services[12].ports[6].number 19117 +hosts[0].services[12].ports[6].tags "jmx rmi" diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java index 810596d6d0b..7c0302ae3dc 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java @@ -62,7 +62,7 @@ public class ConfigSentinelClientTest { } @Test - public void testElastic() throws Exception { + public void testElastic() { String response = "container state=RUNNING mode=AUTO pid=14338 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"get/container.0\"\n" + "container-clustercontroller state=RUNNING mode=AUTO pid=25020 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/cluster-controllers/0\"\n" + "distributor state=RUNNING mode=AUTO pid=25024 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/distributor/0\"\n" + @@ -72,7 +72,6 @@ public class ConfigSentinelClientTest { "metricsproxy state=RUNNING mode=AUTO pid=13107 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"hosts/vespa19.dev.gq1.yahoo.com/metricsproxy\"\n" + "searchnode state=RUNNING mode=AUTO pid=25023 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/search/cluster.search/0\"\n" + "slobrok state=RUNNING mode=AUTO pid=25019 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/slobrok.0\"\n" + - "topleveldispatch state=RUNNING mode=AUTO pid=25026 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/search/cluster.search/tlds/tld.0\"\n" + "\n"; ConfigSentinelDummy configsentinel = new ConfigSentinelDummy(response); diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelDummy.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelDummy.java index 5c746521c1f..def57617f56 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelDummy.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelDummy.java @@ -15,8 +15,6 @@ public class ConfigSentinelDummy { + "logserver state=RUNNING mode=AUTO pid=6518 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/logserver\"\n" + "logd state=RUNNING mode=AUTO pid=6517 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"hosts/dell-bl5s7.trondheim.corp.yahoo.com/logd\"\n" + "searchnode2 state=RUNNING mode=AUTO pid=6527 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c0/r1\"\n" - + "topleveldispatch2 state=RUNNING mode=AUTO pid=6525 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/tlds/tld.1\"\n" - + "topleveldispatch state=RUNNING mode=AUTO pid=6524 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/tlds/tld.0\"\n" + "clustercontroller2 state=RUNNING mode=AUTO pid=6523 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/rtx/1\"\n" + "clustercontroller state=RUNNING mode=AUTO pid=6522 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/rtx/0\"\n" + "slobrok state=RUNNING mode=AUTO pid=6519 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/slobrok.0\"\n" @@ -43,8 +41,6 @@ public class ConfigSentinelDummy { + "logserver state=RUNNING mode=AUTO pid=6518 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/logserver\"\n" + "logd state=RUNNING mode=AUTO pid=6517 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"hosts/dell-bl5s7.trondheim.corp.yahoo.com/logd\"\n" + "searchnode2 state=RUNNING mode=AUTO pid=6527 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c0/r1\"\n" - + "topleveldispatch2 state=RUNNING mode=AUTO pid=6525 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/tlds/tld.1\"\n" - + "topleveldispatch state=RUNNING mode=AUTO pid=6524 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/tlds/tld.0\"\n" + "clustercontroller2 state=RUNNING mode=AUTO pid=6523 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/rtx/1\"\n" + "clustercontroller state=RUNNING mode=AUTO pid=6522 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/rtx/0\"\n" + "slobrok state=RUNNING mode=AUTO pid=6519 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/slobrok.0\"\n" -- cgit v1.2.3 From eff56a86814b5c9ebaf1ede6b3d5235cc7854550 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Fri, 25 Sep 2020 07:32:34 +0200 Subject: Add guard for tenant not existing --- .../java/com/yahoo/vespa/config/server/ApplicationRepository.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index bd704611bcb..6d606d76a1b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -782,7 +782,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye * @return the active session, or null if there is no active session for the given application id. */ public RemoteSession getActiveSession(ApplicationId applicationId) { - return getActiveSession(getTenant(applicationId), applicationId); + Tenant tenant = getTenant(applicationId); + if (tenant == null) throw new IllegalArgumentException("Could not find any tenant for '" + applicationId + "'"); + return getActiveSession(tenant, applicationId); } public long getSessionIdForApplication(ApplicationId applicationId) { -- cgit v1.2.3 From 3d27b3c70bb4699489488d9a4f443fe6dcf3a3a3 Mon Sep 17 00:00:00 2001 From: Yngve Aasheim Date: Fri, 25 Sep 2020 08:25:53 +0200 Subject: Report feeding_blocked.max metric value --- .../main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java | 1 + 1 file changed, 1 insertion(+) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 78e97719af0..56b70bec24e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -405,6 +405,7 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.resource_usage.transient_memory.average")); metrics.add(new Metric("content.proton.resource_usage.memory_mappings.max")); metrics.add(new Metric("content.proton.resource_usage.open_file_descriptors.max")); + metrics.add(new Metric("content.proton.resource_usage.feeding_blocked.max")); metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.enum_store.average")); metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.multi_value.average")); metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.feeding_blocked.last")); // TODO: Remove in Vespa 8 -- cgit v1.2.3 From e8c56e5898d217d2c2dd37710ef2c88d98350ff4 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Fri, 25 Sep 2020 08:47:43 +0200 Subject: Make sure to always delete remote session --- .../vespa/config/server/ApplicationRepository.java | 4 ++-- .../config/server/session/SessionRepository.java | 22 ++++++++++++++-------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index bd704611bcb..e584c01db01 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -536,8 +536,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye transaction.add(new ApplicationRolesStore(curator, tenant.getPath()).delete(applicationId)); // Delete endpoint certificates transaction.add(new EndpointCertificateMetadataStore(curator, tenant.getPath()).delete(applicationId)); - // (When rotations are updated in zk, we need to redeploy the zone app, on the right config server - // this is done asynchronously in application maintenance by the node repository) + // This call will remove application in zookeeper. Watches in TenantApplications will remove the application + // and allocated hosts in model and handlers in RPC server transaction.add(tenantApplications.createDeleteTransaction(applicationId)); hostProvisioner.ifPresent(provisioner -> provisioner.remove(transaction, applicationId)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index c198f1dd0fc..8de196ce7c0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -371,14 +371,20 @@ public class SessionRepository { } public void delete(RemoteSession remoteSession) { - LocalSession localSession = getLocalSession(remoteSession.getSessionId()); - remoteSession.deactivate(); - if (localSession == null) { - // This change will be picked up by directoryCache in this class, which will do the rest of the cleanup - try (Lock lock = lock(remoteSession.getSessionId())) { - remoteSession.delete(); - } - } else { + // This change will be picked up by directoryCache in this class, which will do the rest of + // the cleanup on all config servers + long sessionId = remoteSession.getSessionId(); + try (Lock lock = lock(sessionId)) { + // TODO: Change log level to FINE when debugging is finished + log.log(Level.INFO, "Deactivating and deleting remote session " + sessionId); + remoteSession.deactivate(); + remoteSession.delete(); + remoteSessionCache.remove(sessionId); + } + LocalSession localSession = getLocalSession(sessionId); + if (localSession != null) { + // TODO: Change log level to FINE when debugging is finished + log.log(Level.INFO, "Deleting local session " + sessionId); deleteLocalSession(localSession); } } -- cgit v1.2.3 From ed06bab257cbaba2a2ea21df3d51f91d5ba6470c Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Fri, 25 Sep 2020 08:58:45 +0200 Subject: Avoid double iteration --- .../java/com/yahoo/vespa/curator/stats/LockInfo.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java index c9f3ec4b757..00583311ae9 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.curator.stats; import java.time.Duration; import java.time.Instant; import java.util.Optional; -import java.util.stream.Stream; /** * Information about a lock. @@ -75,14 +74,14 @@ public class LockInfo { var stackTrace = new StringBuilder(); StackTraceElement[] elements = thread.getStackTrace(); - for (int i = 0; i < elements.length && i < 20; ++i) { - Stream.of(elements).forEach(element -> - stackTrace.append(element.getClassName()) - .append('(') - .append(element.getFileName()) - .append(':') - .append(element.getLineNumber()) - .append(")\n")); + for (int i = 0; i < elements.length; ++i) { + var element = elements[i]; + stackTrace.append(element.getClassName()) + .append('(') + .append(element.getFileName()) + .append(':') + .append(element.getLineNumber()) + .append(")\n"); } this.stackTrace = Optional.of(stackTrace.toString()); -- cgit v1.2.3 From f52458bedbbd2856eb6170aecebe008f097b7b5b Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 25 Sep 2020 09:22:02 +0200 Subject: Fix javadoc typo Co-authored-by: Jon Bratseth --- documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java index 721faf281f7..60f70a91338 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java @@ -201,7 +201,7 @@ public interface AsyncSession extends Session { *

Removes a document if it is present. This method returns immediately.

* *

If this result is a success, this - * call will cause one or more {@link DocumentIdResponse} objects to apprear within the timeout time of this session. + * call will cause one or more {@link DocumentIdResponse} objects to appear within the timeout time of this session. * The response returned later will either be a success, or contain the document id submitted here. * If it was not a success, this method has no further effects.

* -- cgit v1.2.3 From db2797a271390eb12be83910402cbc7c540f78f0 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 25 Sep 2020 09:22:12 +0200 Subject: Fix javadoc header Co-authored-by: Jon Bratseth --- .../java/com/yahoo/documentapi/DocumentOperationParameters.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentOperationParameters.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentOperationParameters.java index cfb134ffa4b..38ed4b8eb6e 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentOperationParameters.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentOperationParameters.java @@ -9,7 +9,11 @@ import java.util.OptionalInt; import static java.util.Objects.requireNonNull; -/** Optional parameters for a document operation. Immutable class. */ +/** + * Optional parameters for a document operation. Immutable class. + * + * @author jonmv + */ public class DocumentOperationParameters { private static final DocumentOperationParameters empty = new DocumentOperationParameters(null, null, null, -1); -- cgit v1.2.3 From 5c4cfe18a62db70f25e66b00459e11f573b05b16 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 25 Sep 2020 09:22:22 +0200 Subject: Add copyright header Co-authored-by: Jon Bratseth --- .../src/main/java/com/yahoo/documentapi/DocumentOperationParameters.java | 1 + 1 file changed, 1 insertion(+) diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentOperationParameters.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentOperationParameters.java index 38ed4b8eb6e..3258c2f5b2c 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentOperationParameters.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentOperationParameters.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi; import com.yahoo.document.fieldset.FieldSet; -- cgit v1.2.3 From 032d45a7cdab3ab4f358ac0371a5d6b0eb57415a Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Fri, 25 Sep 2020 09:23:24 +0200 Subject: Add duration of acquire, in locked, and total --- .../vespa/hosted/provision/restapi/LocksResponse.java | 3 +++ .../main/java/com/yahoo/vespa/curator/stats/LockInfo.java | 15 ++++++++++++++- .../com/yahoo/vespa/curator/stats/ThreadLockInfo.java | 6 ++++-- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java index 24321cd8a20..2e22631b830 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java @@ -93,6 +93,9 @@ public class LocksResponse extends HttpResponse { lockInfo.getTimeLockWasAcquired().ifPresent(instant -> lockInfoCursor.setString("lock-acquired-time", toString(instant))); lockInfoCursor.setString("lock-state", lockInfo.getLockState().name()); lockInfo.getTimeTerminalStateWasReached().ifPresent(instant -> lockInfoCursor.setString("terminal-state-time", toString(instant))); + lockInfoCursor.setString("acquire-duration", lockInfo.getDurationOfAcquire().toString()); + lockInfoCursor.setString("locked-duration", lockInfo.getDurationWithLock().toString()); + lockInfoCursor.setString("total-duration", lockInfo.getDuration().toString()); lockInfo.getStackTrace().ifPresent(stackTrace -> lockInfoCursor.setString("stack-trace", stackTrace)); } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java index 00583311ae9..37481a6b1c7 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java @@ -59,8 +59,20 @@ public class LockInfo { public Optional getTimeTerminalStateWasReached() { return terminalStateInstant; } public Optional getStackTrace() { return stackTrace; } + public Duration getDurationOfAcquire() { + return Duration.between(acquireInstant, lockAcquiredInstant.orElseGet(Instant::now)); + } + + public Duration getDurationWithLock() { + return lockAcquiredInstant + .map(start -> Duration.between(start, terminalStateInstant.orElseGet(Instant::now))) + .orElse(Duration.ZERO); + } + + public Duration getDuration() { return Duration.between(acquireInstant, terminalStateInstant.orElseGet(Instant::now)); } + /** Get time from just before trying to acquire lock to the time the terminal state was reached, or ZERO. */ - public Duration getTotalTime() { + public Duration getDurationInTerminalStateAndForPriorityQueue() { return terminalStateInstant.map(instant -> Duration.between(acquireInstant, instant)).orElse(Duration.ZERO); } @@ -74,6 +86,7 @@ public class LockInfo { var stackTrace = new StringBuilder(); StackTraceElement[] elements = thread.getStackTrace(); + // first stack frame is "java.lang.Thread(Thread.java:1606)" or "jdk.internal.misc.Unsafe(Unsafe.java:-2)"!? for (int i = 0; i < elements.length; ++i) { var element = elements[i]; stackTrace.append(element.getClassName()) diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java index 2e4672841bf..0ca9377d360 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java @@ -30,7 +30,8 @@ public class ThreadLockInfo { private static final int MAX_COMPLETED_LOCK_INFOS_SIZE = 5; /** Would have used a thread-safe priority queue. */ private static final Object completedLockInfosMonitor = new Object(); - private static final PriorityQueue completedLockInfos = new PriorityQueue<>(Comparator.comparing(LockInfo::getTotalTime)); + private static final PriorityQueue completedLockInfos = + new PriorityQueue<>(Comparator.comparing(LockInfo::getDurationInTerminalStateAndForPriorityQueue)); private static final ConcurrentHashMap countersByLockPath = new ConcurrentHashMap<>(); @@ -126,7 +127,8 @@ public class ThreadLockInfo { if (completedLockInfos.size() < MAX_COMPLETED_LOCK_INFOS_SIZE) { lockInfo.fillStackTrace(); completedLockInfos.add(lockInfo); - } else if (lockInfo.getTotalTime().compareTo(completedLockInfos.peek().getTotalTime()) > 0) { + } else if (lockInfo.getDurationInTerminalStateAndForPriorityQueue() + .compareTo(completedLockInfos.peek().getDurationInTerminalStateAndForPriorityQueue()) > 0) { completedLockInfos.poll(); lockInfo.fillStackTrace(); completedLockInfos.add(lockInfo); -- cgit v1.2.3 From 0d12e6650c20799240eb9324ef426745d70ee58f Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 25 Sep 2020 09:34:43 +0200 Subject: Non-functional changes --- .../main/java/com/yahoo/container/jdisc/RequestHandlerTestDriver.java | 4 ++-- .../java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java | 4 +--- .../com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/RequestHandlerTestDriver.java b/container-core/src/main/java/com/yahoo/container/jdisc/RequestHandlerTestDriver.java index 22933556d9f..8243ad07760 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/RequestHandlerTestDriver.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/RequestHandlerTestDriver.java @@ -121,7 +121,7 @@ public class RequestHandlerTestDriver implements AutoCloseable { } /** - * Read the next piece of data from this channel even it blocking is needed. + * Read the next piece of data from this channel, blocking if needed. * If all data is already read, this returns null. */ public String read() { @@ -147,7 +147,7 @@ public class RequestHandlerTestDriver implements AutoCloseable { return responseString.toString(); } - /** Consumes all currently available data, or return "" if no data is available right now. Never blocks. */ + /** Consumes all currently available data, or returns "" if no data is available right now. Never blocks. */ public String readIfAvailable() { StringBuilder b = new StringBuilder(); while (content.available()>0) { diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java index d729e2371c9..c46488694de 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java @@ -119,9 +119,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler metric.add(RENDERING_ERRORS, 1, null); long time = System.currentTimeMillis() - startTime; log.log(time < 900 ? Level.INFO : Level.WARNING, - "IO error while responding to " + " [" - + request.getUri() + "] " + "(total time " - + time + " ms) ", e); + "IO error while responding to [" + request.getUri() + "] (total time " + time + " ms) ", e); try { output.flush(); } catch (Exception ignored) { } } finally { if (channel != null && ! (httpResponse instanceof AsyncHttpResponse)) { diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java index ff891dbb298..c60cd9cc378 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java @@ -92,7 +92,7 @@ public class MessageBusDocumentAccess extends DocumentAccess { @Override public MessageBusVisitorSession createVisitorSession(VisitorParameters params) throws ParseException, IllegalArgumentException { MessageBusVisitorSession session = MessageBusVisitorSession.createForMessageBus( - bus.getMessageBus(), scheduledExecutorService, params); + messageBus(), scheduledExecutorService, params); session.start(); return session; } -- cgit v1.2.3 From f6d6f0db9f6dab4a48aea33dd8c41b28ff624ad5 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 25 Sep 2020 09:53:42 +0200 Subject: Let the injectable lazy document access be specifically injected --- .../core/documentapi/DocumentAccessProvider.java | 121 +++++++++++++++++++++ .../MessageBusDocumentAccessProvider.java | 121 --------------------- 2 files changed, 121 insertions(+), 121 deletions(-) create mode 100644 container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java delete mode 100644 container-core/src/main/java/com/yahoo/container/core/documentapi/MessageBusDocumentAccessProvider.java diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java new file mode 100644 index 00000000000..1fc156d4e90 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java @@ -0,0 +1,121 @@ +package com.yahoo.container.core.documentapi; + +import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; +import com.yahoo.container.di.componentgraph.Provider; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.documentapi.AsyncParameters; +import com.yahoo.documentapi.AsyncSession; +import com.yahoo.documentapi.DocumentAccess; +import com.yahoo.documentapi.DocumentAccessParams; +import com.yahoo.documentapi.SubscriptionParameters; +import com.yahoo.documentapi.SubscriptionSession; +import com.yahoo.documentapi.SyncParameters; +import com.yahoo.documentapi.SyncSession; +import com.yahoo.documentapi.VisitorDestinationParameters; +import com.yahoo.documentapi.VisitorDestinationSession; +import com.yahoo.documentapi.VisitorParameters; +import com.yahoo.documentapi.VisitorSession; +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; +import com.yahoo.documentapi.messagebus.MessageBusParams; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.vespa.config.content.LoadTypeConfig; + +/** + * Lets a lazily initialised DocumentAccess forwarding to a real MessageBusDocumentAccess be injected in containers. + * + * @author jonmv + */ +public class DocumentAccessProvider extends AbstractComponent implements Provider { + + private final DocumentAccessProvider.LazyWrapper access; + + @Inject + // TODO jonmv: Have Slobrok and RPC config injected as well. + public DocumentAccessProvider(DocumentmanagerConfig documentmanagerConfig, LoadTypeConfig loadTypeConfig) { + this.access = new LazyWrapper(documentmanagerConfig, loadTypeConfig); + } + + @Override + public DocumentAccessProvider.LazyWrapper get() { + return access; + } + + @Override + public void deconstruct() { + access.shutdown(); + } + + + public static class LazyWrapper extends DocumentAccess { + + private final DocumentmanagerConfig documentmanagerConfig; + private final LoadTypeConfig loadTypeConfig; + private final Object monitor = new Object(); + + private DocumentAccess delegate = null; + private boolean shutDown = false; + + private LazyWrapper(DocumentmanagerConfig documentmanagerConfig, + LoadTypeConfig loadTypeConfig) { + super(new DocumentAccessParams().setDocumentmanagerConfig(documentmanagerConfig)); + this.documentmanagerConfig = documentmanagerConfig; + this.loadTypeConfig = loadTypeConfig; + } + + private DocumentAccess delegate() { + synchronized (monitor) { + if (delegate == null) { + if (shutDown) + throw new IllegalStateException("This document access has been shut down"); + + delegate = new MessageBusDocumentAccess((MessageBusParams) new MessageBusParams(new LoadTypeSet(loadTypeConfig)).setDocumentmanagerConfig(documentmanagerConfig)); + } + return delegate; + } + } + + @Override + public void shutdown() { + synchronized (monitor) { + super.shutdown(); + shutDown = true; + if (delegate != null) + delegate.shutdown(); + } + } + + @Override + public SyncSession createSyncSession(SyncParameters parameters) { + return delegate().createSyncSession(parameters); + } + + @Override + public AsyncSession createAsyncSession(AsyncParameters parameters) { + return delegate().createAsyncSession(parameters); + } + + @Override + public VisitorSession createVisitorSession(VisitorParameters parameters) throws ParseException { + return delegate().createVisitorSession(parameters); + } + + @Override + public VisitorDestinationSession createVisitorDestinationSession(VisitorDestinationParameters parameters) { + return delegate().createVisitorDestinationSession(parameters); + } + + @Override + public SubscriptionSession createSubscription(SubscriptionParameters parameters) { + return delegate().createSubscription(parameters); + } + + @Override + public SubscriptionSession openSubscription(SubscriptionParameters parameters) { + return delegate().openSubscription(parameters); + } + + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/MessageBusDocumentAccessProvider.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/MessageBusDocumentAccessProvider.java deleted file mode 100644 index 78653a81e58..00000000000 --- a/container-core/src/main/java/com/yahoo/container/core/documentapi/MessageBusDocumentAccessProvider.java +++ /dev/null @@ -1,121 +0,0 @@ -package com.yahoo.container.core.documentapi; - -import com.google.inject.Inject; -import com.yahoo.component.AbstractComponent; -import com.yahoo.container.di.componentgraph.Provider; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.document.select.parser.ParseException; -import com.yahoo.documentapi.AsyncParameters; -import com.yahoo.documentapi.AsyncSession; -import com.yahoo.documentapi.DocumentAccess; -import com.yahoo.documentapi.DocumentAccessParams; -import com.yahoo.documentapi.SubscriptionParameters; -import com.yahoo.documentapi.SubscriptionSession; -import com.yahoo.documentapi.SyncParameters; -import com.yahoo.documentapi.SyncSession; -import com.yahoo.documentapi.VisitorDestinationParameters; -import com.yahoo.documentapi.VisitorDestinationSession; -import com.yahoo.documentapi.VisitorParameters; -import com.yahoo.documentapi.VisitorSession; -import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; -import com.yahoo.documentapi.messagebus.MessageBusParams; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; -import com.yahoo.vespa.config.content.LoadTypeConfig; - -/** - * Lets a lazily initialised DocumentAccess forwarding to a real MessageBusDocumentAccess be injected in containers. - * - * @author jonmv - */ -public class MessageBusDocumentAccessProvider extends AbstractComponent implements Provider { - - private final DocumentAccess access; - - @Inject - // TODO jonmv: Have Slobrok and RPC config injected as well. - public MessageBusDocumentAccessProvider(DocumentmanagerConfig documentmanagerConfig, LoadTypeConfig loadTypeConfig) { - this.access = new LazyForwardingMessageBusDocumentAccess(documentmanagerConfig, loadTypeConfig); - } - - @Override - public DocumentAccess get() { - return access; - } - - @Override - public void deconstruct() { - access.shutdown(); - } - - - private static class LazyForwardingMessageBusDocumentAccess extends DocumentAccess { - - private final DocumentmanagerConfig documentmanagerConfig; - private final LoadTypeConfig loadTypeConfig; - private final Object monitor = new Object(); - - private DocumentAccess delegate = null; - private boolean shutDown = false; - - public LazyForwardingMessageBusDocumentAccess(DocumentmanagerConfig documentmanagerConfig, - LoadTypeConfig loadTypeConfig) { - super(new DocumentAccessParams().setDocumentmanagerConfig(documentmanagerConfig)); - this.documentmanagerConfig = documentmanagerConfig; - this.loadTypeConfig = loadTypeConfig; - } - - private DocumentAccess delegate() { - synchronized (monitor) { - if (delegate == null) { - if (shutDown) - throw new IllegalStateException("This document access has been shut down"); - - delegate = new MessageBusDocumentAccess((MessageBusParams) new MessageBusParams(new LoadTypeSet(loadTypeConfig)).setDocumentmanagerConfig(documentmanagerConfig)); - } - return delegate; - } - } - - @Override - public void shutdown() { - synchronized (monitor) { - super.shutdown(); - shutDown = true; - if (delegate != null) - delegate.shutdown(); - } - } - - @Override - public SyncSession createSyncSession(SyncParameters parameters) { - return delegate().createSyncSession(parameters); - } - - @Override - public AsyncSession createAsyncSession(AsyncParameters parameters) { - return delegate().createAsyncSession(parameters); - } - - @Override - public VisitorSession createVisitorSession(VisitorParameters parameters) throws ParseException { - return delegate().createVisitorSession(parameters); - } - - @Override - public VisitorDestinationSession createVisitorDestinationSession(VisitorDestinationParameters parameters) { - return delegate().createVisitorDestinationSession(parameters); - } - - @Override - public SubscriptionSession createSubscription(SubscriptionParameters parameters) { - return delegate().createSubscription(parameters); - } - - @Override - public SubscriptionSession openSubscription(SubscriptionParameters parameters) { - return delegate().openSubscription(parameters); - } - - } - -} -- cgit v1.2.3 From 2ec7549aefa5ca46a61fc97827cdd37ee7ba905b Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Fri, 25 Sep 2020 10:37:16 +0200 Subject: Add static utilities for unlimited and zero quota --- .../vespa/hosted/controller/api/integration/billing/Quota.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Quota.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Quota.java index cae768afc90..89addd97943 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Quota.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Quota.java @@ -7,6 +7,8 @@ import java.util.Optional; * Quota information transmitted to the configserver on deploy. */ public class Quota { + private static final Quota UNLIMITED = new Quota(Optional.empty(), Optional.empty()); + private static final Quota ZERO = new Quota(0, 0); private final Optional maxClusterSize; private final Optional budget; // in USD/hr, as calculated by NodeResources @@ -40,6 +42,14 @@ public class Quota { return new Quota(maxClusterSize, Optional.of(budget)); } + public static Quota zero() { + return ZERO; + } + + public static Quota unlimted() { + return UNLIMITED; + } + @Override public boolean equals(Object o) { if (this == o) return true; -- cgit v1.2.3 From 17c6dc3a23badb17f468fe8b38bda57eab717a4c Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 25 Sep 2020 10:37:50 +0200 Subject: Eliminate config self-subscription from MessageBusDocumentAccess --- .../core/documentapi/DocumentAccessProvider.java | 22 ++++++++++++++-------- .../messagebus/MessageBusDocumentAccess.java | 4 +++- .../com/yahoo/messagebus/MessageBusParams.java | 13 +++++++++++++ .../java/com/yahoo/messagebus/RPCMessageBus.java | 15 +++++++++++++++ 4 files changed, 45 insertions(+), 9 deletions(-) diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java index 1fc156d4e90..deabcbad9b4 100644 --- a/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java +++ b/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java @@ -1,6 +1,7 @@ package com.yahoo.container.core.documentapi; import com.google.inject.Inject; +import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.component.AbstractComponent; import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.document.config.DocumentmanagerConfig; @@ -20,6 +21,7 @@ import com.yahoo.documentapi.VisitorSession; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.messagebus.MessagebusConfig; import com.yahoo.vespa.config.content.LoadTypeConfig; /** @@ -33,8 +35,9 @@ public class DocumentAccessProvider extends AbstractComponent implements Provide @Inject // TODO jonmv: Have Slobrok and RPC config injected as well. - public DocumentAccessProvider(DocumentmanagerConfig documentmanagerConfig, LoadTypeConfig loadTypeConfig) { - this.access = new LazyWrapper(documentmanagerConfig, loadTypeConfig); + public DocumentAccessProvider(DocumentmanagerConfig documentmanagerConfig, LoadTypeConfig loadTypeConfig, + SlobroksConfig slobroksConfig, MessagebusConfig messagebusConfig) { + this.access = new LazyWrapper(documentmanagerConfig, loadTypeConfig, slobroksConfig, messagebusConfig); } @Override @@ -50,18 +53,21 @@ public class DocumentAccessProvider extends AbstractComponent implements Provide public static class LazyWrapper extends DocumentAccess { - private final DocumentmanagerConfig documentmanagerConfig; - private final LoadTypeConfig loadTypeConfig; + private final MessageBusParams parameters; private final Object monitor = new Object(); private DocumentAccess delegate = null; private boolean shutDown = false; private LazyWrapper(DocumentmanagerConfig documentmanagerConfig, - LoadTypeConfig loadTypeConfig) { + LoadTypeConfig loadTypeConfig, + SlobroksConfig slobroksConfig, + MessagebusConfig messagebusConfig) { super(new DocumentAccessParams().setDocumentmanagerConfig(documentmanagerConfig)); - this.documentmanagerConfig = documentmanagerConfig; - this.loadTypeConfig = loadTypeConfig; + this.parameters = new MessageBusParams(new LoadTypeSet(loadTypeConfig)); + this.parameters.setDocumentmanagerConfig(documentmanagerConfig); + this.parameters.getRPCNetworkParams().setSlobroksConfig(slobroksConfig); + this.parameters.getMessageBusParams().setMessageBusConfig(messagebusConfig); } private DocumentAccess delegate() { @@ -70,7 +76,7 @@ public class DocumentAccessProvider extends AbstractComponent implements Provide if (shutDown) throw new IllegalStateException("This document access has been shut down"); - delegate = new MessageBusDocumentAccess((MessageBusParams) new MessageBusParams(new LoadTypeSet(loadTypeConfig)).setDocumentmanagerConfig(documentmanagerConfig)); + delegate = new MessageBusDocumentAccess(parameters); } return delegate; } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java index c60cd9cc378..3e35e9bd12a 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java @@ -60,7 +60,9 @@ public class MessageBusDocumentAccess extends DocumentAccess { bus = new NetworkMessageBus(network, new MessageBus(network, mbusParams)); } else { - bus = new RPCMessageBus(mbusParams, params.getRPCNetworkParams(), params.getRoutingConfigId()); + bus = params.getRPCNetworkParams().getSlobroksConfig() != null && mbusParams.getMessageBusConfig() != null + ? new RPCMessageBus(mbusParams, params.getRPCNetworkParams()) // prefer without self-subscription if config is set + : new RPCMessageBus(mbusParams, params.getRPCNetworkParams(), params.getRoutingConfigId()); } } catch (Exception e) { diff --git a/messagebus/src/main/java/com/yahoo/messagebus/MessageBusParams.java b/messagebus/src/main/java/com/yahoo/messagebus/MessageBusParams.java index 7f55401cf43..2c8382ab715 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/MessageBusParams.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/MessageBusParams.java @@ -19,6 +19,7 @@ public class MessageBusParams { private RetryPolicy retryPolicy; private int maxPendingCount; private int maxPendingSize; + private MessagebusConfig config; /** * Constructs a new instance of this parameter object with default values for all members. @@ -27,6 +28,7 @@ public class MessageBusParams { retryPolicy = new RetryTransientErrorsPolicy(); maxPendingCount = 1024; maxPendingSize = 128 * 1024 * 1024; + config = null; } /** @@ -39,6 +41,7 @@ public class MessageBusParams { retryPolicy = params.retryPolicy; maxPendingCount = params.maxPendingCount; maxPendingSize = params.maxPendingSize; + config = params.config; } /** @@ -143,4 +146,14 @@ public class MessageBusParams { this.maxPendingSize = maxSize; return this; } + + public MessagebusConfig getMessageBusConfig() { + return config; + } + + public MessageBusParams setMessageBusConfig(MessagebusConfig config) { + this.config = config; + return this; + } + } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/RPCMessageBus.java b/messagebus/src/main/java/com/yahoo/messagebus/RPCMessageBus.java index 5ea278c410b..f5bf03c5420 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/RPCMessageBus.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/RPCMessageBus.java @@ -18,6 +18,21 @@ public class RPCMessageBus extends NetworkMessageBus { private final ConfigAgent configAgent; + /** + * Constructs a new instance of this class. + * + * @param mbusParams A complete set of message bus parameters, including messagebus config. + * @param rpcParams A complete set of network parameters, including rpc network config. + */ + public RPCMessageBus(MessageBusParams mbusParams, RPCNetworkParams rpcParams) { + this(mbusParams, new RPCNetwork(rpcParams)); + } + + private RPCMessageBus(MessageBusParams mbusParams, RPCNetwork network) { + super(network, new MessageBus(network, mbusParams)); + configAgent = new ConfigAgent(mbusParams.getMessageBusConfig(), getMessageBus()); + } + /** * Constructs a new instance of this class. * -- cgit v1.2.3 From 9fe53dc16eaf71124bebf16aa283a4ee37625e86 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Fri, 25 Sep 2020 10:42:13 +0200 Subject: Allow creating Deployment with PrepareParams --- .../vespa/config/server/ApplicationRepository.java | 4 +- .../vespa/config/server/deploy/Deployment.java | 119 +++++++++++---------- 2 files changed, 63 insertions(+), 60 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index bd704611bcb..2ea97e92632 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -396,7 +396,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye LocalSession newSession = sessionRepository.createSessionFromExisting(activeSession, logger, true, timeoutBudget); sessionRepository.addLocalSession(newSession); - return Optional.of(Deployment.unprepared(newSession, this, hostProvisioner, tenant, timeout, clock, + return Optional.of(Deployment.unprepared(newSession, this, hostProvisioner, tenant, logger, timeout, clock, false /* don't validate as this is already deployed */, bootstrap)); } @@ -420,7 +420,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } private Deployment deployment(LocalSession session, Tenant tenant, Duration timeout, boolean force) { - return Deployment.prepared(session, this, hostProvisioner, tenant, timeout, clock, false, force); + return Deployment.prepared(session, this, hostProvisioner, tenant, logger, timeout, clock, false, force); } public Transaction deactivateCurrentActivateNew(Session active, LocalSession prepared, boolean force) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 6c111ff0131..d9efe2c4b97 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -1,14 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.deploy; -import com.yahoo.component.Version; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.AthenzDomain; -import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.Provisioner; -import java.util.logging.Level; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.ApplicationRepository.ActionTimer; import com.yahoo.vespa.config.server.TimeoutBudget; @@ -18,13 +16,13 @@ import com.yahoo.vespa.config.server.session.LocalSession; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.RemoteSession; import com.yahoo.vespa.config.server.session.Session; -import com.yahoo.vespa.config.server.session.SilentDeployLogger; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.curator.Lock; import java.time.Clock; import java.time.Duration; import java.util.Optional; +import java.util.logging.Level; import java.util.logging.Logger; import static com.yahoo.vespa.curator.Curator.CompletionWaiter; @@ -44,81 +42,54 @@ public class Deployment implements com.yahoo.config.provision.Deployment { /** The session containing the application instance to activate */ private final LocalSession session; private final ApplicationRepository applicationRepository; - private final Optional hostProvisioner; + private final Supplier params; + private final Optional provisioner; private final Tenant tenant; - private final Duration timeout; + private final DeployLogger logger; private final Clock clock; - private final DeployLogger logger = new SilentDeployLogger(); - - /** The repository part of docker image this application should run on. Version is separate from image repo */ - final Optional dockerImageRepository; - - /** The Vespa version this application should run on */ - private final Version version; - - /** True if this deployment is done to bootstrap the config server */ - private final boolean isBootstrap; - - /** The (optional) Athenz domain this application should use */ - private final Optional athenzDomain; - private boolean prepared = false; - - /** Whether this model should be validated (only takes effect if prepared=false) */ - private final boolean validate; + private boolean prepared; - /** Whether activation of this model should be forced */ - private final boolean force; - - private Deployment(LocalSession session, ApplicationRepository applicationRepository, - Optional hostProvisioner, Tenant tenant, Duration timeout, - Clock clock, boolean prepared, boolean validate, boolean isBootstrap, boolean force) { + private Deployment(LocalSession session, ApplicationRepository applicationRepository, Supplier params, + Optional provisioner, Tenant tenant, DeployLogger logger, Clock clock, boolean prepared) { this.session = session; this.applicationRepository = applicationRepository; - this.hostProvisioner = hostProvisioner; + this.params = params; + this.provisioner = provisioner; this.tenant = tenant; - this.timeout = timeout; + this.logger = logger; this.clock = clock; this.prepared = prepared; - this.validate = validate; - this.dockerImageRepository = session.getDockerImageRepository(); - this.version = session.getVespaVersion(); - this.isBootstrap = isBootstrap; - this.athenzDomain = session.getAthenzDomain(); - this.force = force; } public static Deployment unprepared(LocalSession session, ApplicationRepository applicationRepository, - Optional hostProvisioner, Tenant tenant, + Optional provisioner, Tenant tenant, PrepareParams params, DeployLogger logger, Clock clock) { + return new Deployment(session, applicationRepository, () -> params, provisioner, tenant, logger, clock, false); + } + + public static Deployment unprepared(LocalSession session, ApplicationRepository applicationRepository, + Optional provisioner, Tenant tenant, DeployLogger logger, Duration timeout, Clock clock, boolean validate, boolean isBootstrap) { - return new Deployment(session, applicationRepository, hostProvisioner, tenant, timeout, clock, false, - validate, isBootstrap, false); + Supplier params = createPrepareParams(clock, timeout, session, isBootstrap, !validate, false); + return new Deployment(session, applicationRepository, params, provisioner, tenant, logger, clock, false); } public static Deployment prepared(LocalSession session, ApplicationRepository applicationRepository, - Optional hostProvisioner, Tenant tenant, + Optional provisioner, Tenant tenant, DeployLogger logger, Duration timeout, Clock clock, boolean isBootstrap, boolean force) { - return new Deployment(session, applicationRepository, hostProvisioner, tenant, - timeout, clock, true, true, isBootstrap, force); + Supplier params = createPrepareParams(clock, timeout, session, isBootstrap, false, force); + return new Deployment(session, applicationRepository, params, provisioner, tenant, logger, clock, true); } /** Prepares this. This does nothing if this is already prepared */ @Override public void prepare() { if (prepared) return; + PrepareParams params = this.params.get(); ApplicationId applicationId = session.getApplicationId(); try (ActionTimer timer = applicationRepository.timerFor(applicationId, "deployment.prepareMillis")) { - TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); - - PrepareParams.Builder params = new PrepareParams.Builder().applicationId(applicationId) - .timeoutBudget(timeoutBudget) - .ignoreValidationErrors(!validate) - .vespaVersion(version.toString()) - .isBootstrap(isBootstrap); - dockerImageRepository.ifPresent(params::dockerImageRepository); - athenzDomain.ifPresent(params::athenzDomain); Optional activeApplicationSet = applicationRepository.getCurrentActiveApplicationSet(tenant, applicationId); - tenant.getSessionRepository().prepareLocalSession(session, logger, params.build(), activeApplicationSet, + tenant.getSessionRepository().prepareLocalSession(session, logger, params, activeApplicationSet, tenant.getPath(), clock.instant()); this.prepared = true; } @@ -130,16 +101,17 @@ public class Deployment implements com.yahoo.config.provision.Deployment { prepare(); validateSessionStatus(session); + PrepareParams params = this.params.get(); ApplicationId applicationId = session.getApplicationId(); try (ActionTimer timer = applicationRepository.timerFor(applicationId, "deployment.activateMillis")) { - TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); + TimeoutBudget timeoutBudget = params.getTimeoutBudget(); if ( ! timeoutBudget.hasTimeLeft()) throw new RuntimeException("Timeout exceeded when trying to activate '" + applicationId + "'"); RemoteSession previousActiveSession; CompletionWaiter waiter; try (Lock lock = tenant.getApplicationRepo().lock(applicationId)) { previousActiveSession = applicationRepository.getActiveSession(applicationId); - waiter = applicationRepository.activate(session, previousActiveSession, applicationId, force); + waiter = applicationRepository.activate(session, previousActiveSession, applicationId, params.force()); } catch (RuntimeException e) { throw e; @@ -150,7 +122,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { waiter.awaitCompletion(timeoutBudget.timeLeft()); log.log(Level.INFO, session.logPre() + "Session " + session.getSessionId() + " activated successfully using " + - hostProvisioner.map(provisioner -> provisioner.getClass().getSimpleName()).orElse("no host provisioner") + + provisioner.map(provisioner -> provisioner.getClass().getSimpleName()).orElse("no host provisioner") + ". Config generation " + session.getMetaData().getGeneration() + (previousActiveSession != null ? ". Based on session " + previousActiveSession.getSessionId() : "") + ". File references: " + applicationRepository.getFileReferences(applicationId)); @@ -165,7 +137,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { */ @Override public void restart(HostFilter filter) { - hostProvisioner.get().restart(session.getApplicationId(), filter); + provisioner.get().restart(session.getApplicationId(), filter); } /** Exposes the session of this for testing only */ @@ -180,4 +152,35 @@ public class Deployment implements com.yahoo.config.provision.Deployment { } } + /** + * @param clock system clock + * @param timeout total timeout duration of prepare + activate + * @param session the local session for this deployment + * @param isBootstrap true if this deployment is done to bootstrap the config server + * @param ignoreValidationErrors whether this model should be validated + * @param force whether activation of this model should be forced + */ + private static Supplier createPrepareParams( + Clock clock, Duration timeout, LocalSession session, + boolean isBootstrap, boolean ignoreValidationErrors, boolean force) { + + // Supplier because shouldn't/cant create this before validateSessionStatus() for prepared deployments + // memoized because we want to create this once for unprepared deployments + return Suppliers.memoize(() -> { + TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); + + PrepareParams.Builder params = new PrepareParams.Builder() + .applicationId(session.getApplicationId()) + .vespaVersion(session.getVespaVersion().toString()) + .timeoutBudget(timeoutBudget) + .ignoreValidationErrors(ignoreValidationErrors) + .isBootstrap(isBootstrap) + .force(force); + session.getDockerImageRepository().ifPresent(params::dockerImageRepository); + session.getAthenzDomain().ifPresent(params::athenzDomain); + + return params.build(); + }); + } + } -- cgit v1.2.3 From 0b2ded054339ef1a55645aa3cc770e52dc686518 Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Fri, 25 Sep 2020 10:46:08 +0200 Subject: Copyright and @autho tags --- .../yahoo/vespa/hosted/controller/api/integration/billing/Quota.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Quota.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Quota.java index 89addd97943..9348e7dcbdb 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Quota.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Quota.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.billing; import java.util.Objects; @@ -5,6 +6,9 @@ import java.util.Optional; /** * Quota information transmitted to the configserver on deploy. + * + * @author andreer + * @author ogronnesby */ public class Quota { private static final Quota UNLIMITED = new Quota(Optional.empty(), Optional.empty()); -- cgit v1.2.3 From fdb980d9e035c249974a452535a8181c2ee21879 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 25 Sep 2020 10:50:54 +0200 Subject: Explicitly enable cache stats Disabled by default. --- .../com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java | 2 +- .../yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 21ec3d48aef..f7521ba786d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -123,7 +123,7 @@ public class NodeSerializer { public NodeSerializer(NodeFlavors flavors, long cacheSize) { this.flavors = flavors; - this.cache = CacheBuilder.newBuilder().maximumSize(cacheSize).build(); + this.cache = CacheBuilder.newBuilder().maximumSize(cacheSize).recordStats().build(); } public byte[] toJson(Node node) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 557453daa49..872db9d62ca 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -123,7 +123,7 @@ public class MetricsReporterTest { expectedMetrics.put("suspendedSeconds", 123L); expectedMetrics.put("numberOfServices", 0L); - expectedMetrics.put("cache.nodeObject.hitRate", 1.0D); + expectedMetrics.put("cache.nodeObject.hitRate", 0.6D); expectedMetrics.put("cache.nodeObject.evictionCount", 0L); expectedMetrics.put("cache.nodeObject.size", 2L); -- cgit v1.2.3 From 4fb114d59b873a66ac5ea4d4e9693f7540d9e275 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Fri, 25 Sep 2020 11:06:59 +0200 Subject: Use deployment to prepare --- .../vespa/config/server/ApplicationRepository.java | 15 ++++++--------- .../yahoo/vespa/config/server/deploy/Deployment.java | 17 ++++++++++++++--- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 2ea97e92632..351ff262fa9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -283,16 +283,13 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye validateThatLocalSessionIsNotActive(tenant, sessionId); LocalSession session = getLocalSession(tenant, sessionId); ApplicationId applicationId = prepareParams.getApplicationId(); - Optional currentActiveApplicationSet = getCurrentActiveApplicationSet(tenant, applicationId); DeployHandlerLogger logger = DeployHandlerLogger.forApplication(applicationId, prepareParams.isVerbose()); - try (ActionTimer timer = timerFor(applicationId, "deployment.prepareMillis")) { - SessionRepository sessionRepository = tenant.getSessionRepository(); - ConfigChangeActions actions = sessionRepository.prepareLocalSession(session, logger, prepareParams, - currentActiveApplicationSet, tenant.getPath(), now); - logConfigChangeActions(actions, logger); - log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " prepared successfully. "); - return new PrepareResult(sessionId, actions, logger); - } + Deployment deployment = Deployment.unprepared(session, this, hostProvisioner, tenant, prepareParams, logger, clock); + deployment.prepare(); + + logConfigChangeActions(deployment.configChangeActions(), logger); + log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " prepared successfully. "); + return new PrepareResult(sessionId, deployment.configChangeActions(), logger); } public PrepareResult deploy(CompressedApplicationInputStream in, PrepareParams prepareParams, Instant now) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index d9efe2c4b97..f04f25c23dc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.ApplicationRepository.ActionTimer; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.session.LocalSession; import com.yahoo.vespa.config.server.session.PrepareParams; @@ -49,6 +50,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { private final Clock clock; private boolean prepared; + private ConfigChangeActions configChangeActions; private Deployment(LocalSession session, ApplicationRepository applicationRepository, Supplier params, Optional provisioner, Tenant tenant, DeployLogger logger, Clock clock, boolean prepared) { @@ -86,11 +88,11 @@ public class Deployment implements com.yahoo.config.provision.Deployment { public void prepare() { if (prepared) return; PrepareParams params = this.params.get(); - ApplicationId applicationId = session.getApplicationId(); + ApplicationId applicationId = params.getApplicationId(); try (ActionTimer timer = applicationRepository.timerFor(applicationId, "deployment.prepareMillis")) { Optional activeApplicationSet = applicationRepository.getCurrentActiveApplicationSet(tenant, applicationId); - tenant.getSessionRepository().prepareLocalSession(session, logger, params, activeApplicationSet, - tenant.getPath(), clock.instant()); + this.configChangeActions = tenant.getSessionRepository().prepareLocalSession( + session, logger, params, activeApplicationSet, tenant.getPath(), clock.instant()); this.prepared = true; } } @@ -143,6 +145,15 @@ public class Deployment implements com.yahoo.config.provision.Deployment { /** Exposes the session of this for testing only */ public LocalSession session() { return session; } + /** + * @return config change actions that need to be performed as result of prepare + * @throws IllegalArgumentException if called without being prepared by this + */ + public ConfigChangeActions configChangeActions() { + if (configChangeActions != null) return configChangeActions; + throw new IllegalArgumentException("No config change actions: " + (prepared ? "was already prepared" : "not yet prepared")); + } + private void validateSessionStatus(LocalSession localSession) { long sessionId = localSession.getSessionId(); if (Session.Status.NEW.equals(localSession.getStatus())) { -- cgit v1.2.3 From 2cd240de82eedbad02868290c89036e91d3f8e7d Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 25 Sep 2020 11:23:06 +0200 Subject: Move injectable lazy document access to top level and rename to VespaDocumentAccess --- .../core/documentapi/DocumentAccessProvider.java | 97 +------------------- .../core/documentapi/VespaDocumentAccess.java | 101 +++++++++++++++++++++ 2 files changed, 105 insertions(+), 93 deletions(-) create mode 100644 container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java index deabcbad9b4..6eb80ae1aaf 100644 --- a/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java +++ b/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java @@ -5,22 +5,6 @@ import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.component.AbstractComponent; import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.document.select.parser.ParseException; -import com.yahoo.documentapi.AsyncParameters; -import com.yahoo.documentapi.AsyncSession; -import com.yahoo.documentapi.DocumentAccess; -import com.yahoo.documentapi.DocumentAccessParams; -import com.yahoo.documentapi.SubscriptionParameters; -import com.yahoo.documentapi.SubscriptionSession; -import com.yahoo.documentapi.SyncParameters; -import com.yahoo.documentapi.SyncSession; -import com.yahoo.documentapi.VisitorDestinationParameters; -import com.yahoo.documentapi.VisitorDestinationSession; -import com.yahoo.documentapi.VisitorParameters; -import com.yahoo.documentapi.VisitorSession; -import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; -import com.yahoo.documentapi.messagebus.MessageBusParams; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.messagebus.MessagebusConfig; import com.yahoo.vespa.config.content.LoadTypeConfig; @@ -29,19 +13,19 @@ import com.yahoo.vespa.config.content.LoadTypeConfig; * * @author jonmv */ -public class DocumentAccessProvider extends AbstractComponent implements Provider { +public class DocumentAccessProvider extends AbstractComponent implements Provider { - private final DocumentAccessProvider.LazyWrapper access; + private final VespaDocumentAccess access; @Inject // TODO jonmv: Have Slobrok and RPC config injected as well. public DocumentAccessProvider(DocumentmanagerConfig documentmanagerConfig, LoadTypeConfig loadTypeConfig, SlobroksConfig slobroksConfig, MessagebusConfig messagebusConfig) { - this.access = new LazyWrapper(documentmanagerConfig, loadTypeConfig, slobroksConfig, messagebusConfig); + this.access = new VespaDocumentAccess(documentmanagerConfig, loadTypeConfig, slobroksConfig, messagebusConfig); } @Override - public DocumentAccessProvider.LazyWrapper get() { + public VespaDocumentAccess get() { return access; } @@ -51,77 +35,4 @@ public class DocumentAccessProvider extends AbstractComponent implements Provide } - public static class LazyWrapper extends DocumentAccess { - - private final MessageBusParams parameters; - private final Object monitor = new Object(); - - private DocumentAccess delegate = null; - private boolean shutDown = false; - - private LazyWrapper(DocumentmanagerConfig documentmanagerConfig, - LoadTypeConfig loadTypeConfig, - SlobroksConfig slobroksConfig, - MessagebusConfig messagebusConfig) { - super(new DocumentAccessParams().setDocumentmanagerConfig(documentmanagerConfig)); - this.parameters = new MessageBusParams(new LoadTypeSet(loadTypeConfig)); - this.parameters.setDocumentmanagerConfig(documentmanagerConfig); - this.parameters.getRPCNetworkParams().setSlobroksConfig(slobroksConfig); - this.parameters.getMessageBusParams().setMessageBusConfig(messagebusConfig); - } - - private DocumentAccess delegate() { - synchronized (monitor) { - if (delegate == null) { - if (shutDown) - throw new IllegalStateException("This document access has been shut down"); - - delegate = new MessageBusDocumentAccess(parameters); - } - return delegate; - } - } - - @Override - public void shutdown() { - synchronized (monitor) { - super.shutdown(); - shutDown = true; - if (delegate != null) - delegate.shutdown(); - } - } - - @Override - public SyncSession createSyncSession(SyncParameters parameters) { - return delegate().createSyncSession(parameters); - } - - @Override - public AsyncSession createAsyncSession(AsyncParameters parameters) { - return delegate().createAsyncSession(parameters); - } - - @Override - public VisitorSession createVisitorSession(VisitorParameters parameters) throws ParseException { - return delegate().createVisitorSession(parameters); - } - - @Override - public VisitorDestinationSession createVisitorDestinationSession(VisitorDestinationParameters parameters) { - return delegate().createVisitorDestinationSession(parameters); - } - - @Override - public SubscriptionSession createSubscription(SubscriptionParameters parameters) { - return delegate().createSubscription(parameters); - } - - @Override - public SubscriptionSession openSubscription(SubscriptionParameters parameters) { - return delegate().openSubscription(parameters); - } - - } - } diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java new file mode 100644 index 00000000000..fffa915a840 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java @@ -0,0 +1,101 @@ +package com.yahoo.container.core.documentapi; + +import com.yahoo.cloud.config.SlobroksConfig; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.documentapi.AsyncParameters; +import com.yahoo.documentapi.AsyncSession; +import com.yahoo.documentapi.DocumentAccess; +import com.yahoo.documentapi.DocumentAccessParams; +import com.yahoo.documentapi.SubscriptionParameters; +import com.yahoo.documentapi.SubscriptionSession; +import com.yahoo.documentapi.SyncParameters; +import com.yahoo.documentapi.SyncSession; +import com.yahoo.documentapi.VisitorDestinationParameters; +import com.yahoo.documentapi.VisitorDestinationSession; +import com.yahoo.documentapi.VisitorParameters; +import com.yahoo.documentapi.VisitorSession; +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; +import com.yahoo.documentapi.messagebus.MessageBusParams; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.messagebus.MessagebusConfig; +import com.yahoo.vespa.config.content.LoadTypeConfig; + +/** + * Wraps a lazily initialised MessageBusDocumentAccess. Lazy to allow it to always be set up. + * Inject this class directly (instead of DocumentAccess) for use in internal code. + * + * @author jonmv + */ +public class VespaDocumentAccess extends DocumentAccess { + + private final MessageBusParams parameters; + private final Object monitor = new Object(); + + private DocumentAccess delegate = null; + private boolean shutDown = false; + + VespaDocumentAccess(DocumentmanagerConfig documentmanagerConfig, + LoadTypeConfig loadTypeConfig, + SlobroksConfig slobroksConfig, + MessagebusConfig messagebusConfig) { + super(new DocumentAccessParams().setDocumentmanagerConfig(documentmanagerConfig)); + this.parameters = new MessageBusParams(new LoadTypeSet(loadTypeConfig)); + this.parameters.setDocumentmanagerConfig(documentmanagerConfig); + this.parameters.getRPCNetworkParams().setSlobroksConfig(slobroksConfig); + this.parameters.getMessageBusParams().setMessageBusConfig(messagebusConfig); + } + + private DocumentAccess delegate() { + synchronized (monitor) { + if (delegate == null) { + if (shutDown) + throw new IllegalStateException("This document access has been shut down"); + + delegate = new MessageBusDocumentAccess(parameters); + } + return delegate; + } + } + + @Override + public void shutdown() { + synchronized (monitor) { + super.shutdown(); + shutDown = true; + if (delegate != null) + delegate.shutdown(); + } + } + + @Override + public SyncSession createSyncSession(SyncParameters parameters) { + return delegate().createSyncSession(parameters); + } + + @Override + public AsyncSession createAsyncSession(AsyncParameters parameters) { + return delegate().createAsyncSession(parameters); + } + + @Override + public VisitorSession createVisitorSession(VisitorParameters parameters) throws ParseException { + return delegate().createVisitorSession(parameters); + } + + @Override + public VisitorDestinationSession createVisitorDestinationSession(VisitorDestinationParameters parameters) { + return delegate().createVisitorDestinationSession(parameters); + } + + @Override + public SubscriptionSession createSubscription(SubscriptionParameters parameters) { + return delegate().createSubscription(parameters); + } + + @Override + public SubscriptionSession openSubscription(SubscriptionParameters parameters) { + return delegate().openSubscription(parameters); + } + +} -- cgit v1.2.3 From 1ce342752d01f2118688d9a7016944b98c4946b1 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 25 Sep 2020 11:23:12 +0200 Subject: Update abi spec --- messagebus/abi-spec.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/messagebus/abi-spec.json b/messagebus/abi-spec.json index 159c82971cd..4bec4bd91fe 100644 --- a/messagebus/abi-spec.json +++ b/messagebus/abi-spec.json @@ -362,7 +362,9 @@ "public int getMaxPendingCount()", "public com.yahoo.messagebus.MessageBusParams setMaxPendingCount(int)", "public int getMaxPendingSize()", - "public com.yahoo.messagebus.MessageBusParams setMaxPendingSize(int)" + "public com.yahoo.messagebus.MessageBusParams setMaxPendingSize(int)", + "public com.yahoo.messagebus.MessagebusConfig getMessageBusConfig()", + "public com.yahoo.messagebus.MessageBusParams setMessageBusConfig(com.yahoo.messagebus.MessagebusConfig)" ], "fields": [] }, @@ -643,6 +645,7 @@ "public" ], "methods": [ + "public void (com.yahoo.messagebus.MessageBusParams, com.yahoo.messagebus.network.rpc.RPCNetworkParams)", "public void (com.yahoo.messagebus.MessageBusParams, com.yahoo.messagebus.network.rpc.RPCNetworkParams, java.lang.String)", "public void (java.util.List, com.yahoo.messagebus.network.rpc.RPCNetworkParams, java.lang.String)", "public void (com.yahoo.messagebus.Protocol, java.lang.String)", -- cgit v1.2.3 From 929128a2e44bc55d56202a252527107b786a13f3 Mon Sep 17 00:00:00 2001 From: Håvard Pettersen Date: Fri, 25 Sep 2020 10:00:50 +0000 Subject: benchmark type detection --- vespalib/CMakeLists.txt | 3 +- .../src/tests/detect_type_benchmark/.gitignore | 1 + .../src/tests/detect_type_benchmark/CMakeLists.txt | 8 ++ .../detect_type_benchmark.cpp | 150 +++++++++++++++++++++ 4 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 vespalib/src/tests/detect_type_benchmark/.gitignore create mode 100644 vespalib/src/tests/detect_type_benchmark/CMakeLists.txt create mode 100644 vespalib/src/tests/detect_type_benchmark/detect_type_benchmark.cpp diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index c894a798b91..ec003329999 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -26,6 +26,7 @@ vespa_define_module( src/tests/benchmark_timer src/tests/box src/tests/btree + src/tests/child_process src/tests/closure src/tests/component src/tests/compress @@ -46,6 +47,7 @@ vespa_define_module( src/tests/datastore/unique_store_dictionary src/tests/datastore/unique_store_string_allocator src/tests/delegatelist + src/tests/detect_type_benchmark src/tests/dotproduct src/tests/drop-file-from-cache src/tests/dual_merge_director @@ -97,7 +99,6 @@ vespa_define_module( src/tests/sharedptr src/tests/signalhandler src/tests/simple_thread_bundle - src/tests/child_process src/tests/slime src/tests/slime/external_data_value src/tests/slime/summary-feature-benchmark diff --git a/vespalib/src/tests/detect_type_benchmark/.gitignore b/vespalib/src/tests/detect_type_benchmark/.gitignore new file mode 100644 index 00000000000..3d2fbf713c6 --- /dev/null +++ b/vespalib/src/tests/detect_type_benchmark/.gitignore @@ -0,0 +1 @@ +/vespalib_detect_type_benchmark_app diff --git a/vespalib/src/tests/detect_type_benchmark/CMakeLists.txt b/vespalib/src/tests/detect_type_benchmark/CMakeLists.txt new file mode 100644 index 00000000000..279622dd452 --- /dev/null +++ b/vespalib/src/tests/detect_type_benchmark/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_detect_type_benchmark_app TEST + SOURCES + detect_type_benchmark.cpp + DEPENDS + vespalib + GTest::GTest +) diff --git a/vespalib/src/tests/detect_type_benchmark/detect_type_benchmark.cpp b/vespalib/src/tests/detect_type_benchmark/detect_type_benchmark.cpp new file mode 100644 index 00000000000..44e15de2399 --- /dev/null +++ b/vespalib/src/tests/detect_type_benchmark/detect_type_benchmark.cpp @@ -0,0 +1,150 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include +#include +#include + +// Typically when you want a flexible way of identifying whether you +// are encountering a specific subclass, you try to dynamic_cast the +// pointer and check for a non-null return value. This is the most +// flexible way since it requires no extra code in the class itself +// and you will also detect any subclasses of the subclass you are +// testing for. Sometimes you only need to identify the exact class +// and speed in doing so is all that matters. This benchmark tries to +// isolate and measure the cost of different strategies. Note that +// dynamic_cast may be more expensive for more complex class +// hierarchies. + +using vespalib::BenchmarkTimer; + +constexpr int A_ID = 1; +constexpr int B_ID = 2; + +constexpr size_t LOOP_CNT = 1000000; + +class BaseClass { +private: + int _static_id; +public: + BaseClass(int id) : _static_id(id) {} + int static_id() const { return _static_id; } + virtual int dynamic_id() const = 0; + virtual ~BaseClass() {} +}; + +struct A : BaseClass { + A() : BaseClass(A_ID) {} + int dynamic_id() const override { return A_ID; } +}; + +struct B : BaseClass { + B() : BaseClass(B_ID) {} + int dynamic_id() const override { return B_ID; } +}; + +using is_A = bool (*)(const BaseClass *); + +//----------------------------------------------------------------------------- + +struct CheckType { + BaseClass *ptr; + is_A pred; + CheckType(BaseClass *ptr_in, is_A pred_in) : ptr(ptr_in), pred(pred_in) {} + void operator()() const { + bool result = pred(ptr); + (void) result; + } +}; + +struct Nop { + void operator()() const {} +}; + +//----------------------------------------------------------------------------- + +A a; +B b; +Nop nop; +double baseline = 0.0; +const auto &typeid_A = typeid(A); + +//----------------------------------------------------------------------------- + +bool always_true(const BaseClass *) __attribute__((noinline)); +bool always_true(const BaseClass *) { + return true; +} + +bool always_false(const BaseClass *) __attribute__((noinline)); +bool always_false(const BaseClass *) { + return false; +} + +//----------------------------------------------------------------------------- + +bool use_dynamic_cast(const BaseClass *) __attribute__((noinline)); +bool use_dynamic_cast(const BaseClass *ptr) { + return (dynamic_cast(ptr)); +} + +bool use_type_index(const BaseClass *) __attribute__((noinline)); +bool use_type_index(const BaseClass *ptr) { + return (std::type_index(typeid(*ptr)) == std::type_index(typeid_A)); +} + +bool use_type_id(const BaseClass *) __attribute__((noinline)); +bool use_type_id(const BaseClass *ptr) { + return (typeid(*ptr) == typeid_A); +} + +bool use_dynamic_id(const BaseClass *) __attribute__((noinline)); +bool use_dynamic_id(const BaseClass *ptr) { + return (ptr->dynamic_id() == A_ID); +} + +bool use_static_id(const BaseClass *) __attribute__((noinline)); +bool use_static_id(const BaseClass *ptr) { + return (ptr->static_id() == A_ID); +} + +//----------------------------------------------------------------------------- + +double estimate_cost_ns(CheckType check) { + return BenchmarkTimer::benchmark(check, nop, LOOP_CNT, 5.0) * 1000.0 * 1000.0 * 1000.0; +} + +void benchmark(const char *desc, is_A pred) { + EXPECT_TRUE(pred(&a)) << desc; + EXPECT_FALSE(pred(&b)) << desc; + CheckType yes(&a, pred); + CheckType no(&b, pred); + double t1 = estimate_cost_ns(yes); + double t2 = estimate_cost_ns(no); + double my_cost = ((t1 + t2) / 2.0) - baseline; + fprintf(stderr, "%s cost is %5.2f ns (true %5.2f, false %5.2f, baseline %5.2f)\n", + desc, my_cost, t1, t2, baseline); +} + +//----------------------------------------------------------------------------- + +TEST(DetectTypeBenchmark, find_baseline) { + CheckType check_true(&a, always_true); + CheckType check_false(&b, always_false); + double t1 = estimate_cost_ns(check_true); + double t2 = estimate_cost_ns(check_false); + baseline = (t1 + t2) / 2.0; + fprintf(stderr, "baseline cost is %5.2f ns (true %5.2f, false %5.2f)\n", + baseline, t1, t2); +} + +TEST(DetectTypeBenchmark, measure_overhead) { + benchmark("[dynamic_cast]", use_dynamic_cast); + benchmark(" [type_index]", use_type_index); + benchmark(" [typeid]", use_type_id); + benchmark(" [dynamic id]", use_dynamic_id); + benchmark(" [static id]", use_static_id); +} + +//----------------------------------------------------------------------------- + +GTEST_MAIN_RUN_ALL_TESTS() -- cgit v1.2.3 From e51cf8cde83e94a1ce33bdc1ff00301851f5f165 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 25 Sep 2020 12:15:04 +0200 Subject: Log on self-subscroption and fix copyright hreaders --- .../container/core/documentapi/DocumentAccessProvider.java | 1 + .../container/core/documentapi/VespaDocumentAccess.java | 1 + .../documentapi/messagebus/MessageBusDocumentAccess.java | 13 ++++++++++--- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java index 6eb80ae1aaf..5e3f873bba9 100644 --- a/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java +++ b/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.core.documentapi; import com.google.inject.Inject; diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java index fffa915a840..2918ffb2c80 100644 --- a/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java +++ b/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.core.documentapi; import com.yahoo.cloud.config.SlobroksConfig; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java index 3e35e9bd12a..8b1e25758ad 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java @@ -20,6 +20,8 @@ import com.yahoo.messagebus.network.local.LocalNetwork; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.logging.Level; +import java.util.logging.Logger; /** * This class implements the {@link DocumentAccess} interface using message bus for communication. @@ -29,6 +31,8 @@ import java.util.concurrent.ScheduledExecutorService; */ public class MessageBusDocumentAccess extends DocumentAccess { + private static final Logger log = Logger.getLogger(MessageBusDocumentAccess.class.getName()); + private final NetworkMessageBus bus; private final MessageBusParams params; @@ -60,9 +64,12 @@ public class MessageBusDocumentAccess extends DocumentAccess { bus = new NetworkMessageBus(network, new MessageBus(network, mbusParams)); } else { - bus = params.getRPCNetworkParams().getSlobroksConfig() != null && mbusParams.getMessageBusConfig() != null - ? new RPCMessageBus(mbusParams, params.getRPCNetworkParams()) // prefer without self-subscription if config is set - : new RPCMessageBus(mbusParams, params.getRPCNetworkParams(), params.getRoutingConfigId()); + if (params.getRPCNetworkParams().getSlobroksConfig() != null && mbusParams.getMessageBusConfig() != null) + bus = new RPCMessageBus(mbusParams, params.getRPCNetworkParams()); + else { + log.log(Level.INFO, "Setting up self-subscription to config because explicit config was missing; try to avoid this in containers"); + bus = new RPCMessageBus(mbusParams, params.getRPCNetworkParams(), params.getRoutingConfigId()); + } } } catch (Exception e) { -- cgit v1.2.3 From 77978067f2c6eddedfbb08ae7abc732f18adfd8e Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 25 Sep 2020 12:18:14 +0200 Subject: Refer to correct class i config model --- .../com/yahoo/vespa/model/container/ApplicationContainerCluster.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 2f33684c64e..7822b03db08 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -94,7 +94,7 @@ public final class ApplicationContainerCluster extends ContainerCluster Date: Fri, 25 Sep 2020 12:18:27 +0200 Subject: Remove reentrant lock no longer needed --- .../hosted/provision/restapi/LocksResponse.java | 20 ++++++----- .../main/java/com/yahoo/vespa/curator/Lock.java | 42 ++++++++-------------- .../yahoo/vespa/curator/stats/LockCounters.java | 4 +-- .../com/yahoo/vespa/curator/stats/LockInfo.java | 13 +++---- .../yahoo/vespa/curator/stats/ThreadLockInfo.java | 21 +++++------ 5 files changed, 41 insertions(+), 59 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java index 2e22631b830..56e3ed53ab2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java @@ -13,7 +13,6 @@ import java.io.IOException; import java.io.OutputStream; import java.time.Instant; import java.util.List; -import java.util.Map; import java.util.TreeMap; /** @@ -23,31 +22,36 @@ import java.util.TreeMap; */ public class LocksResponse extends HttpResponse { - private final Slime slime; + private final Slime slime = new Slime(); public LocksResponse() { + this(new TreeMap<>(ThreadLockInfo.getLockCountersByPath()), + ThreadLockInfo.getThreadLockInfos(), + ThreadLockInfo.getSlowLockInfos()); + } + + /** For testing */ + LocksResponse(TreeMap lockCountersByPath, + List threadLockInfos, + List slowLockInfos) { super(200); - this.slime = new Slime(); Cursor root = slime.setObject(); - Map lockCountersByPath = new TreeMap<>(ThreadLockInfo.getLockCountersByPath()); - Cursor lockPathsCursor = root.setArray("lock-paths"); lockCountersByPath.forEach((lockPath, lockCounters) -> { Cursor lockPathCursor = lockPathsCursor.addObject(); lockPathCursor.setString("path", lockPath); lockPathCursor.setLong("in-critical-region", lockCounters.inCriticalRegionCount()); lockPathCursor.setLong("invoke-acquire", lockCounters.invokeAcquireCount()); + lockPathCursor.setLong("acquire-failed", lockCounters.acquireFailedCount()); lockPathCursor.setLong("acquire-timed-out", lockCounters.acquireTimedOutCount()); lockPathCursor.setLong("lock-acquired", lockCounters.lockAcquiredCount()); lockPathCursor.setLong("locks-released", lockCounters.locksReleasedCount()); - lockPathCursor.setLong("acquire-reentrant-lock-errors", lockCounters.failedToAcquireReentrantLockCount()); lockPathCursor.setLong("no-locks-errors", lockCounters.noLocksErrorCount()); lockPathCursor.setLong("timeout-on-reentrancy-errors", lockCounters.timeoutOnReentrancyErrorCount()); }); - List threadLockInfos = ThreadLockInfo.getThreadLockInfos(); Cursor threadsCursor = root.setArray("threads"); int numberOfStackTraces = 0; for (var threadLockInfo : threadLockInfos) { @@ -67,7 +71,6 @@ public class LocksResponse extends HttpResponse { } } - List slowLockInfos = ThreadLockInfo.getSlowLockInfos(); Cursor slowLocksCursor = root.setArray("slow-locks"); slowLockInfos.forEach(lockInfo -> setLockInfo(slowLocksCursor.addObject(), lockInfo, true)); } @@ -88,7 +91,6 @@ public class LocksResponse extends HttpResponse { lockInfoCursor.setString("lock-path", lockInfo.getLockPath()); } lockInfoCursor.setString("invoke-acquire-time", toString(lockInfo.getTimeAcquiredWasInvoked())); - lockInfoCursor.setLong("reentrancy-hold-count-on-acquire", lockInfo.getThreadHoldCountOnAcquire()); lockInfoCursor.setString("acquire-timeout", lockInfo.getAcquireTimeout().toString()); lockInfo.getTimeLockWasAcquired().ifPresent(instant -> lockInfoCursor.setString("lock-acquired-time", toString(instant))); lockInfoCursor.setString("lock-state", lockInfo.getLockState().name()); diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java index da6e3109695..7fb6b88cf99 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java @@ -9,7 +9,6 @@ import org.apache.curator.framework.recipes.locks.InterProcessLock; import java.time.Duration; import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantLock; /** * A cluster-wide re-entrant mutex which is released on (the last symmetric) close. @@ -21,13 +20,11 @@ import java.util.concurrent.locks.ReentrantLock; */ public class Lock implements Mutex { - private final ReentrantLock lock; private final InterProcessLock mutex; private final String lockPath; public Lock(String lockPath, Curator curator) { this.lockPath = lockPath; - this.lock = new ReentrantLock(true); mutex = curator.createMutex(lockPath); } @@ -35,37 +32,26 @@ public class Lock implements Mutex { public void acquire(Duration timeout) throws UncheckedTimeoutException { ThreadLockInfo threadLockInfo = getThreadLockInfo(); threadLockInfo.invokingAcquire(timeout); - try { - if ( ! mutex.acquire(timeout.toMillis(), TimeUnit.MILLISECONDS)) { - threadLockInfo.acquireTimedOut(); - - throw new UncheckedTimeoutException("Timed out after waiting " + timeout + - " to acquire lock '" + lockPath + "'"); - } - threadLockInfo.lockAcquired(); - if ( ! lock.tryLock()) { // Should be available to only this thread, while holding the above mutex. - release(); - threadLockInfo.failedToAcquireReentrantLock(); - throw new IllegalStateException("InterProcessMutex acquired, but guarded lock held by someone else, for lock '" + lockPath + "'"); - } - } - catch (UncheckedTimeoutException | IllegalStateException e) { - throw e; - } - catch (Exception e) { + final boolean acquired; + try { + acquired = mutex.acquire(timeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (Exception e) { + threadLockInfo.acquireFailed(); throw new RuntimeException("Exception acquiring lock '" + lockPath + "'", e); } + + if (!acquired) { + threadLockInfo.acquireTimedOut(); + throw new UncheckedTimeoutException("Timed out after waiting " + timeout + + " to acquire lock '" + lockPath + "'"); + } + threadLockInfo.lockAcquired(); } @Override public void close() { - try { - lock.unlock(); - } - finally { - release(); - } + release(); } private void release() { @@ -79,7 +65,7 @@ public class Lock implements Mutex { } private ThreadLockInfo getThreadLockInfo() { - return ThreadLockInfo.getCurrentThreadLockInfo(lockPath, lock); + return ThreadLockInfo.getCurrentThreadLockInfo(lockPath); } } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java index 9052db5ce5a..4d1b48ef7f3 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java @@ -11,20 +11,20 @@ import java.util.concurrent.atomic.AtomicInteger; public class LockCounters { final AtomicInteger invokeAcquireCount = new AtomicInteger(0); final AtomicInteger inCriticalRegionCount = new AtomicInteger(0); + final AtomicInteger acquireFailedCount = new AtomicInteger(0); final AtomicInteger acquireTimedOutCount = new AtomicInteger(0); final AtomicInteger lockAcquiredCount = new AtomicInteger(0); final AtomicInteger locksReleasedCount = new AtomicInteger(0); - final AtomicInteger failedToAcquireReentrantLockCount = new AtomicInteger(0); final AtomicInteger noLocksErrorCount = new AtomicInteger(0); final AtomicInteger timeoutOnReentrancyErrorCount = new AtomicInteger(0); public int invokeAcquireCount() { return invokeAcquireCount.get(); } public int inCriticalRegionCount() { return inCriticalRegionCount.get(); } + public int acquireFailedCount() { return acquireFailedCount.get(); } public int acquireTimedOutCount() { return acquireTimedOutCount.get(); } public int lockAcquiredCount() { return lockAcquiredCount.get(); } public int locksReleasedCount() { return locksReleasedCount.get(); } - public int failedToAcquireReentrantLockCount() { return failedToAcquireReentrantLockCount.get(); } public int noLocksErrorCount() { return noLocksErrorCount.get(); } public int timeoutOnReentrancyErrorCount() { return timeoutOnReentrancyErrorCount.get(); } } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java index 37481a6b1c7..1e46b1cf668 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java @@ -17,7 +17,6 @@ public class LockInfo { private final Thread thread; private final String lockPath; - private final int threadHoldCountOnAcquire; private final Instant acquireInstant; private final Duration timeout; @@ -25,12 +24,12 @@ public class LockInfo { private volatile Optional terminalStateInstant = Optional.empty(); private volatile Optional stackTrace = Optional.empty(); - public static LockInfo invokingAcquire(Thread thread, String lockPath, int holdCount, Duration timeout) { - return new LockInfo(thread, lockPath, holdCount, timeout); + public static LockInfo invokingAcquire(Thread thread, String lockPath, Duration timeout) { + return new LockInfo(thread, lockPath, timeout); } public enum LockState { - ACQUIRING(false), TIMED_OUT(true), ACQUIRED(false), FAILED_TO_REENTER(true), RELEASED(true); + ACQUIRING(false), ACQUIRE_FAILED(true), TIMED_OUT(true), ACQUIRED(false), RELEASED(true); private final boolean terminal; @@ -41,17 +40,15 @@ public class LockInfo { private volatile LockState lockState = LockState.ACQUIRING; - private LockInfo(Thread thread, String lockPath, int threadHoldCountOnAcquire, Duration timeout) { + private LockInfo(Thread thread, String lockPath, Duration timeout) { this.thread = thread; this.lockPath = lockPath; - this.threadHoldCountOnAcquire = threadHoldCountOnAcquire; this.acquireInstant = Instant.now(); this.timeout = timeout; } public String getThreadName() { return thread.getName(); } public String getLockPath() { return lockPath; } - public int getThreadHoldCountOnAcquire() { return threadHoldCountOnAcquire; } public Instant getTimeAcquiredWasInvoked() { return acquireInstant; } public Duration getAcquireTimeout() { return timeout; } public LockState getLockState() { return lockState; } @@ -100,8 +97,8 @@ public class LockInfo { this.stackTrace = Optional.of(stackTrace.toString()); } + void acquireFailed() { setTerminalState(LockState.ACQUIRE_FAILED); } void timedOut() { setTerminalState(LockState.TIMED_OUT); } - void failedToAcquireReentrantLock() { setTerminalState(LockState.FAILED_TO_REENTER); } void released() { setTerminalState(LockState.RELEASED); } void lockAcquired() { diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java index 0ca9377d360..9f92b6444be 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java @@ -12,7 +12,6 @@ import java.util.PriorityQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; /** @@ -37,7 +36,6 @@ public class ThreadLockInfo { private final Thread thread; private final String lockPath; - private final ReentrantLock lock; private final LockCounters lockCountersForPath; /** The locks are reentrant so there may be more than 1 lock for this thread. */ @@ -54,19 +52,18 @@ public class ThreadLockInfo { } /** Returns the per-thread singleton ThreadLockInfo. */ - public static ThreadLockInfo getCurrentThreadLockInfo(String lockPath, ReentrantLock lock) { + public static ThreadLockInfo getCurrentThreadLockInfo(String lockPath) { return locks.computeIfAbsent( Thread.currentThread(), currentThread -> { LockCounters lockCounters = countersByLockPath.computeIfAbsent(lockPath, ignored -> new LockCounters()); - return new ThreadLockInfo(currentThread, lockPath, lock, lockCounters); + return new ThreadLockInfo(currentThread, lockPath, lockCounters); }); } - ThreadLockInfo(Thread currentThread, String lockPath, ReentrantLock lock, LockCounters lockCountersForPath) { + ThreadLockInfo(Thread currentThread, String lockPath, LockCounters lockCountersForPath) { this.thread = currentThread; this.lockPath = lockPath; - this.lock = lock; this.lockCountersForPath = lockCountersForPath; } @@ -78,7 +75,12 @@ public class ThreadLockInfo { public void invokingAcquire(Duration timeout) { lockCountersForPath.invokeAcquireCount.incrementAndGet(); lockCountersForPath.inCriticalRegionCount.incrementAndGet(); - lockInfos.add(LockInfo.invokingAcquire(thread, lockPath, lock.getHoldCount(), timeout)); + lockInfos.add(LockInfo.invokingAcquire(thread, lockPath, timeout)); + } + + /** Mutable method (see class doc) */ + public void acquireFailed() { + removeLastLockInfo(lockCountersForPath.acquireFailedCount, LockInfo::acquireFailed); } /** Mutable method (see class doc) */ @@ -97,11 +99,6 @@ public class ThreadLockInfo { getLastLockInfo().ifPresent(LockInfo::lockAcquired); } - /** Mutable method (see class doc) */ - public void failedToAcquireReentrantLock() { - removeLastLockInfo(lockCountersForPath.failedToAcquireReentrantLockCount, LockInfo::failedToAcquireReentrantLock); - } - /** Mutable method (see class doc) */ public void lockReleased() { removeLastLockInfo(lockCountersForPath.locksReleasedCount, LockInfo::released); -- cgit v1.2.3 From f55fed1e7da7b4ed4f4d35cb7359ec9d1ac5a6a2 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Fri, 25 Sep 2020 11:13:06 +0200 Subject: Remove unused argument --- .../com/yahoo/vespa/config/server/ApplicationRepository.java | 12 ++++-------- .../vespa/config/server/http/v2/ApplicationApiHandler.java | 4 +--- .../vespa/config/server/http/v2/SessionPrepareHandler.java | 3 +-- .../yahoo/vespa/config/server/ApplicationRepositoryTest.java | 8 ++++---- .../com/yahoo/vespa/config/server/deploy/DeployTester.java | 2 +- .../config/server/http/v2/SessionActiveHandlerTest.java | 3 +-- .../vespa/config/server/maintenance/MaintainerTester.java | 4 +--- 7 files changed, 13 insertions(+), 23 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 351ff262fa9..4839ccb800b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -279,7 +279,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye bootstrapping.set(false); } - public PrepareResult prepare(Tenant tenant, long sessionId, PrepareParams prepareParams, Instant now) { + public PrepareResult prepare(Tenant tenant, long sessionId, PrepareParams prepareParams) { validateThatLocalSessionIsNotActive(tenant, sessionId); LocalSession session = getLocalSession(tenant, sessionId); ApplicationId applicationId = prepareParams.getApplicationId(); @@ -292,11 +292,11 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return new PrepareResult(sessionId, deployment.configChangeActions(), logger); } - public PrepareResult deploy(CompressedApplicationInputStream in, PrepareParams prepareParams, Instant now) { + public PrepareResult deploy(CompressedApplicationInputStream in, PrepareParams prepareParams) { File tempDir = uncheck(() -> Files.createTempDirectory("deploy")).toFile(); PrepareResult prepareResult; try { - prepareResult = deploy(decompressApplication(in, tempDir), prepareParams, now); + prepareResult = deploy(decompressApplication(in, tempDir), prepareParams); } finally { cleanupTempDirectory(tempDir); } @@ -304,17 +304,13 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } public PrepareResult deploy(File applicationPackage, PrepareParams prepareParams) { - return deploy(applicationPackage, prepareParams, Instant.now()); - } - - public PrepareResult deploy(File applicationPackage, PrepareParams prepareParams, Instant now) { if (prepareParams.internalRestart() && hostProvisioner.isEmpty()) throw new IllegalArgumentException("Internal restart not supported without HostProvisioner"); ApplicationId applicationId = prepareParams.getApplicationId(); long sessionId = createSession(applicationId, prepareParams.getTimeoutBudget(), applicationPackage); Tenant tenant = getTenant(applicationId); - PrepareResult result = prepare(tenant, sessionId, prepareParams, now); + PrepareResult result = prepare(tenant, sessionId, prepareParams); activate(tenant, sessionId, prepareParams.getTimeoutBudget(), prepareParams.force()); if (prepareParams.internalRestart() && !result.configChangeActions().getRestartActions().isEmpty()) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java index d6badb8a9a2..9ea96b97af3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java @@ -13,11 +13,9 @@ import com.yahoo.vespa.config.server.application.CompressedApplicationInputStrea import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.http.Utils; import com.yahoo.vespa.config.server.session.PrepareParams; -import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import java.time.Duration; -import java.time.Instant; import static com.yahoo.vespa.config.server.application.CompressedApplicationInputStream.createFromCompressedStream; import static com.yahoo.vespa.config.server.http.Utils.checkThatTenantExists; @@ -56,7 +54,7 @@ public class ApplicationApiHandler extends SessionHandler { TenantName tenantName = validateTenant(request); PrepareParams prepareParams = PrepareParams.fromHttpRequest(request, tenantName, zookeeperBarrierTimeout); CompressedApplicationInputStream compressedStream = createFromCompressedStream(request.getData(), request.getHeader(contentTypeHeader)); - PrepareResult result = applicationRepository.deploy(compressedStream, prepareParams, Instant.now()); + PrepareResult result = applicationRepository.deploy(compressedStream, prepareParams); return new SessionPrepareAndActivateResponse(result, request, prepareParams.getApplicationId(), zone); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java index 4cb07e37f28..258af35be6f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java @@ -14,7 +14,6 @@ import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.http.Utils; import java.time.Duration; -import java.time.Instant; /** * A handler that prepares a session given by an id in the request. v2 of application API @@ -41,7 +40,7 @@ public class SessionPrepareHandler extends SessionHandler { TenantName tenantName = tenant.getName(); long sessionId = getSessionIdV2(request); PrepareParams prepareParams = PrepareParams.fromHttpRequest(request, tenantName, zookeeperBarrierTimeout); - PrepareResult result = applicationRepository.prepare(tenant, sessionId, prepareParams, Instant.now()); + PrepareResult result = applicationRepository.prepare(tenant, sessionId, prepareParams); return new SessionPrepareResponse(result, tenantName, request); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 0a49a19d728..e90ef38a92f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -559,7 +559,7 @@ public class ApplicationRepositoryTest { long firstSession = result.sessionId(); long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, testAppJdiscOnly); - applicationRepository.prepare(applicationRepository.getTenant(applicationId()), sessionId, prepareParams(), clock.instant()); + applicationRepository.prepare(applicationRepository.getTenant(applicationId()), sessionId, prepareParams()); exceptionRule.expect(RuntimeException.class); exceptionRule.expectMessage(containsString("Timeout exceeded when trying to activate 'test1.testapp'")); applicationRepository.activate(applicationRepository.getTenant(applicationId()), sessionId, new TimeoutBudget(clock, Duration.ofSeconds(0)), false); @@ -584,7 +584,7 @@ public class ApplicationRepositoryTest { PrepareResult result2 = deployApp(testAppJdiscOnly); result2.sessionId(); - applicationRepository.prepare(applicationRepository.getTenant(applicationId()), sessionId2, prepareParams(), clock.instant()); + applicationRepository.prepare(applicationRepository.getTenant(applicationId()), sessionId2, prepareParams()); exceptionRule.expect(ActivationConflictException.class); exceptionRule.expectMessage(containsString("tenant:test1 app:testapp:default Cannot activate session 3 because the currently active session (4) has changed since session 3 was created (was 2 at creation time)")); applicationRepository.activate(applicationRepository.getTenant(applicationId()), sessionId2, timeoutBudget, false); @@ -597,7 +597,7 @@ public class ApplicationRepositoryTest { exceptionRule.expect(IllegalStateException.class); exceptionRule.expectMessage(containsString("Session is active: 2")); - applicationRepository.prepare(applicationRepository.getTenant(applicationId()), sessionId, prepareParams(), clock.instant()); + applicationRepository.prepare(applicationRepository.getTenant(applicationId()), sessionId, prepareParams()); exceptionRule.expect(IllegalStateException.class); exceptionRule.expectMessage(containsString("tenant:test1 app:testapp:default Session 2 is already active")); @@ -705,7 +705,7 @@ public class ApplicationRepositoryTest { } private PrepareResult prepareAndActivate(File application) { - return applicationRepository.deploy(application, prepareParams(), Instant.now()); + return applicationRepository.deploy(application, prepareParams()); } private PrepareResult deployApp(File applicationPackage) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 3b6e6c00d8b..7553583e70c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -160,7 +160,7 @@ public class DeployTester { paramsBuilder.applicationId(applicationId) .timeoutBudget(new TimeoutBudget(clock, Duration.ofSeconds(60))); - return applicationRepository.deploy(new File(applicationPath), paramsBuilder.build(), now); + return applicationRepository.deploy(new File(applicationPath), paramsBuilder.build()); } public AllocatedHosts getAllocatedHostsOf(ApplicationId applicationId) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index 4ac1d633e75..511717acfc0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -141,8 +141,7 @@ public class SessionActiveHandlerTest { testApp); applicationRepository.prepare(tenant, sessionId, - new PrepareParams.Builder().applicationId(applicationId()).build(), - componentRegistry.getClock().instant()); + new PrepareParams.Builder().applicationId(applicationId()).build()); actResponse = handler.handle(createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.ACTIVE, sessionId, subPath)); LocalSession session = applicationRepository.getActiveLocalSession(tenant, applicationId()); metaData = session.getMetaData(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java index 0bc23b4d442..712242a69e6 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java @@ -36,10 +36,8 @@ class MaintainerTester { private final Curator curator; private final TenantRepository tenantRepository; private final ApplicationRepository applicationRepository; - private final Clock clock; MaintainerTester(Clock clock, TemporaryFolder temporaryFolder) throws IOException { - this.clock = clock; this.curator = new MockCurator(); InMemoryProvisioner hostProvisioner = new InMemoryProvisioner(true, "host0", "host1", "host2", "host3", "host4"); ProvisionerAdapter provisioner = new ProvisionerAdapter(hostProvisioner); @@ -68,7 +66,7 @@ class MaintainerTester { } void deployApp(File applicationPath, PrepareParams.Builder prepareParams) { - applicationRepository.deploy(applicationPath, prepareParams.ignoreValidationErrors(true).build(), clock.instant()); + applicationRepository.deploy(applicationPath, prepareParams.ignoreValidationErrors(true).build()); } Curator curator() { return curator; } -- cgit v1.2.3 From 1e3f05536e75344d4e5444b656ca5904387a4041 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Fri, 25 Sep 2020 12:02:52 +0200 Subject: Move internal restart to Deployment --- .../vespa/config/server/ApplicationRepository.java | 17 ----------------- .../vespa/config/server/deploy/Deployment.java | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 4839ccb800b..b9ee501bf67 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -304,29 +304,12 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } public PrepareResult deploy(File applicationPackage, PrepareParams prepareParams) { - if (prepareParams.internalRestart() && hostProvisioner.isEmpty()) - throw new IllegalArgumentException("Internal restart not supported without HostProvisioner"); - ApplicationId applicationId = prepareParams.getApplicationId(); long sessionId = createSession(applicationId, prepareParams.getTimeoutBudget(), applicationPackage); Tenant tenant = getTenant(applicationId); PrepareResult result = prepare(tenant, sessionId, prepareParams); activate(tenant, sessionId, prepareParams.getTimeoutBudget(), prepareParams.force()); - if (prepareParams.internalRestart() && !result.configChangeActions().getRestartActions().isEmpty()) { - Set hostnames = result.configChangeActions().getRestartActions().getEntries().stream() - .flatMap(entry -> entry.getServices().stream()) - .map(ServiceInfo::getHostName) - .collect(Collectors.toUnmodifiableSet()); - - hostProvisioner.get().restart(applicationId, HostFilter.from(hostnames, Set.of(), Set.of(), Set.of())); - result.deployLogger().log(Level.INFO, String.format("Scheduled service restart of %d nodes: %s", - hostnames.size(), hostnames.stream().sorted().collect(Collectors.joining(", ")))); - - ConfigChangeActions newActions = new ConfigChangeActions(new RestartActions(), result.configChangeActions().getRefeedActions()); - return new PrepareResult(result.sessionId(), newActions, result.deployLogger()); - } - return result; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index f04f25c23dc..89d960008dc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.deploy; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.Provisioner; @@ -12,6 +13,7 @@ import com.yahoo.vespa.config.server.ApplicationRepository.ActionTimer; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; +import com.yahoo.vespa.config.server.configchange.RestartActions; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.session.LocalSession; import com.yahoo.vespa.config.server.session.PrepareParams; @@ -23,8 +25,10 @@ import com.yahoo.vespa.curator.Lock; import java.time.Clock; import java.time.Duration; import java.util.Optional; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; import static com.yahoo.vespa.curator.Curator.CompletionWaiter; @@ -88,6 +92,9 @@ public class Deployment implements com.yahoo.config.provision.Deployment { public void prepare() { if (prepared) return; PrepareParams params = this.params.get(); + if (params.internalRestart() && provisioner.isEmpty()) + throw new IllegalArgumentException("Internal restart not supported without Provisioner"); + ApplicationId applicationId = params.getApplicationId(); try (ActionTimer timer = applicationRepository.timerFor(applicationId, "deployment.prepareMillis")) { Optional activeApplicationSet = applicationRepository.getCurrentActiveApplicationSet(tenant, applicationId); @@ -128,6 +135,20 @@ public class Deployment implements com.yahoo.config.provision.Deployment { ". Config generation " + session.getMetaData().getGeneration() + (previousActiveSession != null ? ". Based on session " + previousActiveSession.getSessionId() : "") + ". File references: " + applicationRepository.getFileReferences(applicationId)); + + if (params.internalRestart() && !configChangeActions.getRestartActions().isEmpty()) { + Set hostnames = configChangeActions.getRestartActions().getEntries().stream() + .flatMap(entry -> entry.getServices().stream()) + .map(ServiceInfo::getHostName) + .collect(Collectors.toUnmodifiableSet()); + + provisioner.get().restart(applicationId, HostFilter.from(hostnames, Set.of(), Set.of(), Set.of())); + log.log(Level.INFO, String.format("Scheduled service restart of %d nodes: %s", + hostnames.size(), hostnames.stream().sorted().collect(Collectors.joining(", ")))); + + this.configChangeActions = new ConfigChangeActions(new RestartActions(), configChangeActions.getRefeedActions()); + } + return session.getMetaData().getGeneration(); } } -- cgit v1.2.3 From 22accc554d2cbe3e07dc21a4e9af375468b95118 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Fri, 25 Sep 2020 12:25:03 +0200 Subject: Use same Deployment across prepare & activate --- .../yahoo/vespa/config/server/ApplicationRepository.java | 16 +++++++++++----- .../vespa/config/server/deploy/DeployHandlerLogger.java | 5 +++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index b9ee501bf67..9a1488a8a27 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -280,16 +280,21 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } public PrepareResult prepare(Tenant tenant, long sessionId, PrepareParams prepareParams) { + DeployHandlerLogger logger = DeployHandlerLogger.forPrepareParams(prepareParams); + Deployment deployment = prepare(tenant, sessionId, prepareParams, logger); + return new PrepareResult(sessionId, deployment.configChangeActions(), logger); + } + + private Deployment prepare(Tenant tenant, long sessionId, PrepareParams prepareParams, DeployHandlerLogger logger) { validateThatLocalSessionIsNotActive(tenant, sessionId); LocalSession session = getLocalSession(tenant, sessionId); ApplicationId applicationId = prepareParams.getApplicationId(); - DeployHandlerLogger logger = DeployHandlerLogger.forApplication(applicationId, prepareParams.isVerbose()); Deployment deployment = Deployment.unprepared(session, this, hostProvisioner, tenant, prepareParams, logger, clock); deployment.prepare(); logConfigChangeActions(deployment.configChangeActions(), logger); log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " prepared successfully. "); - return new PrepareResult(sessionId, deployment.configChangeActions(), logger); + return deployment; } public PrepareResult deploy(CompressedApplicationInputStream in, PrepareParams prepareParams) { @@ -307,10 +312,11 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye ApplicationId applicationId = prepareParams.getApplicationId(); long sessionId = createSession(applicationId, prepareParams.getTimeoutBudget(), applicationPackage); Tenant tenant = getTenant(applicationId); - PrepareResult result = prepare(tenant, sessionId, prepareParams); - activate(tenant, sessionId, prepareParams.getTimeoutBudget(), prepareParams.force()); + DeployHandlerLogger logger = DeployHandlerLogger.forPrepareParams(prepareParams); + Deployment deployment = prepare(tenant, sessionId, prepareParams, logger); + deployment.activate(); - return result; + return new PrepareResult(sessionId, deployment.configChangeActions(), logger); } /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java index b6cd22d78b4..110c6464eba 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.log.LogLevel; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.TenantRepository; import java.util.logging.Level; @@ -60,4 +61,8 @@ public class DeployHandlerLogger implements DeployLogger { public static DeployHandlerLogger forTenant(TenantName tenantName, boolean verbose) { return new DeployHandlerLogger(TenantRepository.logPre(tenantName), verbose); } + + public static DeployHandlerLogger forPrepareParams(PrepareParams prepareParams) { + return forApplication(prepareParams.getApplicationId(), prepareParams.isVerbose()); + } } -- cgit v1.2.3 From 27c498346fcf15c999f5c93ddf8e9e36e055a2b8 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Fri, 25 Sep 2020 12:30:10 +0200 Subject: Deploy with internal restart if flag is set in zone --- .../com/yahoo/vespa/config/server/ApplicationRepository.java | 8 +++++++- .../java/com/yahoo/vespa/config/server/deploy/Deployment.java | 11 ++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 9a1488a8a27..83779bcde77 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -64,7 +64,10 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.orchestrator.Orchestrator; @@ -127,6 +130,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private final LogRetriever logRetriever; private final TesterClient testerClient; private final Metric metric; + private final BooleanFlag deployWithInternalRestart; @Inject public ApplicationRepository(TenantRepository tenantRepository, @@ -176,6 +180,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye this.clock = Objects.requireNonNull(clock); this.testerClient = Objects.requireNonNull(testerClient); this.metric = Objects.requireNonNull(metric); + this.deployWithInternalRestart = Flags.DEPLOY_WITH_INTERNAL_RESTART.bindTo(flagSource); } public static class Builder { @@ -377,9 +382,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye SessionRepository sessionRepository = tenant.getSessionRepository(); LocalSession newSession = sessionRepository.createSessionFromExisting(activeSession, logger, true, timeoutBudget); sessionRepository.addLocalSession(newSession); + boolean internalRestart = deployWithInternalRestart.with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()).value(); return Optional.of(Deployment.unprepared(newSession, this, hostProvisioner, tenant, logger, timeout, clock, - false /* don't validate as this is already deployed */, bootstrap)); + false /* don't validate as this is already deployed */, bootstrap, internalRestart)); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 89d960008dc..81712f256fe 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -75,15 +75,15 @@ public class Deployment implements com.yahoo.config.provision.Deployment { public static Deployment unprepared(LocalSession session, ApplicationRepository applicationRepository, Optional provisioner, Tenant tenant, DeployLogger logger, - Duration timeout, Clock clock, boolean validate, boolean isBootstrap) { - Supplier params = createPrepareParams(clock, timeout, session, isBootstrap, !validate, false); + Duration timeout, Clock clock, boolean validate, boolean isBootstrap, boolean internalRestart) { + Supplier params = createPrepareParams(clock, timeout, session, isBootstrap, !validate, false, internalRestart); return new Deployment(session, applicationRepository, params, provisioner, tenant, logger, clock, false); } public static Deployment prepared(LocalSession session, ApplicationRepository applicationRepository, Optional provisioner, Tenant tenant, DeployLogger logger, Duration timeout, Clock clock, boolean isBootstrap, boolean force) { - Supplier params = createPrepareParams(clock, timeout, session, isBootstrap, false, force); + Supplier params = createPrepareParams(clock, timeout, session, isBootstrap, false, force, false); return new Deployment(session, applicationRepository, params, provisioner, tenant, logger, clock, true); } @@ -194,7 +194,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { */ private static Supplier createPrepareParams( Clock clock, Duration timeout, LocalSession session, - boolean isBootstrap, boolean ignoreValidationErrors, boolean force) { + boolean isBootstrap, boolean ignoreValidationErrors, boolean force, boolean internalRestart) { // Supplier because shouldn't/cant create this before validateSessionStatus() for prepared deployments // memoized because we want to create this once for unprepared deployments @@ -207,7 +207,8 @@ public class Deployment implements com.yahoo.config.provision.Deployment { .timeoutBudget(timeoutBudget) .ignoreValidationErrors(ignoreValidationErrors) .isBootstrap(isBootstrap) - .force(force); + .force(force) + .internalRestart(internalRestart); session.getDockerImageRepository().ifPresent(params::dockerImageRepository); session.getAthenzDomain().ifPresent(params::athenzDomain); -- cgit v1.2.3 From a2cfeb42a27c65b75aca1fad5aa795a2f8c5e1c2 Mon Sep 17 00:00:00 2001 From: Håvard Pettersen Date: Fri, 25 Sep 2020 10:30:41 +0000 Subject: inline typeid(A) --- vespalib/src/tests/detect_type_benchmark/detect_type_benchmark.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/vespalib/src/tests/detect_type_benchmark/detect_type_benchmark.cpp b/vespalib/src/tests/detect_type_benchmark/detect_type_benchmark.cpp index 44e15de2399..6d178093069 100644 --- a/vespalib/src/tests/detect_type_benchmark/detect_type_benchmark.cpp +++ b/vespalib/src/tests/detect_type_benchmark/detect_type_benchmark.cpp @@ -66,7 +66,6 @@ A a; B b; Nop nop; double baseline = 0.0; -const auto &typeid_A = typeid(A); //----------------------------------------------------------------------------- @@ -89,12 +88,12 @@ bool use_dynamic_cast(const BaseClass *ptr) { bool use_type_index(const BaseClass *) __attribute__((noinline)); bool use_type_index(const BaseClass *ptr) { - return (std::type_index(typeid(*ptr)) == std::type_index(typeid_A)); + return (std::type_index(typeid(*ptr)) == std::type_index(typeid(A))); } bool use_type_id(const BaseClass *) __attribute__((noinline)); bool use_type_id(const BaseClass *ptr) { - return (typeid(*ptr) == typeid_A); + return (typeid(*ptr) == typeid(A)); } bool use_dynamic_id(const BaseClass *) __attribute__((noinline)); -- cgit v1.2.3 From e37c257c1ca3d78f7ea00a808f1d0c710ae5e577 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Fri, 25 Sep 2020 12:34:58 +0200 Subject: Don't call commit() on visibility handler during replay. --- searchcore/src/vespa/searchcore/proton/server/documentdb.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 46bcb0e49bb..20f8853a722 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -911,7 +911,9 @@ DocumentDB::syncFeedView() IFeedView::SP newFeedView(_subDBs.getFeedView()); _writeService.sync(); - _visibility.commit(); + if (_state.getAllowReconfig()) { + _visibility.commit(); + } _writeService.sync(); _feedView.set(newFeedView); -- cgit v1.2.3 From 584d2f6d5d09832a27b1dce1d1e3e430e9fc46f3 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 25 Sep 2020 13:01:06 +0200 Subject: Avoid parsing IP addresses during deserialization `InetAddresses#forString` is among the slowest calls during deserialization. Do it only at serialization time. --- .../com/yahoo/vespa/hosted/provision/node/IP.java | 39 ++++++++++++---------- .../provision/persistence/NodeSerializer.java | 12 +++---- .../yahoo/vespa/hosted/provision/node/IPTest.java | 21 +++++++----- .../hosted/provision/restapi/NodesV2ApiTest.java | 2 +- 4 files changed, 41 insertions(+), 33 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java index 836583022ca..66061ad4e9f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java @@ -9,8 +9,7 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.persistence.NameResolver; -import java.net.Inet4Address; -import java.net.Inet6Address; +import java.net.InetAddress; import java.util.Collections; import java.util.Comparator; import java.util.HashSet; @@ -33,9 +32,9 @@ import static com.yahoo.config.provision.NodeType.proxyhost; public class IP { /** Comparator for sorting IP addresses by their natural order */ - public static final Comparator NATURAL_ORDER = (ip1, ip2) -> { - byte[] address1 = InetAddresses.forString(ip1).getAddress(); - byte[] address2 = InetAddresses.forString(ip2).getAddress(); + public static final Comparator NATURAL_ORDER = (ip1, ip2) -> { + byte[] address1 = ip1.getAddress(); + byte[] address2 = ip2.getAddress(); // IPv4 always sorts before IPv6 if (address1.length < address2.length) return -1; @@ -88,7 +87,7 @@ public class IP { /** Returns a copy of this with pool set to given value */ public Config with(Set primary) { - return new Config(require(primary), pool.asSet()); + return new Config(primary, pool.asSet()); } @Override @@ -110,16 +109,6 @@ public class IP { return String.format("ip config primary=%s pool=%s", primary, pool.asSet()); } - /** Validates and returns the given addresses */ - public static Set require(Set addresses) { - try { - addresses.forEach(InetAddresses::forString); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Found one or more invalid addresses in " + addresses, e); - } - return addresses; - } - /** * Verify IP config of given nodes * @@ -414,14 +403,28 @@ public class IP { } + /** Validate IP address*/ + public static InetAddress parse(String ipAddress) { + try { + return InetAddresses.forString(ipAddress); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid IP address '" + ipAddress + "'", e); + } + } + + /** Convert IP address to string. This uses :: for zero compression in IPv6 addresses. */ + public static String asString(InetAddress inetAddress) { + return InetAddresses.toAddrString(inetAddress); + } + /** Returns whether given string is an IPv4 address */ public static boolean isV4(String ipAddress) { - return InetAddresses.forString(ipAddress) instanceof Inet4Address; + return ipAddress.contains("."); } /** Returns whether given string is an IPv6 address */ public static boolean isV6(String ipAddress) { - return InetAddresses.forString(ipAddress) instanceof Inet6Address; + return ipAddress.contains(":"); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index f7521ba786d..397182e204e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -43,7 +43,6 @@ import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; -import java.util.function.UnaryOperator; /** * Serializes a node to/from JSON. @@ -145,8 +144,8 @@ public class NodeSerializer { private void toSlime(Node node, Cursor object) { object.setString(hostnameKey, node.hostname()); - toSlime(node.ipConfig().primary(), object.setArray(ipAddressesKey), IP.Config::require); - toSlime(node.ipConfig().pool().asSet(), object.setArray(ipAddressPoolKey), UnaryOperator.identity() /* Pool already holds a validated address list */); + toSlime(node.ipConfig().primary(), object.setArray(ipAddressesKey)); + toSlime(node.ipConfig().pool().asSet(), object.setArray(ipAddressPoolKey)); object.setString(idKey, node.id()); node.parentHostname().ifPresent(hostname -> object.setString(parentHostnameKey, hostname)); toSlime(node.flavor(), object); @@ -207,9 +206,10 @@ public class NodeSerializer { object.setString(agentKey, toString(event.agent())); } - private void toSlime(Set ipAddresses, Cursor array, UnaryOperator> validator) { - // Sorting IP addresses is expensive, so we do it at serialization time instead of Node construction time - validator.apply(ipAddresses).stream().sorted(IP.NATURAL_ORDER).forEach(array::addString); + private void toSlime(Set ipAddresses, Cursor array) { + // Validating IP address string literals is expensive, so we do it at serialization time instead of Node + // construction time + ipAddresses.stream().map(IP::parse).sorted(IP.NATURAL_ORDER).map(IP::asString).forEach(array::addString); } // ---------------- Deserialization -------------------------------------------------- diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java index 4602a8f3560..bf83b074387 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.provision.node; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.ImmutableSortedSet; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.LockedNodeList; @@ -11,14 +10,15 @@ import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import org.junit.Test; -import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -59,10 +59,14 @@ public class IPTest { "::1", "::10", "::20", - "2001:db8:0:0:0:0:0:ffff", - "2001:db8:85a3:0:0:8a2e:370:7334", - "2001:db8:95a3:0:0:0:0:7334"), - new ArrayList<>(ImmutableSortedSet.copyOf(IP.NATURAL_ORDER, ipAddresses)) + "2001:db8::ffff", + "2001:db8:85a3::8a2e:370:7334", + "2001:db8:95a3::7334"), + ipAddresses.stream() + .map(IP::parse) + .sorted(IP.NATURAL_ORDER) + .map(IP::asString) + .collect(Collectors.toList()) ); } @@ -99,7 +103,6 @@ public class IPTest { @Test public void test_find_allocation_ipv4_only() { var pool = testPool(false); - assertTrue(pool instanceof IP.Ipv4Pool); var allocation = pool.findAllocation(emptyList, resolver); assertFalse("Found allocation", allocation.isEmpty()); assertEquals("127.0.0.1", allocation.get().primary()); @@ -173,7 +176,9 @@ public class IPTest { .addReverseRecord("::2", "host1"); } - return node.ipConfig().pool(); + IP.Pool pool = node.ipConfig().pool(); + assertNotEquals(dualStack, pool instanceof IP.Ipv4Pool); + return pool; } private static Node createNode(Set ipAddresses) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index 1a354686ae4..9f6f5043ae8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -305,7 +305,7 @@ public class NodesV2ApiTest { ("[" + asNodeJson("host-with-ip.yahoo.com", "default", "foo") + "]"). getBytes(StandardCharsets.UTF_8), Request.Method.POST); - tester.assertResponse(req, 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Found one or more invalid addresses in [foo]: 'foo' is not an IP string literal.\"}"); + tester.assertResponse(req, 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Invalid IP address 'foo': 'foo' is not an IP string literal.\"}"); // Attempt to POST tenant node with already assigned IP tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", -- cgit v1.2.3 From cfa3aafbbff8c8da64bbce1df5885744b6389ab6 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Fri, 25 Sep 2020 13:09:42 +0200 Subject: Add feature flag to enable new restapi handler --- .../com/yahoo/config/model/api/ModelContext.java | 11 ++++++++--- .../vespa/model/clients/ContainerDocumentApi.java | 21 ++++++++++++++------- .../container/xml/DocumentApiOptionsBuilder.java | 3 ++- .../config/server/deploy/ModelContextImpl.java | 6 ++++++ .../src/main/java/com/yahoo/vespa/flags/Flags.java | 6 ++++++ 5 files changed, 36 insertions(+), 11 deletions(-) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 6a7a9ed16ae..0c78aafcf20 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -69,10 +69,10 @@ public interface ModelContext { default int defaultNumResponseThreads() { return 2; } - // TODO Revisit in May or June 2020 + // TODO(bjorncs) Temporary feature flag double threadPoolSizeFactor(); - // TODO Revisit in May or June 2020 + // TODO(bjorncs) Temporary feature flag double queueSizeFactor(); /// Default setting for the gc-options attribute if not specified explicit by application @@ -95,7 +95,9 @@ public interface ModelContext { // TODO Remove on 7.XXX when this is default on. boolean useDirectStorageApiRpc(); - default String proxyProtocol() { return "https+proxy-protocol"; } // TODO bjorncs: Remove after end of May + // TODO(bjorncs) Temporary feature flag + default String proxyProtocol() { return "https+proxy-protocol"; } + default Optional athenzDomain() { return Optional.empty(); } // TODO(mpolden): Remove after May 2020 @@ -113,6 +115,9 @@ public interface ModelContext { return Quota.empty(); } + // TODO(bjorncs): Temporary feature flag + default boolean useNewRestapiHandler() { return false; } + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java index feaa6eb5940..9018a0231db 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java @@ -41,13 +41,17 @@ public class ContainerDocumentApi { private static void addRestApiHandler(ContainerCluster cluster, Options options) { - var handler = newVespaClientHandler( - "com.yahoo.document.restapi.resource.RestApi", "/document/v1/*", options); + String handlerClassName = options.useNewRestapiHandler + ? "com.yahoo.document.restapi.resource.DocumentV1ApiHandler" + : "com.yahoo.document.restapi.resource.RestApi"; + var handler = newVespaClientHandler(handlerClassName, "/document/v1/*", options); cluster.addComponent(handler); - var executor = new Threadpool( - "restapi-handler", cluster, options.restApiThreadpoolOptions, options.feedThreadPoolSizeFactor); - handler.inject(executor); - handler.addComponent(executor); + if (!options.useNewRestapiHandler) { + var executor = new Threadpool( + "restapi-handler", cluster, options.restApiThreadpoolOptions, options.feedThreadPoolSizeFactor); + handler.inject(executor); + handler.addComponent(executor); + } } private static Handler> newVespaClientHandler( @@ -76,15 +80,18 @@ public class ContainerDocumentApi { private final ContainerThreadpool.UserOptions restApiThreadpoolOptions; private final ContainerThreadpool.UserOptions feedApiThreadpoolOptions; private final double feedThreadPoolSizeFactor; + private final boolean useNewRestapiHandler; public Options(Collection bindings, ContainerThreadpool.UserOptions restApiThreadpoolOptions, ContainerThreadpool.UserOptions feedApiThreadpoolOptions, - double feedThreadPoolSizeFactor) { + double feedThreadPoolSizeFactor, + boolean useNewRestapiHandler) { this.bindings = Collections.unmodifiableCollection(bindings); this.restApiThreadpoolOptions = restApiThreadpoolOptions; this.feedApiThreadpoolOptions = feedApiThreadpoolOptions; this.feedThreadPoolSizeFactor = feedThreadPoolSizeFactor; + this.useNewRestapiHandler = useNewRestapiHandler; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java index 99ae6184f5c..3baf792dfba 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java @@ -25,7 +25,8 @@ public class DocumentApiOptionsBuilder { getBindings(spec), threadpoolOptions(spec, "rest-api"), threadpoolOptions(spec, "http-client-api"), - deployState.getProperties().feedCoreThreadPoolSizeFactor()); + deployState.getProperties().feedCoreThreadPoolSizeFactor(), + deployState.getProperties().useNewRestapiHandler()); } private static ContainerThreadpool.UserOptions threadpoolOptions(Element spec, String elementName) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 2c6b30a2e6e..48d3fd6a176 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -168,6 +168,7 @@ public class ModelContextImpl implements ModelContext { private final Quota quota; private final boolean tlsUseFSync; private final String tlsCompressionType; + private final boolean useNewRestapiHandler; public Properties(ApplicationId applicationId, boolean multitenantFromConfig, @@ -234,6 +235,9 @@ public class ModelContextImpl implements ModelContext { feedCoreThreadPoolSizeFactor = Flags.FEED_CORE_THREAD_POOL_SIZE_FACTOR.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.quota = maybeQuota.orElseGet(Quota::empty); + this.useNewRestapiHandler = Flags.USE_NEW_RESTAPI_HANDLER.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()) + .value(); } @Override @@ -326,6 +330,8 @@ public class ModelContextImpl implements ModelContext { @Override public boolean tlsUseFSync() { return tlsUseFSync; } @Override public String tlsCompressionType() { return tlsCompressionType; } @Override public Quota quota() { return quota; } + + @Override public boolean useNewRestapiHandler() { return useNewRestapiHandler; } } } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index d42521e4f5e..87351e12108 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -416,6 +416,12 @@ public class Flags { "The number of deserialized Node objects to store in-memory.", "Takes effect on config server restart"); + public static final UnboundBooleanFlag USE_NEW_RESTAPI_HANDLER = defineFeatureFlag( + "use-restapi-handler", + false, + "Whether application containers should use the new restapi handler implementation", + "Takes effect on next internal redeployment"); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { -- cgit v1.2.3 From 620aea4f309ca930372906c27048e64b542dd942 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Fri, 25 Sep 2020 13:17:16 +0200 Subject: Add comment describing why commit() should not be called during replay. --- searchcore/src/vespa/searchcore/proton/server/documentdb.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 20f8853a722..c63785faa35 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -911,6 +911,13 @@ DocumentDB::syncFeedView() IFeedView::SP newFeedView(_subDBs.getFeedView()); _writeService.sync(); + /* + * Don't call commit() on visibility handler during transaction + * log replay since the serial number used for the commit will be + * too high until the replay is complete. This check can be + * removed again when feed handler has improved tracking of serial + * numbers during replay. + */ if (_state.getAllowReconfig()) { _visibility.commit(); } -- cgit v1.2.3 From 050c1c4c713c7de1fb3d6db51d84fa65ee40922e Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 25 Sep 2020 13:20:05 +0200 Subject: Fix comment --- .../yahoo/vespa/hosted/controller/application/SystemApplication.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index 7dd6126dabc..91be3a4df21 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -65,7 +65,7 @@ public enum SystemApplication { .orElse(false); } - /** Returns whether this should receive OS upgrades in given cloud */ + /** Returns whether this should receive OS upgrades in given zone */ public boolean shouldUpgradeOsIn(ZoneId zone, Controller controller) { if (controller.zoneRegistry().zones().reprovisionToUpgradeOs().ids().contains(zone)) { return nodeType == NodeType.host; // TODO(mpolden): Remove once all node types are supported -- cgit v1.2.3 From deefb5fac00e0cad03910f3ac926ac8c03842367 Mon Sep 17 00:00:00 2001 From: Tor Brede Vekterli Date: Fri, 25 Sep 2020 11:28:03 +0000 Subject: Automatically fallback to MessageBus if direct storage RPC not supported To avoid need to track this per peer, currently falls back to MBus for _all_ peers if any of them indicates it's on an old version that does not understand the new RPC method. Also fix test teardown race condition where RPC threads could still be running and invoking callbacks when the API RPC service had already been destroyed. --- .../rpc/storage_api_rpc_service_test.cpp | 75 +++++++++++++++++++--- .../storage/storageserver/communicationmanager.cpp | 2 +- .../storageserver/rpc/storage_api_rpc_service.cpp | 43 +++++++++++-- .../storageserver/rpc/storage_api_rpc_service.h | 7 ++ 4 files changed, 109 insertions(+), 18 deletions(-) diff --git a/storage/src/tests/storageserver/rpc/storage_api_rpc_service_test.cpp b/storage/src/tests/storageserver/rpc/storage_api_rpc_service_test.cpp index 69259ee08ec..ee70f265297 100644 --- a/storage/src/tests/storageserver/rpc/storage_api_rpc_service_test.cpp +++ b/storage/src/tests/storageserver/rpc/storage_api_rpc_service_test.cpp @@ -96,18 +96,18 @@ vespalib::string to_slobrok_id(const api::StorageMessageAddress& address) { return CachingRpcTargetResolver::address_to_slobrok_id(address); } -class StorageApiNode { +class RpcNode { +protected: vdstestlib::DirConfig _config; std::shared_ptr _doc_type_repo; std::shared_ptr _load_type_set; LockingMockOperationDispatcher _messages; std::unique_ptr _codec_provider; std::unique_ptr _shared_rpc_resources; - std::unique_ptr _service; api::StorageMessageAddress _node_address; vespalib::string _slobrok_id; public: - StorageApiNode(uint16_t node_index, bool is_distributor, const mbus::Slobrok& slobrok) + RpcNode(uint16_t node_index, bool is_distributor, const mbus::Slobrok& slobrok) : _config(getStandardConfig(true)), _doc_type_repo(document::TestDocRepo().getTypeRepoSp()), _load_type_set(std::make_shared()), @@ -122,13 +122,10 @@ public: _shared_rpc_resources = std::make_unique(_config.getConfigId(), 0, 1); // TODO make codec provider into interface so we can test decode-failures more easily? _codec_provider = std::make_unique(_doc_type_repo, _load_type_set); - StorageApiRpcService::Params params; - _service = std::make_unique(_messages, *_shared_rpc_resources, *_codec_provider, params); - - _shared_rpc_resources->start_server_and_register_slobrok(_slobrok_id); - // Explicitly wait until we are visible in Slobrok. Just waiting for mirror readiness is not enough. - wait_until_visible_in_slobrok(_slobrok_id); } + ~RpcNode(); + + const api::StorageMessageAddress& node_address() const noexcept { return _node_address; } void wait_until_visible_in_slobrok(vespalib::stringref id) { const auto deadline = std::chrono::steady_clock::now() + slobrok_register_timeout; @@ -139,8 +136,24 @@ public: std::this_thread::sleep_for(10ms); } } +}; - const api::StorageMessageAddress& node_address() const noexcept { return _node_address; } +RpcNode::~RpcNode() = default; + +class StorageApiNode : public RpcNode { + std::unique_ptr _service; +public: + StorageApiNode(uint16_t node_index, bool is_distributor, const mbus::Slobrok& slobrok) + : RpcNode(node_index, is_distributor, slobrok) + { + StorageApiRpcService::Params params; + _service = std::make_unique(_messages, *_shared_rpc_resources, *_codec_provider, params); + + _shared_rpc_resources->start_server_and_register_slobrok(_slobrok_id); + // Explicitly wait until we are visible in Slobrok. Just waiting for mirror readiness is not enough. + wait_until_visible_in_slobrok(_slobrok_id); + } + ~StorageApiNode(); std::shared_ptr create_dummy_put_command() const { auto doc_type = _doc_type_repo->getDocumentType("testdoctype1"); @@ -177,6 +190,26 @@ public: _messages.wait_until_n_messages_received(1); return _messages.pop_first_message(); } + + bool target_supports_direct_rpc(const api::StorageMessageAddress& addr) const noexcept { + return _service->target_supports_direct_rpc(addr); + } +}; + +StorageApiNode::~StorageApiNode() { + // Ensure we shut down the underlying RPC threads before destroying + // the RPC service that may receive callbacks from it. + _shared_rpc_resources->shutdown(); +} + +struct NodeWithoutStorageApiService : RpcNode { + NodeWithoutStorageApiService(uint16_t node_index, bool is_distributor, const mbus::Slobrok& slobrok) + : RpcNode(node_index, is_distributor, slobrok) + { + _shared_rpc_resources->start_server_and_register_slobrok(_slobrok_id); + // Explicitly wait until we are visible in Slobrok. Just waiting for mirror readiness is not enough. + wait_until_visible_in_slobrok(_slobrok_id); + } }; } // anonymous namespace @@ -304,4 +337,26 @@ TEST_F(StorageApiRpcServiceTest, response_trace_only_propagated_if_trace_level_s EXPECT_THAT(trace_str, Not(HasSubstr("Doing cool things"))); } +TEST_F(StorageApiRpcServiceTest, rpc_method_not_found_toggles_rpc_as_not_supported) { + NodeWithoutStorageApiService dummy_node(10, false, _slobrok); + _node_0->wait_until_visible_in_slobrok(to_slobrok_id(dummy_node.node_address())); + + // Initially we assume targets are on a new enough version to understand storage API RPCs. + EXPECT_TRUE(_node_0->target_supports_direct_rpc(dummy_node.node_address())); + EXPECT_TRUE(_node_0->target_supports_direct_rpc(_node_1->node_address())); + + // Send to an endpoint exposing RPC but not the Storage API server method. + // It will bounce back immediately with an FRT "no such method" error. + auto cmd = _node_0->create_dummy_put_command(); + cmd->setAddress(dummy_node.node_address()); + _node_0->send_request(cmd); + auto bounced_msg = _node_0->wait_and_receive_single_message(); + ASSERT_TRUE(bounced_msg); + + // For now (and for the sake of simplicity), fall back to assuming no targets + // support direct storage API RPC. + EXPECT_FALSE(_node_0->target_supports_direct_rpc(dummy_node.node_address())); + EXPECT_FALSE(_node_0->target_supports_direct_rpc(_node_1->node_address())); +} + } diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index 99fdb97e435..667c577645a 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -575,7 +575,7 @@ CommunicationManager::sendCommand( case api::StorageMessageAddress::STORAGE: { LOG(debug, "Send to %s: %s", address.toString().c_str(), msg->toString().c_str()); - if (_use_direct_storageapi_rpc) { + if (_use_direct_storageapi_rpc && _storage_api_rpc_service->target_supports_direct_rpc(address)) { _storage_api_rpc_service->send_rpc_v1_request(msg); } else { auto cmd = std::make_unique(msg); diff --git a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp index 68339e9c493..c62fd9e091d 100644 --- a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp @@ -33,7 +33,8 @@ StorageApiRpcService::StorageApiRpcService(MessageDispatcher& message_dispatcher _rpc_resources(rpc_resources), _message_codec_provider(message_codec_provider), _params(params), - _target_resolver(std::make_unique(_rpc_resources.slobrok_mirror(), _rpc_resources.target_factory())) + _target_resolver(std::make_unique(_rpc_resources.slobrok_mirror(), _rpc_resources.target_factory())), + _direct_rpc_supported(true) { register_server_methods(rpc_resources); } @@ -225,12 +226,7 @@ void StorageApiRpcService::RequestDone(FRT_RPCRequest* raw_req) { std::unique_ptr req(raw_req); auto* req_ctx = static_cast(req->GetContext()._value.VOIDP); if (!req->CheckReturnTypes("bixbix")) { - api::ReturnCode error = map_frt_error_to_storage_api_error(*req, *req_ctx); - LOG(debug, "Client: received rpc.v1 error response: %s", error.toString().c_str()); - auto error_reply = req_ctx->_originator_cmd->makeReply(); - error_reply->setResult(std::move(error)); - // TODO needs tracing of received-event! - _message_dispatcher.dispatch_sync(std::move(error_reply)); + handle_request_done_rpc_error(*req, *req_ctx); return; } LOG(debug, "Client: received rpc.v1 OK response"); @@ -259,6 +255,22 @@ void StorageApiRpcService::RequestDone(FRT_RPCRequest* raw_req) { _message_dispatcher.dispatch_sync(std::move(reply)); } +void StorageApiRpcService::handle_request_done_rpc_error(FRT_RPCRequest& req, + const RpcRequestContext& req_ctx) { + auto error_reply = req_ctx._originator_cmd->makeReply(); + api::ReturnCode error; + if (req.GetErrorCode() == FRTE_RPC_NO_SUCH_METHOD) { + mark_peer_without_direct_rpc_support(*req_ctx._originator_cmd->getAddress()); + error = api::ReturnCode(api::ReturnCode::NOT_CONNECTED, "Direct Storage RPC protocol not supported"); + } else { + error = map_frt_error_to_storage_api_error(req, req_ctx); + } + LOG(debug, "Client: received rpc.v1 error response: %s", error.toString().c_str()); + error_reply->setResult(std::move(error)); + // TODO needs tracing of received-event! + _message_dispatcher.dispatch_sync(std::move(error_reply)); +} + api::ReturnCode StorageApiRpcService::map_frt_error_to_storage_api_error(FRT_RPCRequest& req, const RpcRequestContext& req_ctx) { @@ -297,6 +309,23 @@ StorageApiRpcService::make_no_address_for_service_error(const api::StorageMessag return api::ReturnCode(error_code, std::move(error_msg)); } +void StorageApiRpcService::mark_peer_without_direct_rpc_support(const api::StorageMessageAddress& addr) { + bool expected = true; + if (_direct_rpc_supported.compare_exchange_strong(expected, false, std::memory_order_relaxed)) { + LOG(info, "Node %s does not support direct Storage API RPC; falling back " + "to legacy MessageBus protocol. Not logging this for any further nodes", + addr.toString().c_str()); + } +} + +bool StorageApiRpcService::target_supports_direct_rpc( + [[maybe_unused]] const api::StorageMessageAddress& addr) const noexcept { + // Stale reads isn't an issue here, since the worst case is just receiving + // a few more "no such method" errors. + return _direct_rpc_supported.load(std::memory_order_relaxed); +} + + /* * Major TODOs: * - tracing and trace propagation diff --git a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.h b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.h index 3fca08acc15..c8152ebfdbd 100644 --- a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.h +++ b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.h @@ -47,6 +47,7 @@ private: MessageCodecProvider& _message_codec_provider; const Params _params; std::unique_ptr _target_resolver; + std::atomic _direct_rpc_supported; public: StorageApiRpcService(MessageDispatcher& message_dispatcher, SharedRpcResources& rpc_resources, @@ -54,6 +55,8 @@ public: const Params& params); ~StorageApiRpcService() override; + [[nodiscard]] bool target_supports_direct_rpc(const api::StorageMessageAddress& addr) const noexcept; + void RPC_rpc_v1_send(FRT_RPCRequest* req); void encode_rpc_v1_response(FRT_RPCRequest& request, const api::StorageReply& reply); void send_rpc_v1_request(std::shared_ptr cmd); @@ -76,8 +79,12 @@ private: void encode_and_compress_rpc_payload(const MessageType& msg, FRT_Values& params); void RequestDone(FRT_RPCRequest* request) override; + void handle_request_done_rpc_error(FRT_RPCRequest& req, const RpcRequestContext& req_ctx); + api::ReturnCode map_frt_error_to_storage_api_error(FRT_RPCRequest& req, const RpcRequestContext& req_ctx); api::ReturnCode make_no_address_for_service_error(const api::StorageMessageAddress& addr) const; + + void mark_peer_without_direct_rpc_support(const api::StorageMessageAddress& addr); }; } // rpc -- cgit v1.2.3 From f8d5b66a8c3a474be1da3b1f4e782a883bbfd4ba Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Fri, 25 Sep 2020 13:30:19 +0200 Subject: Style fixes for storage link setup. --- .../vespa/storage/storageserver/servicelayernode.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/storage/src/vespa/storage/storageserver/servicelayernode.cpp b/storage/src/vespa/storage/storageserver/servicelayernode.cpp index 1b90e235d20..b64f6db60fc 100644 --- a/storage/src/vespa/storage/storageserver/servicelayernode.cpp +++ b/storage/src/vespa/storage/storageserver/servicelayernode.cpp @@ -227,29 +227,29 @@ ServiceLayerNode::createChain() StorageLink::UP chain; chain.reset(_communicationManager = new CommunicationManager(compReg, _configUri)); - chain->push_back(StorageLink::UP(new Bouncer(compReg, _configUri))); + chain->push_back(std::make_unique(compReg, _configUri)); if (_noUsablePartitionMode) { /* * No usable partitions. Use minimal chain. Still needs to be * able to report state back to cluster controller. */ - chain->push_back(StorageLink::UP(releaseStateManager().release())); + chain->push_back(releaseStateManager()); return chain; } - chain->push_back(StorageLink::UP(new OpsLogger(compReg, _configUri))); + chain->push_back(std::make_unique(compReg, _configUri)); auto* merge_throttler = new MergeThrottler(_configUri, compReg); chain->push_back(StorageLink::UP(merge_throttler)); - chain->push_back(StorageLink::UP(new ChangedBucketOwnershipHandler(_configUri, compReg))); - chain->push_back(StorageLink::UP(new StorageBucketDBInitializer( - _configUri, _partitions, getDoneInitializeHandler(), compReg))); - chain->push_back(StorageLink::UP(new BucketManager(_configUri, _context.getComponentRegister()))); + chain->push_back(std::make_unique(_configUri, compReg)); + chain->push_back(std::make_unique( + _configUri, _partitions, getDoneInitializeHandler(), compReg)); + chain->push_back(std::make_unique(_configUri, _context.getComponentRegister())); chain->push_back(StorageLink::UP(new VisitorManager( _configUri, _context.getComponentRegister(), *this, _externalVisitors))); - chain->push_back(StorageLink::UP(new ModifiedBucketChecker( - _context.getComponentRegister(), _persistenceProvider, _configUri))); + chain->push_back(std::make_unique( + _context.getComponentRegister(), _persistenceProvider, _configUri)); chain->push_back(StorageLink::UP(_fileStorManager = new FileStorManager( _configUri, _partitions, _persistenceProvider, _context.getComponentRegister()))); - chain->push_back(StorageLink::UP(releaseStateManager().release())); + chain->push_back(releaseStateManager()); // Lifetimes of all referenced components shall outlive the last call going // through the SPI, as queues are flushed and worker threads joined when -- cgit v1.2.3 From ca3beba075657aef53f55009b637257d9240561e Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Fri, 25 Sep 2020 13:43:00 +0200 Subject: Fix typo in flag name --- flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 87351e12108..5fb4dc436be 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -417,7 +417,7 @@ public class Flags { "Takes effect on config server restart"); public static final UnboundBooleanFlag USE_NEW_RESTAPI_HANDLER = defineFeatureFlag( - "use-restapi-handler", + "use-new-restapi-handler", false, "Whether application containers should use the new restapi handler implementation", "Takes effect on next internal redeployment"); -- cgit v1.2.3 From cb8ea8bed267836054cd5ebfd9c93985ddbdff12 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 25 Sep 2020 14:12:21 +0200 Subject: FINE log level on self subscription warning ̯๏๏ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java index 8b1e25758ad..9832529c157 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java @@ -67,7 +67,7 @@ public class MessageBusDocumentAccess extends DocumentAccess { if (params.getRPCNetworkParams().getSlobroksConfig() != null && mbusParams.getMessageBusConfig() != null) bus = new RPCMessageBus(mbusParams, params.getRPCNetworkParams()); else { - log.log(Level.INFO, "Setting up self-subscription to config because explicit config was missing; try to avoid this in containers"); + log.log(Level.FINE, () -> "Setting up self-subscription to config because explicit config was missing; try to avoid this in containers"); bus = new RPCMessageBus(mbusParams, params.getRPCNetworkParams(), params.getRoutingConfigId()); } } -- cgit v1.2.3 From c30cacf71b30236ec286e6f19a52e424423040e9 Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Fri, 25 Sep 2020 14:21:26 +0200 Subject: Fix typo in quota method name --- .../yahoo/vespa/hosted/controller/api/integration/billing/Quota.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Quota.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Quota.java index 9348e7dcbdb..4e1efc1b6d9 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Quota.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Quota.java @@ -50,7 +50,7 @@ public class Quota { return ZERO; } - public static Quota unlimted() { + public static Quota unlimited() { return UNLIMITED; } -- cgit v1.2.3 From a6fdc963ac309d894f0ed12159c566834e713c2d Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Fri, 25 Sep 2020 14:39:36 +0200 Subject: Run NodeFailer with larger interval If all config servers have application package run NodeFailer with 3 times larger interval --- .../com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java | 6 +++--- .../hosted/provision/maintenance/NodeRepositoryMaintenance.java | 5 ++++- .../yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java | 3 ++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index a2a189769bf..77ef88f0952 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -75,10 +75,10 @@ public class NodeFailer extends NodeRepositoryMaintainer { public NodeFailer(Deployer deployer, HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor, NodeRepository nodeRepository, - Duration downTimeLimit, Clock clock, Orchestrator orchestrator, + Duration downTimeLimit, Duration interval, Clock clock, Orchestrator orchestrator, ThrottlePolicy throttlePolicy, Metric metric) { - // check ping status every five minutes, but at least twice as often as the down time limit - super(nodeRepository, min(downTimeLimit.dividedBy(2), Duration.ofMinutes(5)), metric); + // check ping status every interval, but at least twice as often as the down time limit + super(nodeRepository, min(downTimeLimit.dividedBy(2), interval), metric); this.deployer = deployer; this.hostLivenessTracker = hostLivenessTracker; this.serviceMonitor = serviceMonitor; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 7f57575974f..452c3b1c6ca 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -73,7 +73,8 @@ public class NodeRepositoryMaintenance extends AbstractComponent { NodeMetrics nodeMetrics, NodeMetricsDb nodeMetricsDb) { DefaultTimes defaults = new DefaultTimes(zone, Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.bindTo(flagSource).value()); - nodeFailer = new NodeFailer(deployer, hostLivenessTracker, serviceMonitor, nodeRepository, defaults.failGrace, clock, orchestrator, throttlePolicyFromEnv().orElse(defaults.throttlePolicy), metric); + nodeFailer = new NodeFailer(deployer, hostLivenessTracker, serviceMonitor, nodeRepository, defaults.failGrace, + defaults.nodeFailerInterval, clock, orchestrator, throttlePolicyFromEnv().orElse(defaults.throttlePolicy), metric); periodicApplicationMaintainer = new PeriodicApplicationMaintainer(deployer, metric, nodeRepository, defaults.redeployMaintainerInterval, defaults.periodicRedeployInterval); operatorChangeApplicationMaintainer = new OperatorChangeApplicationMaintainer(deployer, metric, nodeRepository, defaults.operatorChangeRedeployInterval); reservationExpirer = new ReservationExpirer(nodeRepository, clock, defaults.reservationExpiry, metric); @@ -156,6 +157,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final Duration provisionedExpiry; private final Duration spareCapacityMaintenanceInterval; private final Duration metricsInterval; + private final Duration nodeFailerInterval; private final Duration retiredInterval; private final Duration infrastructureProvisionInterval; private final Duration loadBalancerExpirerInterval; @@ -176,6 +178,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { infrastructureProvisionInterval = Duration.ofMinutes(1); loadBalancerExpirerInterval = Duration.ofMinutes(5); metricsInterval = Duration.ofMinutes(1); + nodeFailerInterval = deploymentExistsOnAllConfigServers ? Duration.ofMinutes(15) : Duration.ofMinutes(5); nodeMetricsCollectionInterval = Duration.ofMinutes(1); operatorChangeRedeployInterval = deploymentExistsOnAllConfigServers ? Duration.ofMinutes(3) : Duration.ofMinutes(1); osUpgradeActivatorInterval = zone.system().isCd() ? Duration.ofSeconds(30) : Duration.ofMinutes(5); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java index 435dcdf9223..21f836d2663 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java @@ -199,7 +199,8 @@ public class NodeFailTester { } public NodeFailer createFailer() { - return new NodeFailer(deployer, hostLivenessTracker, serviceMonitor, nodeRepository, downtimeLimitOneHour, clock, orchestrator, NodeFailer.ThrottlePolicy.hosted, metric); + return new NodeFailer(deployer, hostLivenessTracker, serviceMonitor, nodeRepository, downtimeLimitOneHour, + Duration.ofMinutes(5), clock, orchestrator, NodeFailer.ThrottlePolicy.hosted, metric); } public void allNodesMakeAConfigRequestExcept(Node ... deadNodeArray) { -- cgit v1.2.3 From 886ce565ee3890bfe2adf0ac73f033c17d1fd072 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Fri, 25 Sep 2020 14:39:26 +0200 Subject: Log to deploy logger, not class logger --- .../java/com/yahoo/vespa/config/server/deploy/Deployment.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 81712f256fe..5f4f1298528 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -50,20 +50,20 @@ public class Deployment implements com.yahoo.config.provision.Deployment { private final Supplier params; private final Optional provisioner; private final Tenant tenant; - private final DeployLogger logger; + private final DeployLogger deployLogger; private final Clock clock; private boolean prepared; private ConfigChangeActions configChangeActions; private Deployment(LocalSession session, ApplicationRepository applicationRepository, Supplier params, - Optional provisioner, Tenant tenant, DeployLogger logger, Clock clock, boolean prepared) { + Optional provisioner, Tenant tenant, DeployLogger deployLogger, Clock clock, boolean prepared) { this.session = session; this.applicationRepository = applicationRepository; this.params = params; this.provisioner = provisioner; this.tenant = tenant; - this.logger = logger; + this.deployLogger = deployLogger; this.clock = clock; this.prepared = prepared; } @@ -99,7 +99,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { try (ActionTimer timer = applicationRepository.timerFor(applicationId, "deployment.prepareMillis")) { Optional activeApplicationSet = applicationRepository.getCurrentActiveApplicationSet(tenant, applicationId); this.configChangeActions = tenant.getSessionRepository().prepareLocalSession( - session, logger, params, activeApplicationSet, tenant.getPath(), clock.instant()); + session, deployLogger, params, activeApplicationSet, tenant.getPath(), clock.instant()); this.prepared = true; } } @@ -143,7 +143,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { .collect(Collectors.toUnmodifiableSet()); provisioner.get().restart(applicationId, HostFilter.from(hostnames, Set.of(), Set.of(), Set.of())); - log.log(Level.INFO, String.format("Scheduled service restart of %d nodes: %s", + deployLogger.log(Level.INFO, String.format("Scheduled service restart of %d nodes: %s", hostnames.size(), hostnames.stream().sorted().collect(Collectors.joining(", ")))); this.configChangeActions = new ConfigChangeActions(new RestartActions(), configChangeActions.getRefeedActions()); -- cgit v1.2.3 From 98c0ebaf4fcff534637f395167e1b766f873b71d Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Fri, 25 Sep 2020 14:53:15 +0200 Subject: Revert "Revert "Upgrade to Curator 4"" --- .../provision/persistence/CuratorDatabaseTest.java | 1 + parent/pom.xml | 4 +- zkfacade/pom.xml | 1 + .../main/java/com/yahoo/vespa/curator/Curator.java | 2 +- .../com/yahoo/vespa/curator/mock/MockCurator.java | 491 +++++++++++++++++++-- .../apache/curator/framework/api/package-info.java | 2 +- .../framework/api/transaction/package-info.java | 2 +- .../curator/framework/listen/package-info.java | 2 +- .../org/apache/curator/framework/package-info.java | 2 +- .../framework/recipes/atomic/package-info.java | 2 +- .../framework/recipes/barriers/package-info.java | 2 +- .../framework/recipes/cache/package-info.java | 2 +- .../framework/recipes/locks/package-info.java | 2 +- .../curator/framework/state/package-info.java | 2 +- .../main/java/org/apache/curator/package-info.java | 2 +- .../org/apache/curator/retry/package-info.java | 2 +- 16 files changed, 463 insertions(+), 58 deletions(-) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseTest.java index 149510bdc97..44026f835dd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseTest.java @@ -149,6 +149,7 @@ public class CuratorDatabaseTest { this.task = task; } + @SuppressWarnings("deprecation") @Override public org.apache.curator.framework.api.transaction.CuratorTransaction and(org.apache.curator.framework.api.transaction.CuratorTransaction transaction) { task.run(); diff --git a/parent/pom.xml b/parent/pom.xml index a9998af3962..da30e160d1b 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -757,9 +757,9 @@ zkfacade/src/main/java/org/apache/curator/**/package-info.java using something like find zkfacade/src/main/java/org/apache/curator -name package-info.java | \ - xargs perl -pi -e 's/major = [0-9]+, minor = [0-9]+, micro = [0-9]+/major = 2, minor = 9, micro = 1/g' + xargs perl -pi -e 's/major = [0-9]+, minor = [0-9]+, micro = [0-9]+/major = 4, minor = 3, micro = 0/g' --> - 2.13.0 + 4.3.0 4.5.2 3.6.1 5.6.2 diff --git a/zkfacade/pom.xml b/zkfacade/pom.xml index de542b89b67..c160d935788 100644 --- a/zkfacade/pom.xml +++ b/zkfacade/pom.xml @@ -48,6 +48,7 @@ org.apache.curator curator-test + 4.2.0 test diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java index 37b1fa1c9fb..51cb387b734 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java @@ -279,7 +279,7 @@ public class Curator implements AutoCloseable { */ public void createAtomically(Path... paths) { try { - CuratorTransaction transaction = framework().inTransaction(); + @SuppressWarnings("deprecation") CuratorTransaction transaction = framework().inTransaction(); for (Path path : paths) { if ( ! exists(path)) { transaction = transaction.create().forPath(path.getAbsolute(), new byte[0]).and(); diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java index 3da7678c44e..1f583ada7a1 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java @@ -12,31 +12,40 @@ import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.recipes.CuratorLockException; import org.apache.curator.CuratorZookeeperClient; import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.WatcherRemoveCuratorFramework; import org.apache.curator.framework.api.ACLBackgroundPathAndBytesable; import org.apache.curator.framework.api.ACLCreateModeBackgroundPathAndBytesable; import org.apache.curator.framework.api.ACLCreateModePathAndBytesable; +import org.apache.curator.framework.api.ACLCreateModeStatBackgroundPathAndBytesable; import org.apache.curator.framework.api.ACLPathAndBytesable; +import org.apache.curator.framework.api.ACLableExistBuilderMain; import org.apache.curator.framework.api.BackgroundCallback; import org.apache.curator.framework.api.BackgroundPathAndBytesable; import org.apache.curator.framework.api.BackgroundPathable; import org.apache.curator.framework.api.BackgroundVersionable; import org.apache.curator.framework.api.ChildrenDeletable; -import org.apache.curator.framework.api.CreateBackgroundModeACLable; +import org.apache.curator.framework.api.CreateBackgroundModeStatACLable; import org.apache.curator.framework.api.CreateBuilder; +import org.apache.curator.framework.api.CreateBuilder2; +import org.apache.curator.framework.api.CreateBuilderMain; +import org.apache.curator.framework.api.CreateProtectACLCreateModePathAndBytesable; import org.apache.curator.framework.api.CuratorListener; import org.apache.curator.framework.api.CuratorWatcher; import org.apache.curator.framework.api.DeleteBuilder; +import org.apache.curator.framework.api.DeleteBuilderMain; import org.apache.curator.framework.api.ErrorListenerPathAndBytesable; import org.apache.curator.framework.api.ErrorListenerPathable; import org.apache.curator.framework.api.ExistsBuilder; -import org.apache.curator.framework.api.ExistsBuilderMain; import org.apache.curator.framework.api.GetACLBuilder; import org.apache.curator.framework.api.GetChildrenBuilder; +import org.apache.curator.framework.api.GetConfigBuilder; import org.apache.curator.framework.api.GetDataBuilder; import org.apache.curator.framework.api.GetDataWatchBackgroundStatable; import org.apache.curator.framework.api.PathAndBytesable; import org.apache.curator.framework.api.Pathable; -import org.apache.curator.framework.api.ProtectACLCreateModePathAndBytesable; +import org.apache.curator.framework.api.ProtectACLCreateModeStatPathAndBytesable; +import org.apache.curator.framework.api.ReconfigBuilder; +import org.apache.curator.framework.api.RemoveWatchesBuilder; import org.apache.curator.framework.api.SetACLBuilder; import org.apache.curator.framework.api.SetDataBackgroundVersionable; import org.apache.curator.framework.api.SetDataBuilder; @@ -45,13 +54,16 @@ import org.apache.curator.framework.api.UnhandledErrorListener; import org.apache.curator.framework.api.VersionPathAndBytesable; import org.apache.curator.framework.api.WatchPathable; import org.apache.curator.framework.api.Watchable; +import org.apache.curator.framework.api.transaction.CuratorMultiTransaction; import org.apache.curator.framework.api.transaction.CuratorTransaction; import org.apache.curator.framework.api.transaction.CuratorTransactionBridge; import org.apache.curator.framework.api.transaction.CuratorTransactionFinal; import org.apache.curator.framework.api.transaction.CuratorTransactionResult; import org.apache.curator.framework.api.transaction.TransactionCheckBuilder; import org.apache.curator.framework.api.transaction.TransactionCreateBuilder; +import org.apache.curator.framework.api.transaction.TransactionCreateBuilder2; import org.apache.curator.framework.api.transaction.TransactionDeleteBuilder; +import org.apache.curator.framework.api.transaction.TransactionOp; import org.apache.curator.framework.api.transaction.TransactionSetDataBuilder; import org.apache.curator.framework.imps.CuratorFrameworkState; import org.apache.curator.framework.listen.Listenable; @@ -64,6 +76,8 @@ import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; import org.apache.curator.framework.recipes.locks.InterProcessLock; import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreMutex; +import org.apache.curator.framework.schema.SchemaSet; +import org.apache.curator.framework.state.ConnectionStateErrorPolicy; import org.apache.curator.framework.state.ConnectionStateListener; import org.apache.curator.utils.EnsurePath; import org.apache.zookeeper.CreateMode; @@ -71,6 +85,7 @@ import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Stat; +import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; import java.nio.file.Paths; import java.time.Duration; @@ -82,6 +97,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; @@ -633,64 +649,101 @@ public class MockCurator extends Curator { // ----- file system methods above. ----- // ----- There's nothing to see unless you are interested in an illustration of ----- // ----- the folly of fluent API's or, more generally, mankind. ----- + private abstract static class MockProtectACLCreateModeStatPathAndBytesable + implements ProtectACLCreateModeStatPathAndBytesable { - private abstract class MockBackgroundACLPathAndBytesableBuilder implements PathAndBytesable, ProtectACLCreateModePathAndBytesable { - - public BackgroundPathAndBytesable withACL(List list) { + public BackgroundPathAndBytesable withACL(List list) { throw new UnsupportedOperationException("Not implemented in MockCurator"); } - public ACLBackgroundPathAndBytesable withMode(CreateMode createMode) { + public BackgroundPathAndBytesable withACL(List list, boolean b) { throw new UnsupportedOperationException("Not implemented in MockCurator"); } - @Override - public ACLCreateModeBackgroundPathAndBytesable withProtection() { + public ProtectACLCreateModeStatPathAndBytesable withMode(CreateMode createMode) { throw new UnsupportedOperationException("Not implemented in MockCurator"); } - public T forPath(String s, byte[] bytes) throws Exception { - throw new UnsupportedOperationException("Not implemented in MockCurator"); + @Override + public ACLCreateModeBackgroundPathAndBytesable withProtection() { + return null; } - public T forPath(String s) throws Exception { - throw new UnsupportedOperationException("Not implemented in MockCurator"); + @Override + public ErrorListenerPathAndBytesable inBackground() { + return null; } - } + @Override + public ErrorListenerPathAndBytesable inBackground(Object o) { + return null; + } - private class MockCreateBuilder extends MockBackgroundACLPathAndBytesableBuilder implements CreateBuilder { + @Override + public ErrorListenerPathAndBytesable inBackground(BackgroundCallback backgroundCallback) { + return null; + } - private boolean createParents = false; - private CreateMode createMode = CreateMode.PERSISTENT; + @Override + public ErrorListenerPathAndBytesable inBackground(BackgroundCallback backgroundCallback, Object o) { + return null; + } @Override - public ProtectACLCreateModePathAndBytesable creatingParentsIfNeeded() { - createParents = true; - return this; + public ErrorListenerPathAndBytesable inBackground(BackgroundCallback backgroundCallback, Executor executor) { + return null; } @Override - public ACLCreateModeBackgroundPathAndBytesable withProtection() { - // Protection against the server crashing after creating the file but before returning to the client. - // Not relevant for an in-memory mock, obviously - return this; + public ErrorListenerPathAndBytesable inBackground(BackgroundCallback backgroundCallback, Object o, Executor executor) { + return null; } - public ACLBackgroundPathAndBytesable withMode(CreateMode createMode) { - this.createMode = createMode; - return this; + @Override + public ACLBackgroundPathAndBytesable storingStatIn(Stat stat) { + return null; } + } + + private class MockCreateBuilder implements CreateBuilder { + + private boolean createParents = false; + private CreateMode createMode = CreateMode.PERSISTENT; + @Override - public CreateBackgroundModeACLable compressed() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); + public ProtectACLCreateModeStatPathAndBytesable creatingParentsIfNeeded() { + createParents = true; + return new MockProtectACLCreateModeStatPathAndBytesable<>() { + + @Override + public String forPath(String s, byte[] bytes) throws Exception { + return createNode(s, bytes, createParents, createMode, fileSystem.root(), listeners); + } + + @Override + public String forPath(String s) throws Exception { + return createNode(s, new byte[0], createParents, createMode, fileSystem.root(), listeners); + } + + }; } @Override - public ProtectACLCreateModePathAndBytesable creatingParentContainersIfNeeded() { - // TODO: Add proper support for container nodes, see https://issues.apache.org/jira/browse/ZOOKEEPER-2163. - return creatingParentsIfNeeded(); + public ProtectACLCreateModeStatPathAndBytesable creatingParentContainersIfNeeded() { + return new MockProtectACLCreateModeStatPathAndBytesable<>() { + + @Override + public String forPath(String s, byte[] bytes) throws Exception { + return createNode(s, bytes, createParents, createMode, fileSystem.root(), listeners); + } + + @Override + public String forPath(String s) throws Exception { + return createNode(s, new byte[0], createParents, createMode, fileSystem.root(), listeners); + } + + }; } @Override @@ -699,6 +752,11 @@ public class MockCurator extends Curator { throw new UnsupportedOperationException("Not implemented in MockCurator"); } + @Override + public ACLCreateModeStatBackgroundPathAndBytesable withProtection() { + return null; + } + public String forPath(String s) throws Exception { return createNode(s, new byte[0], createParents, createMode, fileSystem.root(), listeners); } @@ -736,9 +794,50 @@ public class MockCurator extends Curator { public ErrorListenerPathAndBytesable inBackground(BackgroundCallback backgroundCallback, Object o, Executor executor) { throw new UnsupportedOperationException("Not implemented in MockCurator"); } + + @Override + public CreateBuilderMain withTtl(long l) { + return null; + } + + @Override + public CreateBuilder2 orSetData() { + return null; + } + + @Override + public CreateBuilder2 orSetData(int i) { + return null; + } + + @Override + public CreateBackgroundModeStatACLable compressed() { + return null; + } + + @Override + public CreateProtectACLCreateModePathAndBytesable storingStatIn(Stat stat) { + return null; + } + + @Override + public BackgroundPathAndBytesable withACL(List list) { + return null; + } + + @Override + public ACLBackgroundPathAndBytesable withMode(CreateMode createMode) { + this.createMode = createMode; + return this; + } + + @Override + public BackgroundPathAndBytesable withACL(List list, boolean b) { + return null; + } } - private class MockBackgroundPathableBuilder implements BackgroundPathable, Watchable> { + private static class MockBackgroundPathableBuilder implements BackgroundPathable, Watchable> { @Override public ErrorListenerPathable inBackground() { @@ -821,7 +920,12 @@ public class MockCurator extends Curator { } @Override - public ExistsBuilderMain creatingParentContainersIfNeeded() { + public ACLableExistBuilderMain creatingParentsIfNeeded() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ACLableExistBuilderMain creatingParentContainersIfNeeded() { throw new UnsupportedOperationException("Not implemented in MockCurator"); } } @@ -851,6 +955,10 @@ public class MockCurator extends Curator { return null; } + @Override + public DeleteBuilderMain quietly() { + return this; + } } private class MockGetDataBuilder extends MockBackgroundPathableBuilder implements GetDataBuilder { @@ -860,18 +968,48 @@ public class MockCurator extends Curator { throw new UnsupportedOperationException("Not implemented in MockCurator"); } + public byte[] forPath(String path) throws Exception { + return getData(path, fileSystem.root()); + } + @Override - public WatchPathable storingStatIn(Stat stat) { + public ErrorListenerPathable inBackground() { throw new UnsupportedOperationException("Not implemented in MockCurator"); } - public byte[] forPath(String path) throws Exception { - return getData(path, fileSystem.root()); + @Override + public ErrorListenerPathable inBackground(Object o) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathable inBackground(BackgroundCallback backgroundCallback) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathable inBackground(BackgroundCallback backgroundCallback, Object o) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathable inBackground(BackgroundCallback backgroundCallback, Executor executor) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathable inBackground(BackgroundCallback backgroundCallback, Object o, Executor executor) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); } + @Override + public WatchPathable storingStatIn(Stat stat) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } } - private class MockSetDataBuilder extends MockBackgroundACLPathAndBytesableBuilder implements SetDataBuilder { + // extends MockBackgroundACLPathAndBytesableBuilder + private class MockSetDataBuilder implements SetDataBuilder { @Override public SetDataBackgroundVersionable compressed() { @@ -889,6 +1027,11 @@ public class MockCurator extends Curator { return null; } + @Override + public Stat forPath(String s) throws Exception { + return null; + } + @Override public ErrorListenerPathAndBytesable inBackground() { throw new UnsupportedOperationException("Not implemented in MockCurator"); @@ -990,11 +1133,6 @@ public class MockCurator extends Curator { private CreateMode createMode = CreateMode.PERSISTENT; - @Override - public PathAndBytesable withACL(List list) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - @Override public ACLCreateModePathAndBytesable compressed() { throw new UnsupportedOperationException("Not implemented in MockCurator"); @@ -1018,6 +1156,20 @@ public class MockCurator extends Curator { return new MockCuratorTransactionBridge(); } + @Override + public TransactionCreateBuilder2 withTtl(long l) { + return this; + } + + @Override + public Object withACL(List list, boolean b) { + return this; + } + + @Override + public Object withACL(List list) { + return this; + } } private class MockTransactionDeleteBuilder implements TransactionDeleteBuilder { @@ -1154,11 +1306,31 @@ public class MockCurator extends Curator { throw new UnsupportedOperationException("Not implemented in MockCurator"); } + @Override + public ReconfigBuilder reconfig() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public GetConfigBuilder getConfig() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + @Override public CuratorTransaction inTransaction() { return new MockCuratorTransactionFinal(); } + @Override + public CuratorMultiTransaction transaction() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public TransactionOp transactionOp() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + @Override @Deprecated public void sync(String path, Object backgroundContextObject) { @@ -1227,11 +1399,242 @@ public class MockCurator extends Curator { } + @Override + public WatcherRemoveCuratorFramework newWatcherRemoveCuratorFramework() { + return new WatcherRemoveCuratorFramework() { + @Override + public void removeWatchers() { + + } + + @Override + public void start() { + + } + + @Override + public void close() { + + } + + @Override + public CuratorFrameworkState getState() { + return null; + } + + @Override + public boolean isStarted() { + return false; + } + + @Override + public CreateBuilder create() { + return null; + } + + @Override + public DeleteBuilder delete() { + return null; + } + + @Override + public ExistsBuilder checkExists() { + return null; + } + + @Override + public GetDataBuilder getData() { + return null; + } + + @Override + public SetDataBuilder setData() { + return null; + } + + @Override + public GetChildrenBuilder getChildren() { + return null; + } + + @Override + public GetACLBuilder getACL() { + return null; + } + + @Override + public SetACLBuilder setACL() { + return null; + } + + @Override + public ReconfigBuilder reconfig() { + return null; + } + + @Override + public GetConfigBuilder getConfig() { + return null; + } + + @Override + public CuratorTransaction inTransaction() { + return null; + } + + @Override + public CuratorMultiTransaction transaction() { + return null; + } + + @Override + public TransactionOp transactionOp() { + return null; + } + + @Override + public void sync(String s, Object o) { + + } + + @Override + public void createContainers(String s) throws Exception { + + } + + @Override + public SyncBuilder sync() { + return null; + } + + @Override + public RemoveWatchesBuilder watches() { + return null; + } + + @Override + public Listenable getConnectionStateListenable() { + return null; + } + + @Override + public Listenable getCuratorListenable() { + return null; + } + + @Override + public Listenable getUnhandledErrorListenable() { + return null; + } + + @Override + public CuratorFramework nonNamespaceView() { + return null; + } + + @Override + public CuratorFramework usingNamespace(String s) { + return null; + } + + @Override + public String getNamespace() { + return null; + } + + @Override + public CuratorZookeeperClient getZookeeperClient() { + return null; + } + + @Override + public EnsurePath newNamespaceAwareEnsurePath(String s) { + return null; + } + + @Override + public void clearWatcherReferences(Watcher watcher) { + + } + + @Override + public boolean blockUntilConnected(int i, TimeUnit timeUnit) throws InterruptedException { + return false; + } + + @Override + public void blockUntilConnected() throws InterruptedException { + + } + + @Override + public WatcherRemoveCuratorFramework newWatcherRemoveCuratorFramework() { + return null; + } + + @Override + public ConnectionStateErrorPolicy getConnectionStateErrorPolicy() { + return null; + } + + @Override + public QuorumVerifier getCurrentConfig() { + return null; + } + + @Override + public SchemaSet getSchemaSet() { + return null; + } + + @Override + public boolean isZk34CompatibilityMode() { + return false; + } + + @Override + public CompletableFuture runSafe(Runnable runnable) { + return null; + } + }; + + } + + @Override + public ConnectionStateErrorPolicy getConnectionStateErrorPolicy() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public QuorumVerifier getCurrentConfig() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public SchemaSet getSchemaSet() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public boolean isZk34CompatibilityMode() { + return false; + } + + @Override + public CompletableFuture runSafe(Runnable runnable) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + @Override public SyncBuilder sync() { throw new UnsupportedOperationException("Not implemented in MockCurator"); } - + + @Override + public RemoveWatchesBuilder watches() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + } } diff --git a/zkfacade/src/main/java/org/apache/curator/framework/api/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/api/package-info.java index be9e84013c1..e3da4ab3efa 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/api/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/api/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 2, minor = 13, micro = 0)) +@ExportPackage(version = @Version(major = 4, minor = 3, micro = 0)) package org.apache.curator.framework.api; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/api/transaction/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/api/transaction/package-info.java index be3ece0357b..94f8b12894e 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/api/transaction/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/api/transaction/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 2, minor = 13, micro = 0)) +@ExportPackage(version = @Version(major = 4, minor = 3, micro = 0)) package org.apache.curator.framework.api.transaction; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/listen/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/listen/package-info.java index 79c67cedf75..71ee8ccfff0 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/listen/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/listen/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 2, minor = 13, micro = 0)) +@ExportPackage(version = @Version(major = 4, minor = 3, micro = 0)) package org.apache.curator.framework.listen; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/package-info.java index 3e3b8433556..2999456bc9d 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 2, minor = 13, micro = 0)) +@ExportPackage(version = @Version(major = 4, minor = 3, micro = 0)) package org.apache.curator.framework; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/recipes/atomic/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/recipes/atomic/package-info.java index a607d5dcda5..dd1dd7a1899 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/recipes/atomic/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/recipes/atomic/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 2, minor = 13, micro = 0)) +@ExportPackage(version = @Version(major = 4, minor = 3, micro = 0)) package org.apache.curator.framework.recipes.atomic; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/recipes/barriers/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/recipes/barriers/package-info.java index 2db4beef75f..4e2aea367de 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/recipes/barriers/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/recipes/barriers/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 2, minor = 13, micro = 0)) +@ExportPackage(version = @Version(major = 4, minor = 3, micro = 0)) package org.apache.curator.framework.recipes.barriers; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/recipes/cache/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/recipes/cache/package-info.java index 0465bbf2039..ad6913d6381 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/recipes/cache/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/recipes/cache/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 2, minor = 13, micro = 0)) +@ExportPackage(version = @Version(major = 4, minor = 3, micro = 0)) package org.apache.curator.framework.recipes.cache; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/recipes/locks/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/recipes/locks/package-info.java index 63b067bcffc..4307c09e30a 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/recipes/locks/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/recipes/locks/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 2, minor = 13, micro = 0)) +@ExportPackage(version = @Version(major = 4, minor = 3, micro = 0)) package org.apache.curator.framework.recipes.locks; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/state/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/state/package-info.java index eec4f00ddb4..4a10e20318d 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/state/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/state/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 2, minor = 13, micro = 0)) +@ExportPackage(version = @Version(major = 4, minor = 3, micro = 0)) package org.apache.curator.framework.state; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/package-info.java b/zkfacade/src/main/java/org/apache/curator/package-info.java index 120aa4558d2..232a5fd46f3 100644 --- a/zkfacade/src/main/java/org/apache/curator/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 2, minor = 13, micro = 0)) +@ExportPackage(version = @Version(major = 4, minor = 3, micro = 0)) package org.apache.curator; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/retry/package-info.java b/zkfacade/src/main/java/org/apache/curator/retry/package-info.java index 98130481c4c..f45a0d927a5 100644 --- a/zkfacade/src/main/java/org/apache/curator/retry/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/retry/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 2, minor = 13, micro = 0)) +@ExportPackage(version = @Version(major = 4, minor = 3, micro = 0)) package org.apache.curator.retry; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; -- cgit v1.2.3 From ce446b7bc567d541b061d72ec0c5d2aa6fe7392e Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Fri, 25 Sep 2020 15:40:48 +0200 Subject: Adds method name to stack trace and adds timeout count and test --- zkfacade/abi-spec.json | 1 + zkfacade/pom.xml | 5 + .../main/java/com/yahoo/vespa/curator/Lock.java | 7 +- .../yahoo/vespa/curator/stats/LockCounters.java | 35 +++++++ .../com/yahoo/vespa/curator/stats/LockInfo.java | 4 +- .../yahoo/vespa/curator/stats/ThreadLockInfo.java | 8 +- .../com/yahoo/vespa/curator/stats/LockTest.java | 113 +++++++++++++++++++++ 7 files changed, 169 insertions(+), 4 deletions(-) create mode 100644 zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java diff --git a/zkfacade/abi-spec.json b/zkfacade/abi-spec.json index f4ad1ab4372..e026559b283 100644 --- a/zkfacade/abi-spec.json +++ b/zkfacade/abi-spec.json @@ -105,6 +105,7 @@ ], "methods": [ "public void (java.lang.String, com.yahoo.vespa.curator.Curator)", + "public void (java.lang.String, org.apache.curator.framework.recipes.locks.InterProcessLock)", "public void acquire(java.time.Duration)", "public void close()" ], diff --git a/zkfacade/pom.xml b/zkfacade/pom.xml index de542b89b67..7f335467751 100644 --- a/zkfacade/pom.xml +++ b/zkfacade/pom.xml @@ -76,6 +76,11 @@ zookeeper ${zookeeper.client.version} + + org.mockito + mockito-core + test + diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java index 7fb6b88cf99..9af83223ae6 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java @@ -24,8 +24,13 @@ public class Lock implements Mutex { private final String lockPath; public Lock(String lockPath, Curator curator) { + this(lockPath, curator.createMutex(lockPath)); + } + + /** Public for testing only */ + public Lock(String lockPath, InterProcessLock mutex) { this.lockPath = lockPath; - mutex = curator.createMutex(lockPath); + this.mutex = mutex; } /** Take the lock with the given timeout. This may be called multiple times from the same thread - each matched by a close */ diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java index 4d1b48ef7f3..5309bf755d7 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.curator.stats; +import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; /** @@ -27,4 +28,38 @@ public class LockCounters { public int locksReleasedCount() { return locksReleasedCount.get(); } public int noLocksErrorCount() { return noLocksErrorCount.get(); } public int timeoutOnReentrancyErrorCount() { return timeoutOnReentrancyErrorCount.get(); } + + @Override + public String toString() { + return "LockCounters{" + + "invokeAcquireCount=" + invokeAcquireCount + + ", inCriticalRegionCount=" + inCriticalRegionCount + + ", acquireFailedCount=" + acquireFailedCount + + ", acquireTimedOutCount=" + acquireTimedOutCount + + ", lockAcquiredCount=" + lockAcquiredCount + + ", locksReleasedCount=" + locksReleasedCount + + ", noLocksErrorCount=" + noLocksErrorCount + + ", timeoutOnReentrancyErrorCount=" + timeoutOnReentrancyErrorCount + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LockCounters that = (LockCounters) o; + return invokeAcquireCount.get() == that.invokeAcquireCount.get() && + inCriticalRegionCount.get() == that.inCriticalRegionCount.get() && + acquireFailedCount.get() == that.acquireFailedCount.get() && + acquireTimedOutCount.get() == that.acquireTimedOutCount.get() && + lockAcquiredCount.get() == that.lockAcquiredCount.get() && + locksReleasedCount.get() == that.locksReleasedCount.get() && + noLocksErrorCount.get() == that.noLocksErrorCount.get() && + timeoutOnReentrancyErrorCount.get() == that.timeoutOnReentrancyErrorCount.get(); + } + + @Override + public int hashCode() { + return Objects.hash(invokeAcquireCount, inCriticalRegionCount, acquireFailedCount, acquireTimedOutCount, lockAcquiredCount, locksReleasedCount, noLocksErrorCount, timeoutOnReentrancyErrorCount); + } } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java index 1e46b1cf668..e959dae2a93 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java @@ -78,8 +78,6 @@ public class LockInfo { // This method is public. If invoked concurrently, the this.stackTrace may be updated twice, // which is fine. - if (this.stackTrace.isPresent()) return; - var stackTrace = new StringBuilder(); StackTraceElement[] elements = thread.getStackTrace(); @@ -87,6 +85,8 @@ public class LockInfo { for (int i = 0; i < elements.length; ++i) { var element = elements[i]; stackTrace.append(element.getClassName()) + .append('.') + .append(element.getMethodName()) .append('(') .append(element.getFileName()) .append(':') diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java index 9f92b6444be..1e46a153164 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java @@ -61,6 +61,12 @@ public class ThreadLockInfo { }); } + static void clearStaticDataForTesting() { + locks.clear(); + completedLockInfos.clear(); + countersByLockPath.clear(); + } + ThreadLockInfo(Thread currentThread, String lockPath, LockCounters lockCountersForPath) { this.thread = currentThread; this.lockPath = lockPath; @@ -89,7 +95,7 @@ public class ThreadLockInfo { lockCountersForPath.timeoutOnReentrancyErrorCount.incrementAndGet(); } - removeLastLockInfo(lockCountersForPath.timeoutOnReentrancyErrorCount, LockInfo::timedOut); + removeLastLockInfo(lockCountersForPath.acquireTimedOutCount, LockInfo::timedOut); } /** Mutable method (see class doc) */ diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java new file mode 100644 index 00000000000..23b603eca5c --- /dev/null +++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java @@ -0,0 +1,113 @@ +package com.yahoo.vespa.curator.stats;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import com.yahoo.vespa.curator.Lock; +import org.apache.curator.framework.recipes.locks.InterProcessLock; +import org.junit.Before; +import org.junit.Test; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class LockTest { + private final InterProcessLock mutex = mock(InterProcessLock.class); + private final String lockPath = "/lock/path"; + private final Duration acquireTimeout = Duration.ofSeconds(10); + private final Lock lock = new Lock(lockPath, mutex); + + @Before + public void setUp() { + ThreadLockInfo.clearStaticDataForTesting(); + } + + @Test + public void acquireThrows() throws Exception { + Exception exception = new Exception("example curator exception"); + when(mutex.acquire(anyLong(), any())).thenThrow(exception); + + try { + lock.acquire(acquireTimeout); + fail(); + } catch (Exception e) { + assertSame(e.getCause(), exception); + } + + var expectedCounters = new LockCounters(); + expectedCounters.invokeAcquireCount.set(1); + expectedCounters.acquireFailedCount.set(1); + assertEquals(Map.of(lockPath, expectedCounters), ThreadLockInfo.getLockCountersByPath()); + + List slowLockInfos = ThreadLockInfo.getSlowLockInfos(); + assertEquals(1, slowLockInfos.size()); + LockInfo slowLockInfo = slowLockInfos.get(0); + assertEquals(acquireTimeout, slowLockInfo.getAcquireTimeout()); + Optional stackTrace = slowLockInfo.getStackTrace(); + assertTrue(stackTrace.isPresent()); + assertTrue("bad stacktrace: " + stackTrace.get(), stackTrace.get().contains(".Lock.acquire(Lock.java")); + assertEquals(LockInfo.LockState.ACQUIRE_FAILED, slowLockInfo.getLockState()); + assertTrue(slowLockInfo.getTimeTerminalStateWasReached().isPresent()); + + List threadLockInfos = ThreadLockInfo.getThreadLockInfos(); + assertEquals(1, threadLockInfos.size()); + ThreadLockInfo threadLockInfo = threadLockInfos.get(0); + assertEquals(0, threadLockInfo.getLockInfos().size()); + } + + @Test + public void acquireTimesOut() throws Exception { + when(mutex.acquire(anyLong(), any())).thenReturn(false); + + try { + lock.acquire(acquireTimeout); + fail(); + } catch (Exception e) { + assertTrue("unexpected exception: " + e.getMessage(), e.getMessage().contains("Timed out")); + } + + var expectedCounters = new LockCounters(); + expectedCounters.invokeAcquireCount.set(1); + expectedCounters.acquireTimedOutCount.set(1); + assertEquals(Map.of(lockPath, expectedCounters), ThreadLockInfo.getLockCountersByPath()); + } + + @Test + public void acquired() throws Exception { + when(mutex.acquire(anyLong(), any())).thenReturn(true); + + lock.acquire(acquireTimeout); + + var expectedCounters = new LockCounters(); + expectedCounters.invokeAcquireCount.set(1); + expectedCounters.lockAcquiredCount.set(1); + expectedCounters.inCriticalRegionCount.set(1); + assertEquals(Map.of(lockPath, expectedCounters), ThreadLockInfo.getLockCountersByPath()); + + // reenter + lock.acquire(acquireTimeout); + expectedCounters.invokeAcquireCount.set(2); + expectedCounters.lockAcquiredCount.set(2); + expectedCounters.inCriticalRegionCount.set(2); + + // inner-most closes + lock.close(); + expectedCounters.inCriticalRegionCount.set(1); + expectedCounters.locksReleasedCount.set(1); + assertEquals(Map.of(lockPath, expectedCounters), ThreadLockInfo.getLockCountersByPath()); + + // outer-most closes + lock.close(); + expectedCounters.inCriticalRegionCount.set(0); + expectedCounters.locksReleasedCount.set(2); + assertEquals(Map.of(lockPath, expectedCounters), ThreadLockInfo.getLockCountersByPath()); + } +} -- cgit v1.2.3 From 855d870737e945a25f99ef9737eafc6e3521c964 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Fri, 25 Sep 2020 15:43:03 +0200 Subject: Remove feature flag 'tls-insecure-authorization-mode' --- flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 5fb4dc436be..2776c374051 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -98,12 +98,6 @@ public class Flags { "Takes effect on next node agent tick. Change is orchestrated, but does NOT require container restart", HOSTNAME, APPLICATION_ID); - public static final UnboundStringFlag TLS_INSECURE_AUTHORIZATION_MODE = defineStringFlag( - "tls-insecure-authorization-mode", "log_only", - "TLS insecure authorization mode. Allowed values: ['disable', 'log_only', 'enforce']", - "Takes effect on restart of Docker container", - NODE_TYPE, APPLICATION_ID, HOSTNAME); - public static final UnboundIntFlag REBOOT_INTERVAL_IN_DAYS = defineIntFlag( "reboot-interval-in-days", 30, "No reboots are scheduled 0x-1x reboot intervals after the previous reboot, while reboot is " + -- cgit v1.2.3 From 8cc41e7cbdb125698d7c05baf064dc40541a2892 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Fri, 25 Sep 2020 15:55:27 +0200 Subject: Add storage chain builder. --- storage/src/vespa/storage/common/CMakeLists.txt | 1 + .../vespa/storage/common/i_storage_chain_builder.h | 22 ++++++++++++ .../vespa/storage/common/storage_chain_builder.cpp | 31 ++++++++++++++++ .../vespa/storage/common/storage_chain_builder.h | 23 ++++++++++++ .../storage/storageserver/distributornode.cpp | 30 ++++++++-------- .../vespa/storage/storageserver/distributornode.h | 2 +- .../storage/storageserver/servicelayernode.cpp | 42 +++++++++++----------- .../vespa/storage/storageserver/servicelayernode.h | 2 +- .../vespa/storage/storageserver/storagenode.cpp | 14 ++++++-- .../src/vespa/storage/storageserver/storagenode.h | 8 ++++- 10 files changed, 135 insertions(+), 40 deletions(-) create mode 100644 storage/src/vespa/storage/common/i_storage_chain_builder.h create mode 100644 storage/src/vespa/storage/common/storage_chain_builder.cpp create mode 100644 storage/src/vespa/storage/common/storage_chain_builder.h diff --git a/storage/src/vespa/storage/common/CMakeLists.txt b/storage/src/vespa/storage/common/CMakeLists.txt index 20b73987bf3..81c6486eaeb 100644 --- a/storage/src/vespa/storage/common/CMakeLists.txt +++ b/storage/src/vespa/storage/common/CMakeLists.txt @@ -15,5 +15,6 @@ vespa_add_library(storage_common OBJECT storagecomponent.cpp storagelink.cpp storagelinkqueued.cpp + storage_chain_builder.cpp DEPENDS ) diff --git a/storage/src/vespa/storage/common/i_storage_chain_builder.h b/storage/src/vespa/storage/common/i_storage_chain_builder.h new file mode 100644 index 00000000000..f50cc7572e9 --- /dev/null +++ b/storage/src/vespa/storage/common/i_storage_chain_builder.h @@ -0,0 +1,22 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include + +namespace storage { + +class StorageLink; + +/* + * Interface class for building a storage chain. + */ +class IStorageChainBuilder +{ +public: + virtual ~IStorageChainBuilder() = default; + virtual void add(std::unique_ptr child) = 0; + virtual std::unique_ptr build() && = 0; +}; + +} diff --git a/storage/src/vespa/storage/common/storage_chain_builder.cpp b/storage/src/vespa/storage/common/storage_chain_builder.cpp new file mode 100644 index 00000000000..45878f452cb --- /dev/null +++ b/storage/src/vespa/storage/common/storage_chain_builder.cpp @@ -0,0 +1,31 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "storage_chain_builder.h" +#include "storagelink.h" + +namespace storage { + +StorageChainBuilder::StorageChainBuilder() + : _top() +{ +} + +StorageChainBuilder::~StorageChainBuilder() = default; + +void +StorageChainBuilder::add(std::unique_ptr link) +{ + if (_top) { + _top->push_back(std::move(link)); + } else { + _top = std::move(link); + } +}; + +std::unique_ptr +StorageChainBuilder::build() && +{ + return std::move(_top); +} + +} diff --git a/storage/src/vespa/storage/common/storage_chain_builder.h b/storage/src/vespa/storage/common/storage_chain_builder.h new file mode 100644 index 00000000000..fc9ca7ec1c9 --- /dev/null +++ b/storage/src/vespa/storage/common/storage_chain_builder.h @@ -0,0 +1,23 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_storage_chain_builder.h" + +namespace storage { + +/** + * Class for building a storage chain. + */ +class StorageChainBuilder : public IStorageChainBuilder +{ +protected: + std::unique_ptr _top; +public: + StorageChainBuilder(); + ~StorageChainBuilder() override; + void add(std::unique_ptr child) override; + std::unique_ptr build() && override; +}; + +} diff --git a/storage/src/vespa/storage/storageserver/distributornode.cpp b/storage/src/vespa/storage/storageserver/distributornode.cpp index 1cd1477e769..3d1f9bbaf2e 100644 --- a/storage/src/vespa/storage/storageserver/distributornode.cpp +++ b/storage/src/vespa/storage/storageserver/distributornode.cpp @@ -5,6 +5,7 @@ #include "communicationmanager.h" #include "opslogger.h" #include "statemanager.h" +#include #include #include #include @@ -84,34 +85,33 @@ DistributorNode::handleConfigChange(vespa::config::content::core::StorVisitordis _context.getComponentRegister().setVisitorConfig(c); } -StorageLink::UP -DistributorNode::createChain() +void +DistributorNode::createChain(IStorageChainBuilder &builder) { DistributorComponentRegister& dcr(_context.getComponentRegister()); // TODO: All components in this chain should use a common thread instead of // each having its own configfetcher. StorageLink::UP chain; if (_retrievedCommunicationManager.get()) { - chain = std::move(_retrievedCommunicationManager); + builder.add(std::move(_retrievedCommunicationManager)); } else { - chain.reset(_communicationManager - = new CommunicationManager(dcr, _configUri)); + auto communication_manager = std::make_unique(dcr, _configUri); + _communicationManager = communication_manager.get(); + builder.add(std::move(communication_manager)); } std::unique_ptr stateManager(releaseStateManager()); - chain->push_back(StorageLink::UP(new Bouncer(dcr, _configUri))); - chain->push_back(StorageLink::UP(new OpsLogger(dcr, _configUri))); + builder.add(std::make_unique(dcr, _configUri)); + builder.add(std::make_unique(dcr, _configUri)); // Distributor instance registers a host info reporter with the state // manager, which is safe since the lifetime of said state manager // extends to the end of the process. - chain->push_back(StorageLink::UP( - new storage::distributor::Distributor( - dcr, *_threadPool, getDoneInitializeHandler(), - _manageActiveBucketCopies, - stateManager->getHostInfo()))); - - chain->push_back(StorageLink::UP(stateManager.release())); - return chain; + builder.add(std::make_unique + (dcr, *_threadPool, getDoneInitializeHandler(), + _manageActiveBucketCopies, + stateManager->getHostInfo())); + + builder.add(std::move(stateManager)); } api::Timestamp diff --git a/storage/src/vespa/storage/storageserver/distributornode.h b/storage/src/vespa/storage/storageserver/distributornode.h index 4db8876dc24..39614674bb5 100644 --- a/storage/src/vespa/storage/storageserver/distributornode.h +++ b/storage/src/vespa/storage/storageserver/distributornode.h @@ -49,7 +49,7 @@ public: private: void initializeNodeSpecific() override; - std::unique_ptr createChain() override; + void createChain(IStorageChainBuilder &builder) override; api::Timestamp getUniqueTimestamp() override; /** diff --git a/storage/src/vespa/storage/storageserver/servicelayernode.cpp b/storage/src/vespa/storage/storageserver/servicelayernode.cpp index b64f6db60fc..4f1db0e1b30 100644 --- a/storage/src/vespa/storage/storageserver/servicelayernode.cpp +++ b/storage/src/vespa/storage/storageserver/servicelayernode.cpp @@ -9,6 +9,7 @@ #include "statemanager.h" #include "priorityconverter.h" #include "service_layer_error_listener.h" +#include #include #include #include @@ -220,43 +221,44 @@ ServiceLayerNode::toDocumentPriority(uint8_t storagePriority) const return _communicationManager->getPriorityConverter().toDocumentPriority(storagePriority); } -StorageLink::UP -ServiceLayerNode::createChain() +void +ServiceLayerNode::createChain(IStorageChainBuilder &builder) { ServiceLayerComponentRegister& compReg(_context.getComponentRegister()); - StorageLink::UP chain; - chain.reset(_communicationManager = new CommunicationManager(compReg, _configUri)); - chain->push_back(std::make_unique(compReg, _configUri)); + auto communication_manager = std::make_unique(compReg, _configUri); + _communicationManager = communication_manager.get(); + builder.add(std::move(communication_manager)); + builder.add(std::make_unique(compReg, _configUri)); if (_noUsablePartitionMode) { /* * No usable partitions. Use minimal chain. Still needs to be * able to report state back to cluster controller. */ - chain->push_back(releaseStateManager()); - return chain; + builder.add(releaseStateManager()); + return; } - chain->push_back(std::make_unique(compReg, _configUri)); - auto* merge_throttler = new MergeThrottler(_configUri, compReg); - chain->push_back(StorageLink::UP(merge_throttler)); - chain->push_back(std::make_unique(_configUri, compReg)); - chain->push_back(std::make_unique( + builder.add(std::make_unique(compReg, _configUri)); + auto merge_throttler_up = std::make_unique(_configUri, compReg); + auto merge_throttler = merge_throttler_up.get(); + builder.add(std::move(merge_throttler_up)); + builder.add(std::make_unique(_configUri, compReg)); + builder.add(std::make_unique( _configUri, _partitions, getDoneInitializeHandler(), compReg)); - chain->push_back(std::make_unique(_configUri, _context.getComponentRegister())); - chain->push_back(StorageLink::UP(new VisitorManager( - _configUri, _context.getComponentRegister(), *this, _externalVisitors))); - chain->push_back(std::make_unique( + builder.add(std::make_unique(_configUri, _context.getComponentRegister())); + builder.add(std::make_unique(_configUri, _context.getComponentRegister(), static_cast(*this), _externalVisitors)); + builder.add(std::make_unique( _context.getComponentRegister(), _persistenceProvider, _configUri)); - chain->push_back(StorageLink::UP(_fileStorManager = new FileStorManager( - _configUri, _partitions, _persistenceProvider, _context.getComponentRegister()))); - chain->push_back(releaseStateManager()); + auto filstor_manager = std::make_unique(_configUri, _partitions, _persistenceProvider, _context.getComponentRegister()); + _fileStorManager = filstor_manager.get(); + builder.add(std::move(filstor_manager)); + builder.add(releaseStateManager()); // Lifetimes of all referenced components shall outlive the last call going // through the SPI, as queues are flushed and worker threads joined when // the storage link chain is closed prior to destruction. auto error_listener = std::make_shared(*_component, *merge_throttler); _fileStorManager->error_wrapper().register_error_listener(std::move(error_listener)); - return chain; } ResumeGuard diff --git a/storage/src/vespa/storage/storageserver/servicelayernode.h b/storage/src/vespa/storage/storageserver/servicelayernode.h index 7c0d6cd8ee3..ad570202f5b 100644 --- a/storage/src/vespa/storage/storageserver/servicelayernode.h +++ b/storage/src/vespa/storage/storageserver/servicelayernode.h @@ -62,7 +62,7 @@ private: void handleLiveConfigUpdate(const InitialGuard & initGuard) override; VisitorMessageSession::UP createSession(Visitor&, VisitorThread&) override; documentapi::Priority::Value toDocumentPriority(uint8_t storagePriority) const override; - std::unique_ptr createChain() override; + void createChain(IStorageChainBuilder &builder) override; void removeConfigSubscriptions() override; }; diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index e962ee4b1b6..aa50391c037 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -111,7 +112,8 @@ StorageNode::StorageNode( _newBucketSpacesConfig(), _component(), _configUri(configUri), - _communicationManager(nullptr) + _communicationManager(nullptr), + _chain_builder(std::make_unique()) { } @@ -203,7 +205,9 @@ StorageNode::initialize() _deadLockDetector->setWaitSlack(framework::MilliSecTime( static_cast(_serverConfig->deadLockDetectorTimeoutSlack * 1000))); - _chain.reset(createChain().release()); + createChain(*_chain_builder); + _chain = std::move(*_chain_builder).build(); + _chain_builder.reset(); assert(_communicationManager != nullptr); _communicationManager->updateBucketSpacesConfig(*_bucketSpacesConfig); @@ -622,4 +626,10 @@ StorageNode::releaseStateManager() { return std::move(_stateManager); } +void +StorageNode::set_storage_chain_builder(std::unique_ptr builder) +{ + _chain_builder = std::move(builder); +} + } // storage diff --git a/storage/src/vespa/storage/storageserver/storagenode.h b/storage/src/vespa/storage/storageserver/storagenode.h index ea1bf1027d4..91a2bae3190 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.h +++ b/storage/src/vespa/storage/storageserver/storagenode.h @@ -44,6 +44,7 @@ struct DeadLockDetector; struct StorageMetricSet; struct StorageNodeContext; class ApplicationGenerationFetcher; +class IStorageChainBuilder; class StorageComponent; namespace lib { class NodeType; } @@ -164,6 +165,9 @@ protected: std::unique_ptr _component; config::ConfigUri _configUri; CommunicationManager* _communicationManager; +private: + std::unique_ptr _chain_builder; +protected: /** * Node subclasses currently need to explicitly acquire ownership of state @@ -177,10 +181,12 @@ protected: void initialize(); virtual void subscribeToConfigs(); virtual void initializeNodeSpecific() = 0; - virtual std::unique_ptr createChain() = 0; + virtual void createChain(IStorageChainBuilder &builder) = 0; virtual void handleLiveConfigUpdate(const InitialGuard & initGuard); void shutdown(); virtual void removeConfigSubscriptions(); +public: + void set_storage_chain_builder(std::unique_ptr builder); }; } // storage -- cgit v1.2.3 From 87a73878b58b8af5a7b282216be99ce8f655dcd1 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Fri, 25 Sep 2020 17:17:46 +0200 Subject: Enable setting of storage chain builder for service layer process. --- storage/src/vespa/storage/common/i_storage_chain_builder.h | 2 +- storage/src/vespa/storage/common/storage_chain_builder.h | 2 +- .../src/vespa/storageserver/app/servicelayerprocess.cpp | 13 +++++++++++++ .../src/vespa/storageserver/app/servicelayerprocess.h | 3 +++ 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/storage/src/vespa/storage/common/i_storage_chain_builder.h b/storage/src/vespa/storage/common/i_storage_chain_builder.h index f50cc7572e9..8f4708ad7ac 100644 --- a/storage/src/vespa/storage/common/i_storage_chain_builder.h +++ b/storage/src/vespa/storage/common/i_storage_chain_builder.h @@ -15,7 +15,7 @@ class IStorageChainBuilder { public: virtual ~IStorageChainBuilder() = default; - virtual void add(std::unique_ptr child) = 0; + virtual void add(std::unique_ptr link) = 0; virtual std::unique_ptr build() && = 0; }; diff --git a/storage/src/vespa/storage/common/storage_chain_builder.h b/storage/src/vespa/storage/common/storage_chain_builder.h index fc9ca7ec1c9..ce4087e0bd0 100644 --- a/storage/src/vespa/storage/common/storage_chain_builder.h +++ b/storage/src/vespa/storage/common/storage_chain_builder.h @@ -16,7 +16,7 @@ protected: public: StorageChainBuilder(); ~StorageChainBuilder() override; - void add(std::unique_ptr child) override; + void add(std::unique_ptr link) override; std::unique_ptr build() && override; }; diff --git a/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp b/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp index 4ff3810d85f..bdb53ce6a60 100644 --- a/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp +++ b/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp @@ -2,6 +2,7 @@ #include "servicelayerprocess.h" #include +#include #include #include #include @@ -24,6 +25,9 @@ bool configured_to_use_btree_db(const config::ConfigUri& config_uri) { ServiceLayerProcess::ServiceLayerProcess(const config::ConfigUri& configUri) : Process(configUri), + _externalVisitors(), + _node(), + _storage_chain_builder(), _context(std::make_unique(), configured_to_use_btree_db(configUri)) { @@ -44,6 +48,9 @@ ServiceLayerProcess::createNode() _externalVisitors["searchvisitor"] = std::make_shared(_configUri); setupProvider(); _node = std::make_unique(_configUri, _context, *this, getProvider(), _externalVisitors); + if (_storage_chain_builder) { + _node->set_storage_chain_builder(std::move(_storage_chain_builder)); + } _node->init(); } @@ -62,4 +69,10 @@ ServiceLayerProcess::getComponentName() const { return "servicelayer"; } +void +ServiceLayerProcess::set_storage_chain_builder(std::unique_ptr builder) +{ + _storage_chain_builder = std::move(builder); +} + } // storage diff --git a/storageserver/src/vespa/storageserver/app/servicelayerprocess.h b/storageserver/src/vespa/storageserver/app/servicelayerprocess.h index b24640cbbd7..63776c02a52 100644 --- a/storageserver/src/vespa/storageserver/app/servicelayerprocess.h +++ b/storageserver/src/vespa/storageserver/app/servicelayerprocess.h @@ -27,10 +27,12 @@ namespace storage { namespace spi { struct PersistenceProvider; } class ServiceLayerNode; +class IStorageChainBuilder; class ServiceLayerProcess : public Process { VisitorFactory::Map _externalVisitors; std::unique_ptr _node; + std::unique_ptr _storage_chain_builder; protected: ServiceLayerNodeContext _context; @@ -48,6 +50,7 @@ public: StorageNode& getNode() override; StorageNodeContext& getContext() override; std::string getComponentName() const override; + void set_storage_chain_builder(std::unique_ptr builder); }; } // storage -- cgit v1.2.3 From 5a601ab4903746dd8dc12aee500d6cf442fb5459 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Fri, 25 Sep 2020 14:33:20 +0200 Subject: Add option to feed using storage api protocol directly on storage chain. --- searchcore/src/apps/vespa-feed-bm/CMakeLists.txt | 1 + .../storage_api_chain_bm_feed_handler.cpp | 168 +++++++++++++++++++++ .../storage_api_chain_bm_feed_handler.h | 29 ++++ .../src/apps/vespa-feed-bm/vespa_feed_bm.cpp | 46 ++++-- 4 files changed, 232 insertions(+), 12 deletions(-) create mode 100644 searchcore/src/apps/vespa-feed-bm/storage_api_chain_bm_feed_handler.cpp create mode 100644 searchcore/src/apps/vespa-feed-bm/storage_api_chain_bm_feed_handler.h diff --git a/searchcore/src/apps/vespa-feed-bm/CMakeLists.txt b/searchcore/src/apps/vespa-feed-bm/CMakeLists.txt index 1d031e2ea30..4ced3fe173b 100644 --- a/searchcore/src/apps/vespa-feed-bm/CMakeLists.txt +++ b/searchcore/src/apps/vespa-feed-bm/CMakeLists.txt @@ -4,6 +4,7 @@ vespa_add_executable(searchcore_vespa_feed_bm_app vespa_feed_bm.cpp spi_bm_feed_handler.cpp storage_api_rpc_bm_feed_handler.cpp + storage_api_chain_bm_feed_handler.cpp OUTPUT_NAME vespa-feed-bm DEPENDS searchcore_server diff --git a/searchcore/src/apps/vespa-feed-bm/storage_api_chain_bm_feed_handler.cpp b/searchcore/src/apps/vespa-feed-bm/storage_api_chain_bm_feed_handler.cpp new file mode 100644 index 00000000000..6ff8f0a2159 --- /dev/null +++ b/searchcore/src/apps/vespa-feed-bm/storage_api_chain_bm_feed_handler.cpp @@ -0,0 +1,168 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "storage_api_chain_bm_feed_handler.h" +#include "pending_tracker.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using document::Document; +using document::DocumentId; +using document::DocumentUpdate; +using storage::StorageLink; + +namespace feedbm { + +namespace { + +std::shared_ptr make_set_cluster_state_cmd() { + storage::lib::ClusterStateBundle bundle(storage::lib::ClusterState("version:2 distributor:1 storage:1")); + auto cmd = std::make_shared(bundle); + cmd->setPriority(storage::api::StorageMessage::VERYHIGH); + return cmd; +} + +} + +class BmStorageLink : public StorageLink +{ + std::mutex _mutex; + vespalib::hash_map _pending; +public: + BmStorageLink(); + ~BmStorageLink() override; + bool onDown(const std::shared_ptr& msg) override; + bool onUp(const std::shared_ptr& msg) override; + void retain(uint64_t msg_id, PendingTracker &tracker) { + tracker.retain(); + std::lock_guard lock(_mutex); + _pending.insert(std::make_pair(msg_id, &tracker)); + } + bool release(uint64_t msg_id) { + PendingTracker *tracker = nullptr; + { + std::lock_guard lock(_mutex); + auto itr = _pending.find(msg_id); + if (itr == _pending.end()) { + return false; + } + tracker = itr->second; + _pending.erase(itr); + } + tracker->release(); + return true; + } +}; + +BmStorageLink::BmStorageLink() + : storage::StorageLink("vespa-bm-feed"), + _mutex(), + _pending() +{ +} + +BmStorageLink::~BmStorageLink() +{ + std::lock_guard lock(_mutex); + assert(_pending.empty()); +} + +bool +BmStorageLink::onDown(const std::shared_ptr& msg) +{ + (void) msg; + return false; +} + +bool +BmStorageLink::onUp(const std::shared_ptr& msg) +{ + return release(msg->getMsgId()); +} + +class MyStorageChainBuilder : public storage::StorageChainBuilder +{ + using Parent = storage::StorageChainBuilder; +public: + MyStorageChainBuilder(); + ~MyStorageChainBuilder() override; + void add(std::unique_ptr link) override; +}; + +BmStorageLink *bm_link = nullptr; + +MyStorageChainBuilder::MyStorageChainBuilder() + : storage::StorageChainBuilder() +{ +} + +MyStorageChainBuilder::~MyStorageChainBuilder() = default; + +void +MyStorageChainBuilder::add(std::unique_ptr link) +{ + vespalib::string name = link->getName(); + Parent::add(std::move(link)); + if (name == "Communication manager") { + auto my_link = std::make_unique(); + bm_link = my_link.get(); + Parent::add(std::move(my_link)); + } +} + +StorageApiChainBmFeedHandler::StorageApiChainBmFeedHandler() + : IBmFeedHandler() +{ + auto cmd = make_set_cluster_state_cmd(); + PendingTracker tracker(1); + send_msg(std::move(cmd), tracker); + tracker.drain(); +} + +StorageApiChainBmFeedHandler::~StorageApiChainBmFeedHandler() = default; + +void +StorageApiChainBmFeedHandler::send_msg(std::shared_ptr cmd, PendingTracker& pending_tracker) +{ + cmd->setSourceIndex(0); + bm_link->retain(cmd->getMsgId(), pending_tracker); + bm_link->sendDown(std::move(cmd)); +} + +void +StorageApiChainBmFeedHandler::put(const document::Bucket& bucket, std::unique_ptr document, uint64_t timestamp, PendingTracker& tracker) +{ + auto cmd = std::make_unique(bucket, std::move(document), timestamp); + send_msg(std::move(cmd), tracker); +} + +void +StorageApiChainBmFeedHandler::update(const document::Bucket& bucket, std::unique_ptr document_update, uint64_t timestamp, PendingTracker& tracker) +{ + auto cmd = std::make_unique(bucket, std::move(document_update), timestamp); + send_msg(std::move(cmd), tracker); +} + +void +StorageApiChainBmFeedHandler::remove(const document::Bucket& bucket, const DocumentId& document_id, uint64_t timestamp, PendingTracker& tracker) +{ + auto cmd = std::make_unique(bucket, document_id, timestamp); + send_msg(std::move(cmd), tracker); +} + +std::unique_ptr +StorageApiChainBmFeedHandler::get_storage_chain_builder() +{ + return std::make_unique(); +} + +} diff --git a/searchcore/src/apps/vespa-feed-bm/storage_api_chain_bm_feed_handler.h b/searchcore/src/apps/vespa-feed-bm/storage_api_chain_bm_feed_handler.h new file mode 100644 index 00000000000..79c04c98de9 --- /dev/null +++ b/searchcore/src/apps/vespa-feed-bm/storage_api_chain_bm_feed_handler.h @@ -0,0 +1,29 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_bm_feed_handler.h" + +namespace storage { class IStorageChainBuilder; } +namespace storage::api { class StorageCommand; } + +namespace feedbm { + +/* + * Benchmark feed handler for feed to service layer using storage api protocol + * directly on the storage chain. + */ +class StorageApiChainBmFeedHandler : public IBmFeedHandler +{ + void send_msg(std::shared_ptr cmd, PendingTracker& tracker); +public: + StorageApiChainBmFeedHandler(); + ~StorageApiChainBmFeedHandler(); + void put(const document::Bucket& bucket, std::unique_ptr document, uint64_t timestamp, PendingTracker& tracker) override; + void update(const document::Bucket& bucket, std::unique_ptr document_update, uint64_t timestamp, PendingTracker& tracker) override; + void remove(const document::Bucket& bucket, const document::DocumentId& document_id, uint64_t timestamp, PendingTracker& tracker) override; + + static std::unique_ptr get_storage_chain_builder(); +}; + +} diff --git a/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp b/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp index 4208e9448ff..2c143c7ab23 100644 --- a/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp +++ b/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp @@ -3,6 +3,7 @@ #include "pending_tracker.h" #include "spi_bm_feed_handler.h" #include "storage_api_rpc_bm_feed_handler.h" +#include "storage_api_chain_bm_feed_handler.h" #include #include #include @@ -56,6 +57,7 @@ #include #include #include +#include #include #include #include @@ -122,6 +124,7 @@ using vespalib::makeLambdaTask; using feedbm::IBmFeedHandler; using feedbm::SpiBmFeedHandler; using feedbm::StorageApiRpcBmFeedHandler; +using feedbm::StorageApiChainBmFeedHandler; using DocumentDBMap = std::map>; @@ -209,6 +212,7 @@ class BMParams { uint32_t _remove_passes; uint32_t _rpc_network_threads; bool _enable_service_layer; + bool _use_storage_chain; uint32_t get_start(uint32_t thread_id) const { return (_documents / _threads) * thread_id + std::min(thread_id, _documents % _threads); } @@ -220,7 +224,8 @@ public: _update_passes(1), _remove_passes(2), _rpc_network_threads(1), - _enable_service_layer(false) + _enable_service_layer(false), + _use_storage_chain(false) { } BMRange get_range(uint32_t thread_id) const { @@ -233,6 +238,7 @@ public: uint32_t get_remove_passes() const { return _remove_passes; } uint32_t get_rpc_network_threads() const { return _rpc_network_threads; } bool get_enable_service_layer() const { return _enable_service_layer; } + bool get_use_storage_chain() const { return _use_storage_chain; } void set_documents(uint32_t documents_in) { _documents = documents_in; } void set_threads(uint32_t threads_in) { _threads = threads_in; } void set_put_passes(uint32_t put_passes_in) { _put_passes = put_passes_in; } @@ -240,6 +246,7 @@ public: void set_remove_passes(uint32_t remove_passes_in) { _remove_passes = remove_passes_in; } void set_rpc_network_threads(uint32_t threads_in) { _rpc_network_threads = threads_in; } void set_enable_service_layer(bool enable_service_layer_in) { _enable_service_layer = enable_service_layer_in; } + void set_use_storage_chain(bool use_storage_chain_in) { _use_storage_chain = use_storage_chain_in; } bool check() const; }; @@ -269,13 +276,13 @@ BMParams::check() const return true; } - class MyServiceLayerProcess : public storage::ServiceLayerProcess { PersistenceProvider& _provider; public: MyServiceLayerProcess(const config::ConfigUri & configUri, - PersistenceProvider &provider); + PersistenceProvider &provider, + bool use_storage_chain); ~MyServiceLayerProcess() override { shutdown(); } void shutdown() override; @@ -284,10 +291,14 @@ public: }; MyServiceLayerProcess::MyServiceLayerProcess(const config::ConfigUri & configUri, - PersistenceProvider &provider) + PersistenceProvider &provider, + bool use_storage_chain) : ServiceLayerProcess(configUri), _provider(provider) { + if (use_storage_chain) { + set_storage_chain_builder(StorageApiChainBmFeedHandler::get_storage_chain_builder()); + } } void @@ -476,7 +487,7 @@ struct PersistenceProviderFixture { std::unique_ptr make_document(uint32_t i) const; std::unique_ptr make_document_update(uint32_t i) const; void create_buckets(); - void start_service_layer(); + void start_service_layer(bool use_storage_chain); void shutdown_service_layer(); }; @@ -615,14 +626,15 @@ PersistenceProviderFixture::create_buckets() } void -PersistenceProviderFixture::start_service_layer() +PersistenceProviderFixture::start_service_layer(bool use_storage_chain) { LOG(info, "start slobrok"); _slobrok = std::make_unique(_slobrok_port); LOG(info, "start service layer"); config::ConfigUri config_uri("bm-servicelayer", _config_context); _service_layer = std::make_unique(config_uri, - *_persistence_engine); + *_persistence_engine, + use_storage_chain); _service_layer->setupConfig(100ms); _service_layer->createNode(); _service_layer->getNode().waitUntilInitialized(); @@ -630,7 +642,11 @@ PersistenceProviderFixture::start_service_layer() config::ConfigUri client_config_uri("bm-rpc-client", _config_context); _rpc_client_shared_rpc_resources = std::make_unique(client_config_uri, _rpc_client_port, 100); _rpc_client_shared_rpc_resources->start_server_and_register_slobrok("bm-rpc-client"); - _feed_handler = std::make_unique(*_rpc_client_shared_rpc_resources, _repo); + if (use_storage_chain) { + _feed_handler = std::make_unique(); + } else { + _feed_handler = std::make_unique(*_rpc_client_shared_rpc_resources, _repo); + } } void @@ -829,7 +845,7 @@ void benchmark_async_spi(const BMParams &bm_params) LOG(info, "create %u buckets", f.num_buckets()); f.create_buckets(); if (bm_params.get_enable_service_layer()) { - f.start_service_layer(); + f.start_service_layer(bm_params.get_use_storage_chain()); } vespalib::ThreadStackExecutor executor(bm_params.get_threads(), 128 * 1024); auto put_feed = make_feed(executor, bm_params, [&f](BMRange range) { return make_put_feed(f, range); }, "put"); @@ -881,7 +897,8 @@ App::usage() "[--update-passes update-passes]\n" "[--remove-passes remove-passes]\n" "[--rpc-network-threads threads]\n" - "[--enable-service-layer]" << std::endl; + "[--enable-service-layer]\n" + "[--use-storage-chain]" << std::endl; } bool @@ -897,7 +914,8 @@ App::get_options() { "update-passes", 1, nullptr, 0 }, { "remove-passes", 1, nullptr, 0 }, { "rpc-network-threads", 1, nullptr, 0 }, - { "enable-service-layer", 0, nullptr, 0 } + { "enable-service-layer", 0, nullptr, 0 }, + { "use-storage-chain", 0, nullptr, 0 } }; enum longopts_enum { LONGOPT_THREADS, @@ -906,7 +924,8 @@ App::get_options() LONGOPT_UPDATE_PASSES, LONGOPT_REMOVE_PASSES, LONGOPT_RPC_NETWORK_THREADS, - LONGOPT_ENABLE_SERVICE_LAYER + LONGOPT_ENABLE_SERVICE_LAYER, + LONGOPT_USE_STORAGE_CHAIN }; int opt_index = 1; resetOptIndex(opt_index); @@ -935,6 +954,9 @@ App::get_options() case LONGOPT_ENABLE_SERVICE_LAYER: _bm_params.set_enable_service_layer(true); break; + case LONGOPT_USE_STORAGE_CHAIN: + _bm_params.set_use_storage_chain(true); + break; default: return false; } -- cgit v1.2.3 From 3e6d66f8aa9cf112df88985577bf9493d0de8cd5 Mon Sep 17 00:00:00 2001 From: gjoranv Date: Fri, 25 Sep 2020 22:27:17 +0200 Subject: Remove test deps that were necessary when using jdisc_http_filters --- container-test/pom.xml | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/container-test/pom.xml b/container-test/pom.xml index 4b5008c8a6f..dfbe5f755ac 100644 --- a/container-test/pom.xml +++ b/container-test/pom.xml @@ -66,28 +66,5 @@ xercesImpl - - - commons-beanutils - commons-beanutils - 1.7.0 - - - - commons-logging - commons-logging - - - - - commons-beanutils - commons-beanutils-core - 1.8.0 - - - commons-digester - commons-digester - 1.8 - -- cgit v1.2.3 From 3097883d260238fdd883034f9448c4d63e08e3fc Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Fri, 25 Sep 2020 23:34:10 +0200 Subject: Fill stack trace for all threads in critical region --- .../java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java index 56e3ed53ab2..5b4e27119d8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java @@ -63,9 +63,7 @@ public class LocksResponse extends HttpResponse { Cursor lockInfosCursor = threadLockInfoCursor.setArray("active-locks"); for (var lockInfo : lockInfos) { - if (numberOfStackTraces++ < 10) { // Expensive to generate stack traces? - lockInfo.fillStackTrace(); - } + lockInfo.fillStackTrace(); setLockInfo(lockInfosCursor.addObject(), lockInfo, false); } } -- cgit v1.2.3 From e38ba64f8253746739674720035d11bdcda5e155 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Sat, 26 Sep 2020 00:20:28 +0200 Subject: Dump stack trace once per thread --- .../hosted/provision/restapi/LocksResponse.java | 3 +- .../com/yahoo/vespa/curator/stats/LockInfo.java | 34 +++++----------------- .../yahoo/vespa/curator/stats/ThreadLockInfo.java | 22 +++++++++++++- 3 files changed, 30 insertions(+), 29 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java index 5b4e27119d8..5a7414e894d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java @@ -53,17 +53,16 @@ public class LocksResponse extends HttpResponse { }); Cursor threadsCursor = root.setArray("threads"); - int numberOfStackTraces = 0; for (var threadLockInfo : threadLockInfos) { List lockInfos = threadLockInfo.getLockInfos(); if (!lockInfos.isEmpty()) { Cursor threadLockInfoCursor = threadsCursor.addObject(); threadLockInfoCursor.setString("thread-name", threadLockInfo.getThreadName()); threadLockInfoCursor.setString("lock-path", threadLockInfo.getLockPath()); + threadLockInfoCursor.setString("stack-trace", threadLockInfo.getStackTrace()); Cursor lockInfosCursor = threadLockInfoCursor.setArray("active-locks"); for (var lockInfo : lockInfos) { - lockInfo.fillStackTrace(); setLockInfo(lockInfosCursor.addObject(), lockInfo, false); } } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java index e959dae2a93..e9c238a40b9 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java @@ -15,8 +15,7 @@ import java.util.Optional; */ public class LockInfo { - private final Thread thread; - private final String lockPath; + private final ThreadLockInfo threadLockInfo; private final Instant acquireInstant; private final Duration timeout; @@ -24,8 +23,8 @@ public class LockInfo { private volatile Optional terminalStateInstant = Optional.empty(); private volatile Optional stackTrace = Optional.empty(); - public static LockInfo invokingAcquire(Thread thread, String lockPath, Duration timeout) { - return new LockInfo(thread, lockPath, timeout); + public static LockInfo invokingAcquire(ThreadLockInfo threadLockInfo, Duration timeout) { + return new LockInfo(threadLockInfo, timeout); } public enum LockState { @@ -40,15 +39,14 @@ public class LockInfo { private volatile LockState lockState = LockState.ACQUIRING; - private LockInfo(Thread thread, String lockPath, Duration timeout) { - this.thread = thread; - this.lockPath = lockPath; + private LockInfo(ThreadLockInfo threadLockInfo, Duration timeout) { + this.threadLockInfo = threadLockInfo; this.acquireInstant = Instant.now(); this.timeout = timeout; } - public String getThreadName() { return thread.getName(); } - public String getLockPath() { return lockPath; } + public String getThreadName() { return threadLockInfo.getThreadName(); } + public String getLockPath() { return threadLockInfo.getLockPath(); } public Instant getTimeAcquiredWasInvoked() { return acquireInstant; } public Duration getAcquireTimeout() { return timeout; } public LockState getLockState() { return lockState; } @@ -78,23 +76,7 @@ public class LockInfo { // This method is public. If invoked concurrently, the this.stackTrace may be updated twice, // which is fine. - var stackTrace = new StringBuilder(); - - StackTraceElement[] elements = thread.getStackTrace(); - // first stack frame is "java.lang.Thread(Thread.java:1606)" or "jdk.internal.misc.Unsafe(Unsafe.java:-2)"!? - for (int i = 0; i < elements.length; ++i) { - var element = elements[i]; - stackTrace.append(element.getClassName()) - .append('.') - .append(element.getMethodName()) - .append('(') - .append(element.getFileName()) - .append(':') - .append(element.getLineNumber()) - .append(")\n"); - } - - this.stackTrace = Optional.of(stackTrace.toString()); + this.stackTrace = Optional.of(threadLockInfo.getStackTrace()); } void acquireFailed() { setTerminalState(LockState.ACQUIRE_FAILED); } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java index 1e46a153164..b796cb9af43 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java @@ -75,13 +75,33 @@ public class ThreadLockInfo { public String getThreadName() { return thread.getName(); } public String getLockPath() { return lockPath; } + + public String getStackTrace() { + var stackTrace = new StringBuilder(); + + StackTraceElement[] elements = thread.getStackTrace(); + for (int i = 0; i < elements.length; ++i) { + var element = elements[i]; + stackTrace.append(element.getClassName()) + .append('.') + .append(element.getMethodName()) + .append('(') + .append(element.getFileName()) + .append(':') + .append(element.getLineNumber()) + .append(")\n"); + } + + return stackTrace.toString(); + } + public List getLockInfos() { return List.copyOf(lockInfos); } /** Mutable method (see class doc) */ public void invokingAcquire(Duration timeout) { lockCountersForPath.invokeAcquireCount.incrementAndGet(); lockCountersForPath.inCriticalRegionCount.incrementAndGet(); - lockInfos.add(LockInfo.invokingAcquire(thread, lockPath, timeout)); + lockInfos.add(LockInfo.invokingAcquire(this, timeout)); } /** Mutable method (see class doc) */ -- cgit v1.2.3 From 11ef6d3406c037ba8bdde9b42b9ba065fc5f37f7 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Sat, 26 Sep 2020 14:00:50 +0200 Subject: Avoid clang warnings. --- eval/src/vespa/eval/instruction/generic_join.h | 2 +- eval/src/vespa/eval/instruction/generic_rename.h | 2 +- eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp | 4 ++-- eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.cpp | 2 +- searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp | 1 - searchcore/src/vespa/searchcore/proton/server/feedhandler.h | 1 - searchcore/src/vespa/searchcore/proton/server/feedstate.h | 2 +- .../storage/storageserver/rpc/cluster_controller_api_rpc_service.cpp | 1 - .../storage/storageserver/rpc/cluster_controller_api_rpc_service.h | 1 - storage/src/vespa/storage/storageserver/rpc/rpc_target_factory.h | 2 +- 10 files changed, 7 insertions(+), 11 deletions(-) diff --git a/eval/src/vespa/eval/instruction/generic_join.h b/eval/src/vespa/eval/instruction/generic_join.h index 81e56494897..25647452dff 100644 --- a/eval/src/vespa/eval/instruction/generic_join.h +++ b/eval/src/vespa/eval/instruction/generic_join.h @@ -6,7 +6,7 @@ #include namespace vespalib { class Stash; } -namespace vespalib::eval { class ValueBuilderFactory; } +namespace vespalib::eval { struct ValueBuilderFactory; } namespace vespalib::eval::instruction { diff --git a/eval/src/vespa/eval/instruction/generic_rename.h b/eval/src/vespa/eval/instruction/generic_rename.h index e4f0146b9a8..ca9f45bd341 100644 --- a/eval/src/vespa/eval/instruction/generic_rename.h +++ b/eval/src/vespa/eval/instruction/generic_rename.h @@ -7,7 +7,7 @@ #include #include -namespace vespalib::eval { class ValueBuilderFactory; } +namespace vespalib::eval { struct ValueBuilderFactory; } namespace vespalib::eval::instruction { diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp index a78a7423520..fdfe5957a3f 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp @@ -54,8 +54,8 @@ PackedMappingsBuilder::target_memory(char *mem_start, char *mem_end) const ssize_t avail_sz = mem_end - mem_start; assert(needs_sz <= avail_sz); - uint32_t * int_store_mem = (uint32_t *) mem_start; - uint32_t * offsets_mem = (uint32_t *) (mem_start + int_store_size); + uint32_t * int_store_mem = (uint32_t *) (void *) mem_start; + uint32_t * offsets_mem = (uint32_t *) (void *) (mem_start + int_store_size); char * labels_mem = mem_start + int_store_size + label_offsets_size; ArrayRef int_store_data(int_store_mem, int_store_cnt); diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.cpp index d311d589a27..75b307b1aa7 100644 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.cpp +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.cpp @@ -39,7 +39,7 @@ PackedMixedTensorBuilder::build(std::unique_ptr>) // copy cells: memcpy(cells_mem, &_cells[0], cells_size); - ConstArrayRef cells((T *)cells_mem, _cells.size()); + ConstArrayRef cells((T *)(void *) cells_mem, _cells.size()); PackedMixedTensor * built = new (mem) PackedMixedTensor(_type, TypedCells(cells), mappings); diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 209a35ce4a2..13427d0e3aa 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -403,7 +403,6 @@ FeedHandler::FeedHandler(IThreadingService &writeService, _serialNum(0), _prunedSerialNum(0), _numPendingCommit(0), - _numCommitsCompleted(0), _delayedPrune(false), _feedLock(), _feedState(make_shared(getDocTypeName())), diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index 97629bfc018..395d11ba2fa 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -77,7 +77,6 @@ private: SerialNum _serialNum; SerialNum _prunedSerialNum; size_t _numPendingCommit; - size_t _numCommitsCompleted; bool _delayedPrune; mutable std::shared_mutex _feedLock; FeedStateSP _feedState; diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstate.h b/searchcore/src/vespa/searchcore/proton/server/feedstate.h index 6de1d7a4322..7d559eb4375 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedstate.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedstate.h @@ -11,7 +11,7 @@ namespace vespalib { namespace proton { class FeedOperation; -class PacketWrapper; +struct PacketWrapper; /** * Class representing the current state of a feed handler. diff --git a/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.cpp b/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.cpp index e34030f9bd7..4c7cf05241a 100644 --- a/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.cpp @@ -20,7 +20,6 @@ ClusterControllerApiRpcService::ClusterControllerApiRpcService( MessageDispatcher& message_dispatcher, SharedRpcResources& rpc_resources) : _message_dispatcher(message_dispatcher), - _rpc_resources(rpc_resources), _closed(false) { register_server_methods(rpc_resources); diff --git a/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.h b/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.h index 793644194dc..d5e753ff04b 100644 --- a/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.h +++ b/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.h @@ -24,7 +24,6 @@ class SharedRpcResources; class ClusterControllerApiRpcService : public FRT_Invokable { MessageDispatcher& _message_dispatcher; - SharedRpcResources& _rpc_resources; std::atomic _closed; public: static constexpr uint32_t StateBundleMaxUncompressedSize = 1024 * 1024 * 16; diff --git a/storage/src/vespa/storage/storageserver/rpc/rpc_target_factory.h b/storage/src/vespa/storage/storageserver/rpc/rpc_target_factory.h index 8411a273dc2..ef438e96305 100644 --- a/storage/src/vespa/storage/storageserver/rpc/rpc_target_factory.h +++ b/storage/src/vespa/storage/storageserver/rpc/rpc_target_factory.h @@ -6,7 +6,7 @@ namespace storage::rpc { -class RpcTarget; +struct RpcTarget; /** * Factory for creating instances of RpcTarget based on a connection spec. -- cgit v1.2.3 From 4098929a837a2d1e433d9248600ff7cf48700a6a Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Sat, 26 Sep 2020 15:18:35 +0200 Subject: Avoid undefined behavior. --- .../src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp index 68339e9c493..ac6cae9be6b 100644 --- a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp @@ -117,7 +117,8 @@ void compress_and_add_payload_to_rpc_params(mbus::BlobRef payload, params.AddInt8(comp_type); params.AddInt32(static_cast(to_compress.size())); - params.AddData(buf.stealBuffer(), buf.getDataLen()); + auto buffer_len = buf.getDataLen(); + params.AddData(buf.stealBuffer(), buffer_len); } } // anon ns -- cgit v1.2.3 From d7118a218d4704b52ed9883e4ff381a519adb7c1 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Sat, 26 Sep 2020 18:33:19 +0200 Subject: Mock lock path from thread to per-lock (bug) --- .../hosted/provision/restapi/LocksResponse.java | 18 ++-- .../main/java/com/yahoo/vespa/curator/Lock.java | 16 ++-- .../com/yahoo/vespa/curator/stats/LockInfo.java | 30 +++--- .../yahoo/vespa/curator/stats/LockInfoSamples.java | 106 +++++++++++++++++++++ .../yahoo/vespa/curator/stats/ThreadLockInfo.java | 91 +++++++----------- .../vespa/curator/stats/LockInfoSamplesTest.java | 58 +++++++++++ .../com/yahoo/vespa/curator/stats/LockTest.java | 2 +- 7 files changed, 233 insertions(+), 88 deletions(-) create mode 100644 zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfoSamples.java create mode 100644 zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockInfoSamplesTest.java diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java index 5a7414e894d..7a110f4c415 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java @@ -27,13 +27,13 @@ public class LocksResponse extends HttpResponse { public LocksResponse() { this(new TreeMap<>(ThreadLockInfo.getLockCountersByPath()), ThreadLockInfo.getThreadLockInfos(), - ThreadLockInfo.getSlowLockInfos()); + ThreadLockInfo.getLockInfoSamples()); } /** For testing */ LocksResponse(TreeMap lockCountersByPath, List threadLockInfos, - List slowLockInfos) { + List historicSamples) { super(200); Cursor root = slime.setObject(); @@ -58,18 +58,18 @@ public class LocksResponse extends HttpResponse { if (!lockInfos.isEmpty()) { Cursor threadLockInfoCursor = threadsCursor.addObject(); threadLockInfoCursor.setString("thread-name", threadLockInfo.getThreadName()); - threadLockInfoCursor.setString("lock-path", threadLockInfo.getLockPath()); - threadLockInfoCursor.setString("stack-trace", threadLockInfo.getStackTrace()); Cursor lockInfosCursor = threadLockInfoCursor.setArray("active-locks"); for (var lockInfo : lockInfos) { setLockInfo(lockInfosCursor.addObject(), lockInfo, false); } + + threadLockInfoCursor.setString("stack-trace", threadLockInfo.getStackTrace()); } } - Cursor slowLocksCursor = root.setArray("slow-locks"); - slowLockInfos.forEach(lockInfo -> setLockInfo(slowLocksCursor.addObject(), lockInfo, true)); + Cursor historicSamplesCursor = root.setArray("historic-samples"); + historicSamples.forEach(lockInfo -> setLockInfo(historicSamplesCursor.addObject(), lockInfo, true)); } @Override @@ -85,8 +85,8 @@ public class LocksResponse extends HttpResponse { private void setLockInfo(Cursor lockInfoCursor, LockInfo lockInfo, boolean includeThreadInfo) { if (includeThreadInfo) { lockInfoCursor.setString("thread-name", lockInfo.getThreadName()); - lockInfoCursor.setString("lock-path", lockInfo.getLockPath()); } + lockInfoCursor.setString("lock-path", lockInfo.getLockPath()); lockInfoCursor.setString("invoke-acquire-time", toString(lockInfo.getTimeAcquiredWasInvoked())); lockInfoCursor.setString("acquire-timeout", lockInfo.getAcquireTimeout().toString()); lockInfo.getTimeLockWasAcquired().ifPresent(instant -> lockInfoCursor.setString("lock-acquired-time", toString(instant))); @@ -95,7 +95,9 @@ public class LocksResponse extends HttpResponse { lockInfoCursor.setString("acquire-duration", lockInfo.getDurationOfAcquire().toString()); lockInfoCursor.setString("locked-duration", lockInfo.getDurationWithLock().toString()); lockInfoCursor.setString("total-duration", lockInfo.getDuration().toString()); - lockInfo.getStackTrace().ifPresent(stackTrace -> lockInfoCursor.setString("stack-trace", stackTrace)); + if (includeThreadInfo) { + lockInfo.getStackTrace().ifPresent(stackTrace -> lockInfoCursor.setString("stack-trace", stackTrace)); + } } private static String toString(Instant time) { diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java index 9af83223ae6..4239d325ba5 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java @@ -35,23 +35,23 @@ public class Lock implements Mutex { /** Take the lock with the given timeout. This may be called multiple times from the same thread - each matched by a close */ public void acquire(Duration timeout) throws UncheckedTimeoutException { - ThreadLockInfo threadLockInfo = getThreadLockInfo(); - threadLockInfo.invokingAcquire(timeout); + ThreadLockInfo threadLockInfo = ThreadLockInfo.getCurrentThreadLockInfo(); + threadLockInfo.invokingAcquire(lockPath, timeout); final boolean acquired; try { acquired = mutex.acquire(timeout.toMillis(), TimeUnit.MILLISECONDS); } catch (Exception e) { - threadLockInfo.acquireFailed(); + threadLockInfo.acquireFailed(lockPath); throw new RuntimeException("Exception acquiring lock '" + lockPath + "'", e); } if (!acquired) { - threadLockInfo.acquireTimedOut(); + threadLockInfo.acquireTimedOut(lockPath); throw new UncheckedTimeoutException("Timed out after waiting " + timeout + " to acquire lock '" + lockPath + "'"); } - threadLockInfo.lockAcquired(); + threadLockInfo.lockAcquired(lockPath); } @Override @@ -60,7 +60,7 @@ public class Lock implements Mutex { } private void release() { - getThreadLockInfo().lockReleased(); + ThreadLockInfo.getCurrentThreadLockInfo().lockReleased(lockPath); try { mutex.release(); } @@ -68,10 +68,6 @@ public class Lock implements Mutex { throw new RuntimeException("Exception releasing lock '" + lockPath + "'"); } } - - private ThreadLockInfo getThreadLockInfo() { - return ThreadLockInfo.getCurrentThreadLockInfo(lockPath); - } } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java index e9c238a40b9..f40c6f0498f 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java @@ -16,15 +16,16 @@ import java.util.Optional; public class LockInfo { private final ThreadLockInfo threadLockInfo; - private final Instant acquireInstant; + private final String lockPath; + private final Instant callAcquireInstant; private final Duration timeout; private volatile Optional lockAcquiredInstant = Optional.empty(); private volatile Optional terminalStateInstant = Optional.empty(); private volatile Optional stackTrace = Optional.empty(); - public static LockInfo invokingAcquire(ThreadLockInfo threadLockInfo, Duration timeout) { - return new LockInfo(threadLockInfo, timeout); + public static LockInfo invokingAcquire(ThreadLockInfo threadLockInfo, String lockPath, Duration timeout) { + return new LockInfo(threadLockInfo, lockPath, timeout, Instant.now()); } public enum LockState { @@ -39,15 +40,16 @@ public class LockInfo { private volatile LockState lockState = LockState.ACQUIRING; - private LockInfo(ThreadLockInfo threadLockInfo, Duration timeout) { + private LockInfo(ThreadLockInfo threadLockInfo, String lockPath, Duration timeout, Instant callAcquireInstant) { this.threadLockInfo = threadLockInfo; - this.acquireInstant = Instant.now(); + this.lockPath = lockPath; + this.callAcquireInstant = callAcquireInstant; this.timeout = timeout; } public String getThreadName() { return threadLockInfo.getThreadName(); } - public String getLockPath() { return threadLockInfo.getLockPath(); } - public Instant getTimeAcquiredWasInvoked() { return acquireInstant; } + public String getLockPath() { return lockPath; } + public Instant getTimeAcquiredWasInvoked() { return callAcquireInstant; } public Duration getAcquireTimeout() { return timeout; } public LockState getLockState() { return lockState; } public Optional getTimeLockWasAcquired() { return lockAcquiredInstant; } @@ -55,7 +57,7 @@ public class LockInfo { public Optional getStackTrace() { return stackTrace; } public Duration getDurationOfAcquire() { - return Duration.between(acquireInstant, lockAcquiredInstant.orElseGet(Instant::now)); + return Duration.between(callAcquireInstant, lockAcquiredInstant.orElseGet(Instant::now)); } public Duration getDurationWithLock() { @@ -64,11 +66,11 @@ public class LockInfo { .orElse(Duration.ZERO); } - public Duration getDuration() { return Duration.between(acquireInstant, terminalStateInstant.orElseGet(Instant::now)); } + public Duration getDuration() { return Duration.between(callAcquireInstant, terminalStateInstant.orElseGet(Instant::now)); } /** Get time from just before trying to acquire lock to the time the terminal state was reached, or ZERO. */ - public Duration getDurationInTerminalStateAndForPriorityQueue() { - return terminalStateInstant.map(instant -> Duration.between(acquireInstant, instant)).orElse(Duration.ZERO); + public Duration getStableTotalDuration() { + return terminalStateInstant.map(instant -> Duration.between(callAcquireInstant, instant)).orElse(Duration.ZERO); } /** Fill in the stack trace starting at the caller's stack frame. */ @@ -88,8 +90,10 @@ public class LockInfo { lockAcquiredInstant = Optional.of(Instant.now()); } - void setTerminalState(LockState terminalState) { + void setTerminalState(LockState terminalState) { setTerminalState(terminalState, Instant.now()); } + + void setTerminalState(LockState terminalState, Instant instant) { lockState = terminalState; - terminalStateInstant = Optional.of(Instant.now()); + terminalStateInstant = Optional.of(instant); } } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfoSamples.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfoSamples.java new file mode 100644 index 00000000000..ae84c5c984a --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfoSamples.java @@ -0,0 +1,106 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator.stats; + +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.PriorityQueue; + +/** + * Collection containing "interesting" {@code LockInfo}s. + * + * @author hakon + */ +// @ThreadSafe +public class LockInfoSamples { + private final int maxSamples; + + /** Ensure atomic operations on this collection. */ + private final Object monitor = new Object(); + + /** Keep at most one sample for each lock path. */ + private final Map byLockPath; + + /** + * Priority queue containing all samples. The head of this queue (peek()/poll()) + * returns the LockInfo with the smallest duration. + */ + private final PriorityQueue priorityQueue = + new PriorityQueue<>(Comparator.comparing(LockInfo::getStableTotalDuration)); + + LockInfoSamples() { this(10); } + + LockInfoSamples(int maxSamples) { + this.maxSamples = maxSamples; + this.byLockPath = new HashMap<>(maxSamples); + } + + int size() { return byLockPath.size(); } + + boolean maybeSample(LockInfo lockInfo) { + final boolean added; + synchronized (monitor) { + if (shouldAdd(lockInfo)) { + byLockPath.put(lockInfo.getLockPath(), lockInfo); + priorityQueue.add(lockInfo); + added = true; + } else { + added = false; + } + } + + if (added) { + // Unnecessary to invoke under synchronized, although it means that some samples + // may be without stack trace (just retry if that happens). + lockInfo.fillStackTrace(); + } + + return added; + } + + private boolean shouldAdd(LockInfo lockInfo) { + LockInfo existingLockInfo = byLockPath.get(lockInfo.getLockPath()); + if (existingLockInfo != null) { + if (hasLongerDurationThan(lockInfo, existingLockInfo)) { + byLockPath.remove(existingLockInfo.getLockPath()); + priorityQueue.remove(existingLockInfo); + return true; + } + + return false; + } + + if (size() < maxSamples) { + return true; + } + + // peek() and poll() retrieves the smallest element. + existingLockInfo = priorityQueue.peek(); // cannot be null + if (hasLongerDurationThan(lockInfo, existingLockInfo)) { + priorityQueue.poll(); + byLockPath.remove(existingLockInfo.getLockPath()); + return true; + } + + return false; + } + + List asList() { + synchronized (monitor) { + return List.copyOf(byLockPath.values()); + } + } + + void clear() { + synchronized (monitor) { + byLockPath.clear(); + priorityQueue.clear(); + } + } + + private static boolean hasLongerDurationThan(LockInfo lockInfo, LockInfo otherLockInfo) { + // Use stable total duration to avoid messing up priority queue. + return lockInfo.getStableTotalDuration().compareTo(otherLockInfo.getStableTotalDuration()) > 0; + } +} diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java index b796cb9af43..5280a049ceb 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java @@ -4,14 +4,11 @@ package com.yahoo.vespa.curator.stats; import com.yahoo.vespa.curator.Lock; import java.time.Duration; -import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.PriorityQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; /** @@ -26,17 +23,11 @@ public class ThreadLockInfo { private static final ConcurrentHashMap locks = new ConcurrentHashMap<>(); - private static final int MAX_COMPLETED_LOCK_INFOS_SIZE = 5; - /** Would have used a thread-safe priority queue. */ - private static final Object completedLockInfosMonitor = new Object(); - private static final PriorityQueue completedLockInfos = - new PriorityQueue<>(Comparator.comparing(LockInfo::getDurationInTerminalStateAndForPriorityQueue)); + private static final LockInfoSamples completedLockInfoSamples = new LockInfoSamples(); private static final ConcurrentHashMap countersByLockPath = new ConcurrentHashMap<>(); private final Thread thread; - private final String lockPath; - private final LockCounters lockCountersForPath; /** The locks are reentrant so there may be more than 1 lock for this thread. */ private final ConcurrentLinkedQueue lockInfos = new ConcurrentLinkedQueue<>(); @@ -45,36 +36,26 @@ public class ThreadLockInfo { public static List getThreadLockInfos() { return List.copyOf(locks.values()); } - public static List getSlowLockInfos() { - synchronized (completedLockInfosMonitor) { - return List.copyOf(completedLockInfos); - } + public static List getLockInfoSamples() { + return completedLockInfoSamples.asList(); } /** Returns the per-thread singleton ThreadLockInfo. */ - public static ThreadLockInfo getCurrentThreadLockInfo(String lockPath) { - return locks.computeIfAbsent( - Thread.currentThread(), - currentThread -> { - LockCounters lockCounters = countersByLockPath.computeIfAbsent(lockPath, ignored -> new LockCounters()); - return new ThreadLockInfo(currentThread, lockPath, lockCounters); - }); + public static ThreadLockInfo getCurrentThreadLockInfo() { + return locks.computeIfAbsent(Thread.currentThread(), ThreadLockInfo::new); } static void clearStaticDataForTesting() { locks.clear(); - completedLockInfos.clear(); + completedLockInfoSamples.clear(); countersByLockPath.clear(); } - ThreadLockInfo(Thread currentThread, String lockPath, LockCounters lockCountersForPath) { + ThreadLockInfo(Thread currentThread) { this.thread = currentThread; - this.lockPath = lockPath; - this.lockCountersForPath = lockCountersForPath; } public String getThreadName() { return thread.getName(); } - public String getLockPath() { return lockPath; } public String getStackTrace() { var stackTrace = new StringBuilder(); @@ -98,64 +79,62 @@ public class ThreadLockInfo { public List getLockInfos() { return List.copyOf(lockInfos); } /** Mutable method (see class doc) */ - public void invokingAcquire(Duration timeout) { - lockCountersForPath.invokeAcquireCount.incrementAndGet(); - lockCountersForPath.inCriticalRegionCount.incrementAndGet(); - lockInfos.add(LockInfo.invokingAcquire(this, timeout)); + public void invokingAcquire(String lockPath, Duration timeout) { + LockCounters lockCounters = getLockCounters(lockPath); + lockCounters.invokeAcquireCount.incrementAndGet(); + lockCounters.inCriticalRegionCount.incrementAndGet(); + lockInfos.add(LockInfo.invokingAcquire(this, lockPath, timeout)); } /** Mutable method (see class doc) */ - public void acquireFailed() { - removeLastLockInfo(lockCountersForPath.acquireFailedCount, LockInfo::acquireFailed); + public void acquireFailed(String lockPath) { + LockCounters lockCounters = getLockCounters(lockPath); + lockCounters.acquireFailedCount.incrementAndGet(); + removeLastLockInfo(lockCounters, LockInfo::acquireFailed); } /** Mutable method (see class doc) */ - public void acquireTimedOut() { + public void acquireTimedOut(String lockPath) { + LockCounters lockCounters = getLockCounters(lockPath); if (lockInfos.size() > 1) { - lockCountersForPath.timeoutOnReentrancyErrorCount.incrementAndGet(); + lockCounters.timeoutOnReentrancyErrorCount.incrementAndGet(); } - removeLastLockInfo(lockCountersForPath.acquireTimedOutCount, LockInfo::timedOut); + lockCounters.acquireTimedOutCount.incrementAndGet(); + removeLastLockInfo(lockCounters, LockInfo::timedOut); } /** Mutable method (see class doc) */ - public void lockAcquired() { - lockCountersForPath.lockAcquiredCount.incrementAndGet(); - + public void lockAcquired(String lockPath) { + getLockCounters(lockPath).lockAcquiredCount.incrementAndGet(); getLastLockInfo().ifPresent(LockInfo::lockAcquired); } /** Mutable method (see class doc) */ - public void lockReleased() { - removeLastLockInfo(lockCountersForPath.locksReleasedCount, LockInfo::released); + public void lockReleased(String lockPath) { + LockCounters lockCounters = getLockCounters(lockPath); + lockCounters.locksReleasedCount.incrementAndGet(); + removeLastLockInfo(lockCounters, LockInfo::released); + } + + private LockCounters getLockCounters(String lockPath) { + return countersByLockPath.computeIfAbsent(lockPath, __ -> new LockCounters()); } private Optional getLastLockInfo() { return lockInfos.isEmpty() ? Optional.empty() : Optional.of(lockInfos.peek()); } - private void removeLastLockInfo(AtomicInteger metricToIncrement, Consumer completeLockInfo) { - metricToIncrement.incrementAndGet(); - lockCountersForPath.inCriticalRegionCount.decrementAndGet(); + private void removeLastLockInfo(LockCounters lockCounters, Consumer completeLockInfo) { + lockCounters.inCriticalRegionCount.decrementAndGet(); if (lockInfos.isEmpty()) { - lockCountersForPath.noLocksErrorCount.incrementAndGet(); + lockCounters.noLocksErrorCount.incrementAndGet(); return; } LockInfo lockInfo = lockInfos.poll(); completeLockInfo.accept(lockInfo); - - synchronized (completedLockInfosMonitor) { - if (completedLockInfos.size() < MAX_COMPLETED_LOCK_INFOS_SIZE) { - lockInfo.fillStackTrace(); - completedLockInfos.add(lockInfo); - } else if (lockInfo.getDurationInTerminalStateAndForPriorityQueue() - .compareTo(completedLockInfos.peek().getDurationInTerminalStateAndForPriorityQueue()) > 0) { - completedLockInfos.poll(); - lockInfo.fillStackTrace(); - completedLockInfos.add(lockInfo); - } - } + completedLockInfoSamples.maybeSample(lockInfo); } } diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockInfoSamplesTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockInfoSamplesTest.java new file mode 100644 index 00000000000..4a14b0cc1b2 --- /dev/null +++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockInfoSamplesTest.java @@ -0,0 +1,58 @@ +package com.yahoo.vespa.curator.stats;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import org.junit.Test; + +import java.time.Duration; +import java.time.Instant; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class LockInfoSamplesTest { + private final LockInfoSamples samples = new LockInfoSamples(2); + private ThreadLockInfo threadLockInfo; + + @Test + public void test() { + threadLockInfo = new ThreadLockInfo(Thread.currentThread()); + + assertTrue(maybeSample("1", 10)); + + // new sample has longer duration + assertTrue(maybeSample("1", 11)); + + // new sample has shorter duration + assertFalse(maybeSample("1", 10)); + + // new path, will be added + assertTrue(maybeSample("2", 5)); + + // new path, too low duration be added + assertFalse(maybeSample("3", 4)); + + // new path, expels "2" + assertTrue(maybeSample("4", 6)); + + Map lockInfos = samples.asList().stream().collect(Collectors.toMap( + lockInfo -> lockInfo.getLockPath(), + lockInfo -> lockInfo)); + assertEquals(2, lockInfos.size()); + + assertTrue(lockInfos.containsKey("1")); + assertEquals(Duration.ofSeconds(11), lockInfos.get("1").getStableTotalDuration()); + + assertTrue(lockInfos.containsKey("4")); + assertEquals(Duration.ofSeconds(6), lockInfos.get("4").getStableTotalDuration()); + } + + private boolean maybeSample(String lockPath, int secondsDuration) { + LockInfo lockInfo = LockInfo.invokingAcquire(threadLockInfo, lockPath, Duration.ofSeconds(1)); + Instant instant = lockInfo.getTimeAcquiredWasInvoked().plus(Duration.ofSeconds(secondsDuration)); + lockInfo.setTerminalState(LockInfo.LockState.RELEASED, instant); + return samples.maybeSample(lockInfo); + } + +} \ No newline at end of file diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java index 23b603eca5c..4b9b6a4429b 100644 --- a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java +++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java @@ -47,7 +47,7 @@ public class LockTest { expectedCounters.acquireFailedCount.set(1); assertEquals(Map.of(lockPath, expectedCounters), ThreadLockInfo.getLockCountersByPath()); - List slowLockInfos = ThreadLockInfo.getSlowLockInfos(); + List slowLockInfos = ThreadLockInfo.getLockInfoSamples(); assertEquals(1, slowLockInfos.size()); LockInfo slowLockInfo = slowLockInfos.get(0); assertEquals(acquireTimeout, slowLockInfo.getAcquireTimeout()); -- cgit v1.2.3 From 4ada4ab6551d15432749b4b06d838194297f687d Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sat, 26 Sep 2020 18:38:58 +0000 Subject: Replace the dangerous stealBuffer method with a static one that requires std::move to make destruction more visible. --- messagebus/src/vespa/messagebus/network/rpcsendv2.cpp | 10 +++++----- .../src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.cpp | 8 +++++--- .../attribute/attributefilewriter/attributefilewriter_test.cpp | 2 +- .../src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp | 2 +- .../vespa/searchlib/attribute/attributefilebufferwriter.cpp | 1 + .../src/vespa/searchlib/attribute/attributefilewriter.cpp | 1 + .../searchlib/attribute/attributememoryfilebufferwriter.cpp | 5 ++--- .../vespa/searchlib/attribute/attributememoryfilewriter.cpp | 5 ++--- searchlib/src/vespa/searchlib/attribute/attrvector.cpp | 1 + searchlib/src/vespa/searchlib/attribute/iattributefilewriter.h | 3 ++- .../src/vespa/searchlib/attribute/singleboolattribute.cpp | 1 + .../vespa/searchlib/attribute/singlenumericattributesaver.cpp | 1 + .../vespa/searchlib/attribute/singlesmallnumericattribute.cpp | 1 + searchlib/src/vespa/searchlib/docstore/chunkformat.cpp | 3 ++- searchlib/src/vespa/searchlib/docstore/value.cpp | 7 ++++--- searchlib/src/vespa/searchlib/docstore/visitcache.cpp | 2 +- searchlib/src/vespa/searchlib/transactionlog/chunks.cpp | 2 +- .../rpc/cluster_controller_rpc_api_service_test.cpp | 3 ++- .../storage/storageserver/rpc/storage_api_rpc_service.cpp | 2 +- vespalib/src/vespa/vespalib/data/databuffer.h | 5 ++++- 20 files changed, 39 insertions(+), 26 deletions(-) diff --git a/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp b/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp index 6afa1528092..4c5b93048bf 100644 --- a/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp @@ -4,7 +4,6 @@ #include "rpcnetwork.h" #include "rpcserviceaddress.h" #include -#include #include #include #include @@ -49,7 +48,8 @@ Memory SERVICE_F("service"); } -bool RPCSendV2::isCompatible(stringref method, stringref request, stringref response) +bool +RPCSendV2::isCompatible(stringref method, stringref request, stringref response) { return (method == METHOD_NAME) && (request == METHOD_PARAMS) && @@ -133,7 +133,7 @@ RPCSendV2::encodeRequest(FRT_RPCRequest &req, const Version &version, const Rout args.AddInt32(toCompress.size()); const auto bufferLength = buf.getDataLen(); assert(bufferLength <= INT32_MAX); - args.AddData(buf.stealBuffer(), bufferLength); + args.AddData(DataBuffer::stealBuffer(std::move(buf)), bufferLength); } namespace { @@ -141,7 +141,7 @@ namespace { class ParamsV2 : public RPCSend::Params { public: - ParamsV2(const FRT_Values &arg) + explicit ParamsV2(const FRT_Values &arg) : _slime() { uint8_t encoding = arg[3]._intval8; @@ -263,7 +263,7 @@ RPCSendV2::createResponse(FRT_Values & ret, const string & version, Reply & repl ret.AddInt32(toCompress.size()); const auto bufferLength = buf.getDataLen(); assert(bufferLength <= INT32_MAX); - ret.AddData(buf.stealBuffer(), bufferLength); + ret.AddData(DataBuffer::stealBuffer(std::move(buf)), bufferLength); } diff --git a/searchcore/src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.cpp b/searchcore/src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.cpp index e2f3a951b99..f8952363509 100644 --- a/searchcore/src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.cpp +++ b/searchcore/src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.cpp @@ -32,7 +32,8 @@ namespace feedbm { namespace { -FRT_RPCRequest *make_set_cluster_state_request() { +FRT_RPCRequest * +make_set_cluster_state_request() { storage::lib::ClusterStateBundle bundle(storage::lib::ClusterState("version:2 distributor:1 storage:1")); storage::rpc::SlimeClusterStateBundleCodec codec; auto encoded_bundle = codec.encode(bundle); @@ -41,12 +42,13 @@ FRT_RPCRequest *make_set_cluster_state_request() { params->AddInt8(static_cast(encoded_bundle._compression_type)); params->AddInt32(encoded_bundle._uncompressed_length); const auto buf_len = encoded_bundle._buffer->getDataLen(); - params->AddData(encoded_bundle._buffer->stealBuffer(), buf_len); + params->AddData(vespalib::DataBuffer::stealBuffer(std::move(*encoded_bundle._buffer)), buf_len); req->SetMethodName("setdistributionstates"); return req; } -void set_cluster_up(SharedRpcResources &shared_rpc_resources, storage::api::StorageMessageAddress &storage_address) { +void +set_cluster_up(SharedRpcResources &shared_rpc_resources, storage::api::StorageMessageAddress &storage_address) { auto req = make_set_cluster_state_request(); auto target_resolver = std::make_unique(shared_rpc_resources.slobrok_mirror(), shared_rpc_resources.target_factory()); auto target = target_resolver->resolve_rpc_target(storage_address); diff --git a/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp b/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp index eeabbf23c27..1e3531dc78e 100644 --- a/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp +++ b/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include -#include #include #include #include @@ -10,6 +9,7 @@ #include #include #include +#include #include #include diff --git a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp index d8761f69d71..2a5b8014299 100644 --- a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp +++ b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -21,6 +20,7 @@ #include #include #include +#include #include diff --git a/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.cpp b/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.cpp index 341112f9b22..4efd64f72dd 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attributefilebufferwriter.h" +#include namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp b/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp index 415c00cb8fd..829720e9c3e 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include diff --git a/searchlib/src/vespa/searchlib/attribute/attributememoryfilebufferwriter.cpp b/searchlib/src/vespa/searchlib/attribute/attributememoryfilebufferwriter.cpp index b354566616b..454cc486f70 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributememoryfilebufferwriter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributememoryfilebufferwriter.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attributememoryfilebufferwriter.h" +#include namespace search { @@ -11,9 +12,7 @@ AttributeMemoryFileBufferWriter(IAttributeFileWriter &memoryFileWriter) } -AttributeMemoryFileBufferWriter::~AttributeMemoryFileBufferWriter() -{ -} +AttributeMemoryFileBufferWriter::~AttributeMemoryFileBufferWriter() = default; void diff --git a/searchlib/src/vespa/searchlib/attribute/attributememoryfilewriter.cpp b/searchlib/src/vespa/searchlib/attribute/attributememoryfilewriter.cpp index c28b7e9d20b..8d412364815 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributememoryfilewriter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributememoryfilewriter.cpp @@ -2,6 +2,7 @@ #include "attributememoryfilewriter.h" #include "attributememoryfilebufferwriter.h" +#include namespace search { @@ -18,9 +19,7 @@ AttributeMemoryFileWriter::AttributeMemoryFileWriter() } -AttributeMemoryFileWriter::~AttributeMemoryFileWriter() -{ -} +AttributeMemoryFileWriter::~AttributeMemoryFileWriter() = default; AttributeMemoryFileWriter::Buffer diff --git a/searchlib/src/vespa/searchlib/attribute/attrvector.cpp b/searchlib/src/vespa/searchlib/attribute/attrvector.cpp index 59771d7ffae..80f72aaea25 100644 --- a/searchlib/src/vespa/searchlib/attribute/attrvector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attrvector.cpp @@ -4,6 +4,7 @@ #include "attrvector.hpp" #include "iattributesavetarget.h" #include "load_utils.h" +#include #include LOG_SETUP(".searchlib.attribute.attr_vector"); diff --git a/searchlib/src/vespa/searchlib/attribute/iattributefilewriter.h b/searchlib/src/vespa/searchlib/attribute/iattributefilewriter.h index bb00124c9fc..94e16b37e9d 100644 --- a/searchlib/src/vespa/searchlib/attribute/iattributefilewriter.h +++ b/searchlib/src/vespa/searchlib/attribute/iattributefilewriter.h @@ -2,8 +2,9 @@ #pragma once -#include +#include +namespace vespalib { class DataBuffer; } namespace search { class BufferWriter; diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp index e1ab47ed434..bbe8c9c8327 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp @@ -8,6 +8,7 @@ #include #include #include +#include namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/singlenumericattributesaver.cpp index db8636f47c3..3c26e960a06 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattributesaver.cpp @@ -2,6 +2,7 @@ #include "singlenumericattributesaver.h" #include "iattributesavetarget.h" +#include using vespalib::GenerationHandler; diff --git a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp index 6eff0da06e8..fd2631ac63d 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp @@ -7,6 +7,7 @@ #include "iattributesavetarget.h" #include #include +#include namespace search { diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp index fbbdcff3c5d..e9689e36180 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp @@ -12,6 +12,7 @@ using vespalib::compression::compress; using vespalib::compression::decompress; using vespalib::compression::computeMaxCompressedsize; using vespalib::compression::CompressionConfig; +using vespalib::DataBuffer; ChunkException::ChunkException(const vespalib::string & msg, vespalib::stringref location) : Exception(make_string("Illegal chunk: %s", msg.c_str()), location) @@ -140,7 +141,7 @@ ChunkFormat::deserializeBody(vespalib::nbostream & is) assert(uncompressed.getData() == uncompressed.getDead()); if (uncompressed.getData() != data.c_str()) { const size_t sz(uncompressed.getDataLen()); - vespalib::nbostream(uncompressed.stealBuffer(), sz).swap(_dataBuf); + vespalib::nbostream(DataBuffer::stealBuffer(std::move(uncompressed)), sz).swap(_dataBuf); } else { _dataBuf = vespalib::nbostream(uncompressed.getData(), uncompressed.getDataLen()); } diff --git a/searchlib/src/vespa/searchlib/docstore/value.cpp b/searchlib/src/vespa/searchlib/docstore/value.cpp index 09725b447cd..40d04b3d7fb 100644 --- a/searchlib/src/vespa/searchlib/docstore/value.cpp +++ b/searchlib/src/vespa/searchlib/docstore/value.cpp @@ -1,12 +1,12 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "value.h" -#include #include #include using vespalib::compression::compress; using vespalib::compression::decompress; +using vespalib::DataBuffer; namespace search::docstore { @@ -66,8 +66,9 @@ Value::set(vespalib::DataBuffer &&buf, ssize_t len, const CompressionConfig &com _compression = type; _uncompressedSize = len; _uncompressedCrc = XXH64(input.c_str(), input.size(), 0); - _buf = std::make_shared(compact(_compressedSize, - (buf.getData() == compressed.getData()) ? buf.stealBuffer() : compressed.stealBuffer())); + _buf = std::make_shared(compact(_compressedSize,(buf.getData() == compressed.getData()) + ? DataBuffer::stealBuffer(std::move(buf)) + : DataBuffer::stealBuffer(std::move(compressed)))); assert(((type == CompressionConfig::NONE) && (len == ssize_t(_compressedSize))) || diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp index e8504480b7d..dada57c15b9 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp @@ -109,7 +109,7 @@ CompressedBlobSet::getBlobSet() const decompress(_compression, getBufferSize(_positions), ConstBufferRef(_buffer->c_str(), _buffer->size()), uncompressed, false); } - return BlobSet(_positions, uncompressed.stealBuffer()); + return BlobSet(_positions, DataBuffer::stealBuffer(std::move(uncompressed))); } size_t CompressedBlobSet::size() const { diff --git a/searchlib/src/vespa/searchlib/transactionlog/chunks.cpp b/searchlib/src/vespa/searchlib/transactionlog/chunks.cpp index a78db61429d..b06ff9397c4 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/chunks.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/chunks.cpp @@ -94,7 +94,7 @@ XXH64CompressedChunk::decompress(nbostream & is, uint32_t uncompressedLen) { ::decompress(_type, uncompressedLen, compressed, uncompressed, false); nbostream data(uncompressed.getData(), uncompressed.getDataLen()); deserializeEntries(data); - _backing = uncompressed.stealBuffer(); + _backing = DataBuffer::stealBuffer(std::move(uncompressed)); is.adjustReadPos(is.size()); } diff --git a/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp b/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp index 09c9ddb1f72..a65e8678a72 100644 --- a/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp +++ b/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp @@ -15,6 +15,7 @@ #include #include +using vespalib::DataBuffer; namespace storage::rpc { using document::FixedBucketSpaces; @@ -83,7 +84,7 @@ struct SetStateFixture : FixtureBase { params->AddInt8(static_cast(encoded_bundle._compression_type)); params->AddInt32(uncompressed_length); const auto buf_len = encoded_bundle._buffer->getDataLen(); - params->AddData(encoded_bundle._buffer->stealBuffer(), buf_len); + params->AddData(DataBuffer::stealBuffer(std::move(*encoded_bundle._buffer)), buf_len); bound_request->SetDetachedPT(&request_is_detached); bound_request->SetReturnHandler(&return_handler); diff --git a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp index ac6cae9be6b..42af872da9b 100644 --- a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp @@ -118,7 +118,7 @@ void compress_and_add_payload_to_rpc_params(mbus::BlobRef payload, params.AddInt8(comp_type); params.AddInt32(static_cast(to_compress.size())); auto buffer_len = buf.getDataLen(); - params.AddData(buf.stealBuffer(), buffer_len); + params.AddData(vespalib::DataBuffer::stealBuffer(std::move(buf)), buffer_len); } } // anon ns diff --git a/vespalib/src/vespa/vespalib/data/databuffer.h b/vespalib/src/vespa/vespalib/data/databuffer.h index a520ecd58bd..e1c21661716 100644 --- a/vespalib/src/vespa/vespalib/data/databuffer.h +++ b/vespalib/src/vespa/vespalib/data/databuffer.h @@ -40,6 +40,7 @@ private: char *_freept; Alloc _buffer; + Alloc stealBuffer(); public: typedef std::unique_ptr UP; DataBuffer(const DataBuffer &) = delete; @@ -606,7 +607,9 @@ public: **/ void swap(DataBuffer &other); - Alloc stealBuffer(); + static Alloc stealBuffer(DataBuffer buf) { + return buf.stealBuffer(); + } }; } // namespace vespalib -- cgit v1.2.3 From 4f11c4d782c70dc99ad8b0c2f4fc20fbec0a95a3 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sat, 26 Sep 2020 18:42:43 +0000 Subject: GC unused code --- searchlib/CMakeLists.txt | 1 - searchlib/src/tests/bytecomplens/.gitignore | 5 - searchlib/src/tests/bytecomplens/CMakeLists.txt | 8 - searchlib/src/tests/bytecomplens/bytecomp.cpp | 101 ------ searchlib/src/tests/bytecomplens/example.txt | 122 ------- searchlib/src/tests/bytecomplens/tblprint.cpp | 356 --------------------- .../src/vespa/searchlib/docstore/CMakeLists.txt | 1 - .../src/vespa/searchlib/docstore/bytecomplens.cpp | 260 --------------- .../src/vespa/searchlib/docstore/bytecomplens.h | 110 ------- 9 files changed, 964 deletions(-) delete mode 100644 searchlib/src/tests/bytecomplens/.gitignore delete mode 100644 searchlib/src/tests/bytecomplens/CMakeLists.txt delete mode 100644 searchlib/src/tests/bytecomplens/bytecomp.cpp delete mode 100644 searchlib/src/tests/bytecomplens/example.txt delete mode 100644 searchlib/src/tests/bytecomplens/tblprint.cpp delete mode 100644 searchlib/src/vespa/searchlib/docstore/bytecomplens.cpp delete mode 100644 searchlib/src/vespa/searchlib/docstore/bytecomplens.h diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index 1f8c054c694..33b78d517a3 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -102,7 +102,6 @@ vespa_define_module( src/tests/bitcompression/expgolomb src/tests/bitvector src/tests/btree - src/tests/bytecomplens src/tests/common/bitvector src/tests/common/location src/tests/common/location_iterator diff --git a/searchlib/src/tests/bytecomplens/.gitignore b/searchlib/src/tests/bytecomplens/.gitignore deleted file mode 100644 index afe9bff02f6..00000000000 --- a/searchlib/src/tests/bytecomplens/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -*.So -.depend* -Makefile -bytecomp_test -searchlib_bytecomp_test_app diff --git a/searchlib/src/tests/bytecomplens/CMakeLists.txt b/searchlib/src/tests/bytecomplens/CMakeLists.txt deleted file mode 100644 index 24ecef59b15..00000000000 --- a/searchlib/src/tests/bytecomplens/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_bytecomp_test_app TEST - SOURCES - bytecomp.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_bytecomp_test_app NO_VALGRIND COMMAND searchlib_bytecomp_test_app) diff --git a/searchlib/src/tests/bytecomplens/bytecomp.cpp b/searchlib/src/tests/bytecomplens/bytecomp.cpp deleted file mode 100644 index adc92b0097d..00000000000 --- a/searchlib/src/tests/bytecomplens/bytecomp.cpp +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include -#include -LOG_SETUP("bytecomplens_test"); -#include -#include -#include - - -class Test : public vespalib::TestApp { -private: - void testRandomLengths(); - -public: - int Main() override { - TEST_INIT("bytecomplens_test"); - testRandomLengths(); TEST_FLUSH(); - TEST_DONE(); - } -}; - -TEST_APPHOOK(Test); - - -void -Test::testRandomLengths() -{ - vespalib::RandomGen rndgen(0x07031969); - -#define TBLSIZ 0xc00000 - - auto lentable = std::unique_ptr(new uint32_t[TBLSIZ]); - auto offtable = std::unique_ptr(new uint64_t[TBLSIZ]); - - uint64_t offset = 16; - - for (int i = 0; i < TBLSIZ; i++) { - int sel = rndgen.nextInt32(); - int val = rndgen.nextInt32(); - switch (sel & 0x7) { - case 0: - val &= 0x7F; - break; - case 1: - val &= 0xFF; - break; - case 3: - val &= 0x1FFF; - break; - case 4: - val &= 0x3FFF; - break; - case 5: - val &= 0x7FFF; - break; - case 6: - val &= 0xFFFF; - break; - case 7: - default: - val &= 0xFFFFF; - break; - } - offtable[i] = offset; - lentable[i] = val; - offset += val; - } - - LOG(info, "made %d random offsets", TBLSIZ); - - search::ByteCompressedLengths foo; - - LOG(info, "empty BCL using %9ld bytes memory", foo.memoryUsed()); - - foo.addOffsetTable(TBLSIZ/4, offtable.get()); - foo.addOffsetTable(TBLSIZ/4, offtable.get() + 1*(TBLSIZ/4)); - - LOG(info, "half BCL using %9ld bytes memory", foo.memoryUsed()); - - search::ByteCompressedLengths bar; - foo.swap(bar); - bar.addOffsetTable(TBLSIZ/4, offtable.get() + 2*(TBLSIZ/4)); - bar.addOffsetTable(TBLSIZ/4, offtable.get() + 3*(TBLSIZ/4)); - foo.swap(bar); - - LOG(info, "full BCL using %9ld bytes memory", foo.memoryUsed()); - - LOG(info, "constructed %d byte compressed lengths", TBLSIZ-1); - - for (int i = 0; i < TBLSIZ-1; i++) { - search::ByteCompressedLengths::OffLen offlen; - offlen = foo.getOffLen(i); - - if ((i % 1000000) == 0) { - LOG(info, "data blob [%d] length %" PRIu64 " offset %" PRIu64, i, offlen.length, offlen.offset); - } - EXPECT_EQUAL(lentable[i], offlen.length); - EXPECT_EQUAL(offtable[i], offlen.offset); - } -} - diff --git a/searchlib/src/tests/bytecomplens/example.txt b/searchlib/src/tests/bytecomplens/example.txt deleted file mode 100644 index 6dc3df0118a..00000000000 --- a/searchlib/src/tests/bytecomplens/example.txt +++ /dev/null @@ -1,122 +0,0 @@ -offset length BCN val L0 len/off skipL1 skipL2 skipL3 - -976 18707 [ 93 92 01 ] 3/0 976/0/0/0 -19683 11527 [ 87 5A ] 2/3 -31210 3926 [ D6 1E ] 2/5 -35136 2 [ 02 ] 1/7 -35138 6060 [ AC 2F ] 2/8 34162/8 -41198 649445 [ E5 D1 27 ] 3/10 -690643 2866 [ B2 16 ] 2/13 -693509 824767 [ BF AB 32 ] 3/15 -1518276 499173 [ E5 BB 1E ] 3/18 1483138/10 -2017449 20455 [ E7 9F 01 ] 3/21 -2037904 11 [ 0B ] 1/24 -2037915 19207 [ 87 96 01 ] 3/25 -2057122 6355 [ D3 31 ] 2/28 538846/10 -2063477 3422 [ DE 1A ] 2/30 -2066899 10683 [ BB 53 ] 2/32 -2077582 7360 [ C0 39 ] 2/34 -2084942 17969 [ B1 8C 01 ] 3/36 2083966/36/12 -2102911 6114 [ E2 2F ] 2/39 -2109025 31741 [ FD F7 01 ] 3/41 -2140766 581588 [ D4 BF 23 ] 3/44 -2722354 5341 [ DD 29 ] 2/47 637412/11 -2727695 13774 [ CE 6B ] 2/49 -2741469 717809 [ F1 E7 2B ] 3/51 -3459278 815406 [ AE E2 31 ] 3/54 -4274684 89 [ 59 ] 1/57 1552330/10 -4274773 4545 [ C1 23 ] 2/58 -4279318 803868 [ 9C 88 31 ] 3/60 -5083186 12865 [ C1 64 ] 2/63 -5096051 75 [ 4B ] 1/65 821367/8 -5096126 40734 [ 9E BE 02 ] 3/66 -5136860 101 [ 65 ] 1/69 -5136961 128 [ 80 01 ] 2/70 -5137089 253 [ FD 01 ] 2/72 3052147/36/12 -5137342 13 [ 0D ] 1/74 -5137355 24986 [ 9A C3 01 ] 3/75 -5162341 231 [ E7 01 ] 2/78 -5162572 997853 [ DD F3 3C ] 3/80 25483/8 -6160425 4728 [ F8 24 ] 2/83 -6165153 2025 [ E9 0F ] 2/85 -6167178 7281 [ F1 38 ] 2/87 -6174459 1026302 [ FE D1 3E ] 3/89 1011887/9 -7200761 848783 [ 8F E7 33 ] 3/92 -8049544 145767 [ E7 F2 08 ] 3/95 -8195311 19103 [ 9F 95 01 ] 3/98 -8214414 22166 [ 96 AD 01 ] 3/101 2039955/12 -8236580 30020 [ C4 EA 01 ] 3/104 -8266600 13 [ 0D ] 1/107 -8266613 120 [ 78 ] 1/108 -8266733 22398 [ FE AE 01 ] 3/109 3129644/37/12 -8289131 10832 [ D0 54 ] 2/112 -8299963 3765 [ B5 1D ] 2/114 -8303728 432771 [ 83 B5 1A ] 3/116 -8736499 30133 [ B5 EB 01 ] 3/119 469766/10 -8766632 6444 [ AC 32 ] 2/122 -8773076 16033 [ A1 7D ] 2/124 -8789109 78 [ 4E ] 1/126 -8789187 12510 [ DE 61 ] 2/127 52688/8 -8801697 12441 [ 99 61 ] 2/129 -8814138 117 [ 75 ] 1/131 -8814255 7147 [ EB 37 ] 2/132 -8821402 189 [ BD 01 ] 2/134 32215/7 -8821591 199704 [ 98 98 0C ] 3/136 -9021295 13240 [ B8 67 ] 2/139 -9034535 110 [ 6E ] 1/141 -9034645 31677 [ BD F7 01 ] 3/142 9034645/142/48/17 -9066322 18547 [ F3 90 01 ] 3/145 -9084869 734679 [ D7 EB 2C ] 3/148 -9819548 112 [ 70 ] 1/151 -9819660 883565 [ ED F6 35 ] 3/152 785015/10 -10703225 10290 [ B2 50 ] 2/155 -10713515 21410 [ A2 A7 01 ] 3/157 -10734925 15 [ 0F ] 1/160 -10734940 747774 [ FE D1 2D ] 3/161 915280/9 -11482714 39 [ 27 ] 1/164 -11482753 77 [ 4D ] 1/165 -11482830 235 [ EB 01 ] 2/166 -11483065 1991 [ C7 0F ] 2/168 748125/7 -11485056 9187 [ E3 47 ] 2/170 -11494243 18800 [ F0 92 01 ] 3/172 -11513043 1042219 [ AB CE 3F ] 3/175 -12555262 9154 [ C2 47 ] 2/178 3520617/36/12 -12564416 43582 [ BE D4 02 ] 3/180 -12607998 847240 [ 88 DB 33 ] 3/183 -13455238 4726 [ F6 24 ] 2/186 -13459964 590348 [ 8C 84 24 ] 3/188 904702/10 -14050312 8659 [ D3 43 ] 2/191 -14058971 116 [ 74 ] 1/193 -14059087 13563 [ FB 69 ] 2/194 -14072650 713064 [ E8 C2 2B ] 3/196 612686/8 -14785714 40321 [ 81 BB 02 ] 3/199 -14826035 2296 [ F8 11 ] 2/202 -14828331 7273 [ E9 38 ] 2/204 -14835604 68285 [ BD 95 04 ] 3/206 762954/10 -14903889 235 [ EB 01 ] 2/209 -14904124 4669 [ BD 24 ] 2/211 -14908793 28535 [ F7 DE 01 ] 3/213 -14937328 19 [ 13 ] 1/216 2382066/38/12 -14937347 5369 [ F9 29 ] 2/217 -14942716 602191 [ CF E0 24 ] 3/219 -15544907 2653 [ DD 14 ] 2/222 -15547560 25755 [ 9B C9 01 ] 3/224 610232/8 -15573315 11349 [ D5 58 ] 2/227 -15584664 15006 [ 9E 75 ] 2/229 -15599670 89 [ 59 ] 1/231 -15599759 52772 [ A4 9C 03 ] 3/232 52199/8 -15652531 776175 [ EF AF 2F ] 3/235 -16428706 126 [ 7E ] 1/238 -16428832 3884 [ AC 1E ] 2/239 -16432716 33958 [ A6 89 02 ] 3/241 832957/9 -16466674 122 [ 7A ] 1/244 -16466796 41895 [ A7 C7 02 ] 3/245 -16508691 105882 [ 9A BB 06 ] 3/248 -16614573 11067 [ BB 56 ] 2/251 1677245/35/12 -16625640 4588 [ EC 23 ] 2/253 -16630228 7349 [ B5 39 ] 2/255 -16637577 902638 [ EE 8B 37 ] 3/257 -17540215 8737 [ A1 44 ] 2/260 925642/9 -17548952 29186 [ 82 E4 01 ] 3/262 -17578138 41 [ 29 ] 1/265 -17578179 diff --git a/searchlib/src/tests/bytecomplens/tblprint.cpp b/searchlib/src/tests/bytecomplens/tblprint.cpp deleted file mode 100644 index 6a7347f3c0d..00000000000 --- a/searchlib/src/tests/bytecomplens/tblprint.cpp +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include -LOG_SETUP("tblprint"); -#include - -#include -#include - - -/** - * Class compressing a table of offsets in memory. - * After adding (n) offsets you can access - * (n-1) pairs of (length, offset). - * All offsets must be increasing, but they - * may be added in several chunks. - **/ -class ByteCompressedLengths -{ -public: - /** - * Construct an empty instance - **/ - ByteCompressedLengths(); - - /** - * add the given offset table. - * @param entries number of offsets to store. - * @param offsets table that contains (entries) offsets. - **/ - void addOffsetTable(uint64_t entries, uint64_t *offsets); - - /** - * free resources - **/ - ~ByteCompressedLengths(); - - /** - * Fetch a length and offset from compressed data. - * Note invariant: id < size(); size() == (entries-1) - * - * @param id The index into the offset table - * @param offset Will be incremented by offset[id] - * @return The delta (offset[id+1] - offset[id]) - **/ - uint64_t getLength(uint64_t id, uint64_t &offset) const; - - /** - * The number of (length, offset) pairs stored - **/ - uint64_t size() const { return _entries; } - - struct L3Entry { - uint64_t offset; - uint64_t l0toff; - uint64_t l1toff; - uint64_t l2toff; - }; - vespalib::DataBuffer _l0space; - vespalib::DataBuffer _l1space; - vespalib::DataBuffer _l2space; - const uint8_t *_l0table; - const uint8_t *_l1table; - const uint8_t *_l2table; - - std::vector _l3table; - - uint64_t _lenSum1; - uint64_t _lenSum2; - uint64_t _l0oSum1; - uint64_t _l0oSum2; - uint64_t _l1oSum2; - uint64_t _last_offset; - uint64_t _entries; - - void addOffset(uint64_t offset); -}; - -/** - * get "Byte Compressed Number" from buffer, incrementing pointer - **/ -static inline uint64_t getBCN(const uint8_t *&buffer) -{ - uint8_t b = *buffer++; - uint64_t len = (b & 127); - unsigned shiftLen = 0; - while (b & 128) { - shiftLen += 7; - b = *buffer++; - len |= ((b & 127) << shiftLen); - } - return len; -} - -static size_t writeLen(vespalib::DataBuffer &buf, uint64_t len) -{ - size_t bytes = 0; - do { - uint8_t b = len & 127; - len >>= 7; - if (len > 0) { - b |= 128; - } - buf.ensureFree(1); - buf.writeInt8(b); - ++bytes; - } while (len > 0); - return bytes; -} - - -ByteCompressedLengths::ByteCompressedLengths() - : _l0space(), - _l1space(), - _l2space(), - _l3table(), - _lenSum1(0), - _lenSum2(0), - _l0oSum1(0), - _l0oSum2(0), - _l1oSum2(0), - _last_offset(0), - _entries(0) -{ -} - - -void -ByteCompressedLengths::addOffset(uint64_t offset) -{ - assert(offset >= _last_offset); - - uint64_t len = offset - _last_offset; - uint64_t i = _entries++; - - if ((i & 3) == 0) { - _lenSum2 += _lenSum1; - _l0oSum2 += _l0oSum1; - - uint64_t t1n = i >> 2; - if ((t1n & 3) == 0) { - uint64_t t2n = t1n >> 2; - - if ((t2n & 3) == 0) { - L3Entry e; - e.offset = _last_offset; - e.l0toff = _l0space.getDataLen(); - e.l1toff = _l1space.getDataLen(); - e.l2toff = _l2space.getDataLen(); - - _l3table.push_back(e); - } else { - writeLen(_l2space, _lenSum2); - writeLen(_l2space, _l0oSum2); - writeLen(_l2space, _l1oSum2); - } - _lenSum2 = 0; - _l0oSum2 = 0; - _l1oSum2 = 0; - } else { - _l1oSum2 += writeLen(_l1space, _lenSum1); - _l1oSum2 += writeLen(_l1space, _l0oSum1); - } - _lenSum1 = 0; - _l0oSum1 = 0; - } - _l0oSum1 += writeLen(_l0space, len); - _lenSum1 += len; - _last_offset = offset; -} - - -void -ByteCompressedLengths::addOffsetTable(uint64_t entries, uint64_t *offsets) -{ - if (entries == 0) return; - // Do we have some offsets already? - if (_entries > 0) { - // yes, add first offset normally - addOffset(offsets[0]); - } else { - // no, special treatment for very first offset - _last_offset = offsets[0]; - } - for (uint64_t cnt = 1; cnt < entries; ++cnt) { - addOffset(offsets[cnt]); - } - _l0table = (uint8_t *)_l0space.getData(); - _l1table = (uint8_t *)_l1space.getData(); - _l2table = (uint8_t *)_l2space.getData(); - - LOG(debug, "compressed %ld offsets", (_entries+1)); - LOG(debug, "(%ld bytes)", (_entries+1)*sizeof(uint64_t)); - LOG(debug, "to (%ld + %ld + %ld) bytes + %ld l3entries", - _l0space.getDataLen(), - _l1space.getDataLen(), - _l2space.getDataLen(), - _l3table.size()); - LOG(debug, "(%ld bytes)", - (_l0space.getDataLen() + _l1space.getDataLen() + _l2space.getDataLen() + - _l3table.size()*sizeof(L3Entry))); -} - - -ByteCompressedLengths::~ByteCompressedLengths() -{ -} - -uint64_t -ByteCompressedLengths::getLength(uint64_t numSkip, uint64_t &offset) const -{ - assert(numSkip < _entries); - - unsigned skipL0 = numSkip & 3; - unsigned skipL1 = (numSkip >> 2) & 3; - unsigned skipL2 = (numSkip >> 4) & 3; - uint64_t skipL3 = (numSkip >> 6); - - offset += _l3table[skipL3].offset; - uint64_t l0toff = _l3table[skipL3].l0toff; - uint64_t l1toff = _l3table[skipL3].l1toff; - uint64_t l2toff = _l3table[skipL3].l2toff; - - // printf("start off %ld l0off %ld l1off %ld l2off %ld\n", offset, l0toff, l1toff, l2toff); - - const uint8_t *l2pos = _l2table + l2toff; - - while (skipL2 > 0) { - --skipL2; - offset += getBCN(l2pos); - l0toff += getBCN(l2pos); - l1toff += getBCN(l2pos); - } - - const uint8_t *l1pos = _l1table + l1toff; - - while (skipL1 > 0) { - --skipL1; - offset += getBCN(l1pos); - l0toff += getBCN(l1pos); - - } - const uint8_t *l0pos = _l0table + l0toff; - - while (skipL0 > 0) { - --skipL0; - offset += getBCN(l0pos); - } - // printf("end off %ld l0off %ld l1off %ld l2off %ld\n", offset, l0toff, l1toff, l2toff); - return getBCN(l0pos); -} - - - -class Test { -public: - static void printTable(); -}; - - - -int main(int /*argc*/, char ** /*argv*/) -{ - Test::printTable(); - return 0; -} - -void -Test::printTable() -{ - vespalib::RandomGen rndgen(0x07031969); -#define TBLSIZ 120 - uint32_t *lentable = new uint32_t[TBLSIZ]; - uint64_t *offtable = new uint64_t[TBLSIZ]; - - uint64_t offset = 16 + TBLSIZ*8; - - for (int i = 0; i < TBLSIZ; i++) { - int sel = rndgen.nextInt32(); - int val = rndgen.nextInt32(); - switch (sel & 0x7) { - case 0: - val &= 0x7F; - break; - case 1: - val &= 0xFF; - break; - case 3: - val &= 0x1FFF; - break; - case 4: - val &= 0x3FFF; - break; - case 5: - val &= 0x7FFF; - break; - case 6: - val &= 0xFFFF; - break; - case 7: - default: - val &= 0xFFFFF; - break; - } - offtable[i] = offset; - lentable[i] = val; - offset += val; - } - - ByteCompressedLengths foo; - foo.addOffsetTable(TBLSIZ, offtable); - - const uint8_t *l1pos = foo._l1table; - const uint8_t *l2pos = foo._l2table; - - printf("%s\t%s\t%s\t%s\t%s\t%s\t%s\n", - "offset", "length", "BCN val", "L0 len/off", "skipL1", "skipL2", "skipL3"); - - int slb = 0; - for (int i = 0; i+1 < TBLSIZ; i++) { - printf("%ld\t%d\t[", offtable[i], lentable[i]); - int bytes=0; - uint64_t len = lentable[i]; - do { - uint8_t b = len & 127; - len >>= 7; - if (len > 0) { - b |= 128; - } - printf(" %02X", b); - ++bytes; - } while (len > 0); - printf(" ]\t%d", bytes); - printf("/%d", slb); - slb += bytes; - - if ((i & 63) == 0) { - printf("\t\t\t%ld/%ld/%ld/%ld", - foo._l3table[i >> 6].offset, - foo._l3table[i >> 6].l0toff, - foo._l3table[i >> 6].l1toff, - foo._l3table[i >> 6].l2toff); - } else - if ((i & 15) == 0) { - printf("\t\t%ld", getBCN(l2pos)); - printf("/%ld", getBCN(l2pos)); - printf("/%ld", getBCN(l2pos)); - } else - if ((i & 3) == 0) { - printf("\t%ld", getBCN(l1pos)); - printf("/%ld", getBCN(l1pos)); - } - printf("\n"); - } - printf("%ld\n", offtable[TBLSIZ-1]); - fflush(stdout); -} diff --git a/searchlib/src/vespa/searchlib/docstore/CMakeLists.txt b/searchlib/src/vespa/searchlib/docstore/CMakeLists.txt index 2b82d9e5af7..b1c27e20210 100644 --- a/searchlib/src/vespa/searchlib/docstore/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/docstore/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchlib_docstore OBJECT SOURCES - bytecomplens.cpp chunk.cpp chunkformat.cpp chunkformats.cpp diff --git a/searchlib/src/vespa/searchlib/docstore/bytecomplens.cpp b/searchlib/src/vespa/searchlib/docstore/bytecomplens.cpp deleted file mode 100644 index 4ef57b77dbd..00000000000 --- a/searchlib/src/vespa/searchlib/docstore/bytecomplens.cpp +++ /dev/null @@ -1,260 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "bytecomplens.h" - -#include -LOG_SETUP(".search.docstore"); - -namespace search { - -static inline uint64_t getBCN(const uint8_t *&buffer) __attribute__((__always_inline__)); - -/** - * get "Byte Compressed Number" from buffer, incrementing pointer - **/ -static inline uint64_t getBCN(const uint8_t *&buffer) -{ - uint8_t b = *buffer++; - uint64_t len = (b & 127); - unsigned shiftLen = 0; - while (b & 128) { - shiftLen += 7; - b = *buffer++; - len |= ((b & 127) << shiftLen); - } - return len; -} - -static size_t writeLen(vespalib::DataBuffer &buf, uint64_t len) -{ - size_t bytes = 0; - do { - uint8_t b = len & 127; - len >>= 7; - if (len > 0) { - b |= 128; - } - buf.writeInt8(b); - ++bytes; - } while (len > 0); - return bytes; -} - - -ByteCompressedLengths::ByteCompressedLengths() - : _l0space(), - _l1space(), - _l2space(), - _l3table(), - _entries(0), - _progress(), - _ptrcache(), - _hasInitialOffset(false) -{ - clear(); -} - - -void -ByteCompressedLengths::clear() -{ - _l0space.clear(); - _l1space.clear(); - _l2space.clear(); - _l3table.clear(); - - _entries = 0; - - _progress.lenSum1 = 0; - _progress.lenSum2 = 0; - _progress.l0oSum1 = 0; - _progress.l0oSum2 = 0; - _progress.l1oSum2 = 0; - _progress.last_offset = 0; - - _ptrcache.l0table = NULL; - _ptrcache.l1table = NULL; - _ptrcache.l2table = NULL; - - _hasInitialOffset = false; -} - - -void -ByteCompressedLengths::swap(ByteCompressedLengths& other) -{ - _l0space.swap(other._l0space); - _l1space.swap(other._l1space); - _l2space.swap(other._l2space); - _l3table.swap(other._l3table); - - std::swap(_entries, other._entries); - std::swap(_progress, other._progress); - std::swap(_ptrcache, other._ptrcache); - std::swap(_hasInitialOffset, other._hasInitialOffset); -} - - -// add a new offset to the compressed tables -void -ByteCompressedLengths::addOffset(uint64_t offset) -{ - assert(offset >= _progress.last_offset); - - // delta from last offset: - uint64_t len = offset - _progress.last_offset; - - // which entry is this: - uint64_t idx = _entries++; - - if ((idx & 31) == 0) { - // add entry to some skip-table - _progress.lenSum2 += _progress.lenSum1; // accumulate to Level2 - _progress.l0oSum2 += _progress.l0oSum1; // accumulate to Level2 - - uint64_t t1n = idx >> 5; - if ((t1n & 31) == 0) { - // add Level2 or Level3 table entry: - uint64_t t2n = t1n >> 5; - - if ((t2n & 31) == 0) { - // add new Level3 table entry: - L3Entry e; - e.offset = _progress.last_offset; - e.l0toff = _l0space.getDataLen(); - e.l1toff = _l1space.getDataLen(); - e.l2toff = _l2space.getDataLen(); - - _l3table.push_back(e); - } else { - // write to Level2 table, sums since last reset: - writeLen(_l2space, _progress.lenSum2); // sum of Level0 lengths - writeLen(_l2space, _progress.l0oSum2); // sum size of Level0 entries - writeLen(_l2space, _progress.l1oSum2); // sum size of Level1 entries - } - // reset Level2 sums: - _progress.lenSum2 = 0; - _progress.l0oSum2 = 0; - _progress.l1oSum2 = 0; - } else { - // write to Level1 table, sums since last reset: - _progress.l1oSum2 += writeLen(_l1space, _progress.lenSum1); // sum of Level0 lengths - _progress.l1oSum2 += writeLen(_l1space, _progress.l0oSum1); // sum size of Level0 entries - } - // reset Level1 sums: - _progress.lenSum1 = 0; - _progress.l0oSum1 = 0; - } - // always write length (offset delta) to Level0 table: - _progress.l0oSum1 += writeLen(_l0space, len); // accumulate to Level1 - _progress.lenSum1 += len; // accumulate to Level1 - _progress.last_offset = offset; -} - - -void -ByteCompressedLengths::addOffsetTable(uint64_t entries, uint64_t *offsets) -{ - // ignore NOP: - if (entries == 0) return; - - // Do we have some offsets already? - if (_hasInitialOffset) { - // yes, add first offset normally - addOffset(offsets[0]); - } else { - // no, special treatment for very first offset - _progress.last_offset = offsets[0]; - _hasInitialOffset = true; - } - for (uint64_t cnt = 1; cnt < entries; ++cnt) { - addOffset(offsets[cnt]); - } - - // Simplify access to actual data: - _ptrcache.l0table = (uint8_t *)_l0space.getData(); - _ptrcache.l1table = (uint8_t *)_l1space.getData(); - _ptrcache.l2table = (uint8_t *)_l2space.getData(); - - // some statistics available when debug logging: - LOG(debug, "compressed %" PRIu64 " offsets", (_entries+1)); - LOG(debug, "(%" PRIu64 " bytes)", (_entries+1)*sizeof(uint64_t)); - LOG(debug, "to (%ld + %ld + %ld) bytes + %ld l3entries", - _l0space.getDataLen(), - _l1space.getDataLen(), - _l2space.getDataLen(), - _l3table.size()); - LOG(debug, "(%ld bytes)", - (_l0space.getDataLen() + _l1space.getDataLen() + _l2space.getDataLen() + - _l3table.size()*sizeof(L3Entry))); -} - - -ByteCompressedLengths::~ByteCompressedLengths() -{ -} - -ByteCompressedLengths::OffLen -ByteCompressedLengths::getOffLen(uint64_t idx) const -{ - assert(idx < _entries); - - unsigned skipL0 = idx & 31; - unsigned skipL1 = (idx >> 5) & 31; - unsigned skipL2 = (idx >> 10) & 31; - uint64_t skipL3 = (idx >> 15); - - uint64_t offset = _l3table[skipL3].offset; - uint64_t l0toff = _l3table[skipL3].l0toff; - uint64_t l1toff = _l3table[skipL3].l1toff; - uint64_t l2toff = _l3table[skipL3].l2toff; - - // printf("start off %ld l0off %ld l1off %ld l2off %ld\n", offset, l0toff, l1toff, l2toff); - - const uint8_t *l2pos = _ptrcache.l2table + l2toff; - - while (skipL2 > 0) { - --skipL2; - offset += getBCN(l2pos); - l0toff += getBCN(l2pos); - l1toff += getBCN(l2pos); - } - - const uint8_t *l1pos = _ptrcache.l1table + l1toff; - - while (skipL1 > 0) { - --skipL1; - offset += getBCN(l1pos); - l0toff += getBCN(l1pos); - - } - const uint8_t *l0pos = _ptrcache.l0table + l0toff; - - while (skipL0 > 0) { - --skipL0; - offset += getBCN(l0pos); - } - // printf("end off %ld l0off %ld l1off %ld l2off %ld\n", offset, l0toff, l1toff, l2toff); - OffLen retval; - retval.offset = offset; - retval.length = getBCN(l0pos); - return retval; -} - - -size_t -ByteCompressedLengths::memoryUsed() const -{ - size_t mem = sizeof(*this); - mem += _l0space.getBufSize(); - mem += _l1space.getBufSize(); - mem += _l2space.getBufSize(); - mem += _l3table.capacity() * sizeof(L3Entry); - return mem; -} - - - - -} // namespace search - diff --git a/searchlib/src/vespa/searchlib/docstore/bytecomplens.h b/searchlib/src/vespa/searchlib/docstore/bytecomplens.h deleted file mode 100644 index 88f6a11c764..00000000000 --- a/searchlib/src/vespa/searchlib/docstore/bytecomplens.h +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include -#include - -namespace search { - -/** - * Class compressing a table of offsets in memory. - * After adding (n) offsets you can access - * (n-1) pairs of (length, offset). - * All offsets must be increasing, but they - * may be added in several chunks. - **/ -class ByteCompressedLengths -{ -public: - /** - * Construct an empty instance - **/ - ByteCompressedLengths(); - - /** - * add the given offset table. - * @param entries number of offsets to store. - * @param offsets pointer to table that contains (entries) offsets. - **/ - void addOffsetTable(uint64_t entries, uint64_t *offsets); - - /** - * free resources - **/ - ~ByteCompressedLengths(); - - struct OffLen - { - uint64_t offset; - uint64_t length; - }; - - /** - * Fetch an offset and length from compressed data. - * Note restriction: idx must be < size() - * - * @param idx The index into the offset table - * @return offset[id] and the delta (offset[id+1] - offset[id]) - **/ - OffLen getOffLen(uint64_t idx) const; - - /** - * The number of (length, offset) pairs stored - * Note that size() == sum(entries) - 1 - **/ - uint64_t size() const { return _entries; } - - /** - * remove all data from this instance - **/ - void clear(); - - /** - * swap all data with another instance - **/ - void swap(ByteCompressedLengths& other); - - /** - * Calculate memory used by this instance - * @return memory usage (in bytes) - **/ - size_t memoryUsed() const; - -private: - struct L3Entry { - uint64_t offset; - uint64_t l0toff; - uint64_t l1toff; - uint64_t l2toff; - }; - vespalib::DataBuffer _l0space; - vespalib::DataBuffer _l1space; - vespalib::DataBuffer _l2space; - - std::vector _l3table; - - uint64_t _entries; - - struct ProgressPoint { - uint64_t lenSum1; - uint64_t lenSum2; - uint64_t l0oSum1; - uint64_t l0oSum2; - uint64_t l1oSum2; - uint64_t last_offset; - } _progress; - - struct CachedPointers { - const uint8_t *l0table; - const uint8_t *l1table; - const uint8_t *l2table; - } _ptrcache; - - bool _hasInitialOffset; - - void addOffset(uint64_t offset); -}; - -} // namespace search - -- cgit v1.2.3 From 72b6f0a82a7f1f7caec8cc58815b3802d94fb92d Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sat, 26 Sep 2020 18:50:07 +0000 Subject: Add && to make it evident that the object must be moved. --- vespalib/src/vespa/vespalib/data/databuffer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vespalib/src/vespa/vespalib/data/databuffer.h b/vespalib/src/vespa/vespalib/data/databuffer.h index e1c21661716..3229921e09c 100644 --- a/vespalib/src/vespa/vespalib/data/databuffer.h +++ b/vespalib/src/vespa/vespalib/data/databuffer.h @@ -607,7 +607,7 @@ public: **/ void swap(DataBuffer &other); - static Alloc stealBuffer(DataBuffer buf) { + static Alloc stealBuffer(DataBuffer && buf) { return buf.stealBuffer(); } }; -- cgit v1.2.3 From 80b2585f2c5bed035d5fb9215533109ad00a3fc5 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sat, 26 Sep 2020 19:19:24 +0000 Subject: Use && qualified member functions. --- fnet/src/vespa/fnet/frt/values.cpp | 7 +++++++ fnet/src/vespa/fnet/frt/values.h | 6 +++++- messagebus/src/vespa/messagebus/network/rpcsendv2.cpp | 7 +++---- .../src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.cpp | 3 +-- searchlib/src/vespa/searchlib/docstore/chunkformat.cpp | 3 +-- searchlib/src/vespa/searchlib/docstore/value.cpp | 5 ++--- searchlib/src/vespa/searchlib/docstore/visitcache.cpp | 2 +- searchlib/src/vespa/searchlib/transactionlog/chunks.cpp | 2 +- .../storageserver/rpc/cluster_controller_rpc_api_service_test.cpp | 4 +--- .../vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp | 3 +-- vespalib/src/vespa/vespalib/data/databuffer.cpp | 2 +- vespalib/src/vespa/vespalib/data/databuffer.h | 5 +---- 12 files changed, 25 insertions(+), 24 deletions(-) diff --git a/fnet/src/vespa/fnet/frt/values.cpp b/fnet/src/vespa/fnet/frt/values.cpp index 3b37aa9a1bc..593bbd173c6 100644 --- a/fnet/src/vespa/fnet/frt/values.cpp +++ b/fnet/src/vespa/fnet/frt/values.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include static_assert(sizeof(uint8_t) == 1, "uint8_t must be 1 byte."); @@ -299,6 +300,12 @@ FRT_Values::AddData(vespalib::alloc::Alloc && buf, uint32_t len) { AddSharedData(&_stash.create(std::move(buf), len)); } +void +FRT_Values::AddData(vespalib::DataBuffer && buf) { + const auto len = buf.getDataLen(); + AddSharedData(&_stash.create(std::move(buf).stealBuffer(), len)); +} + void FRT_Values::AddData(const char *buf, uint32_t len) { if (len > SHARED_LIMIT) { diff --git a/fnet/src/vespa/fnet/frt/values.h b/fnet/src/vespa/fnet/frt/values.h index 2aa7551c423..bac1f609fbb 100644 --- a/fnet/src/vespa/fnet/frt/values.h +++ b/fnet/src/vespa/fnet/frt/values.h @@ -5,7 +5,10 @@ #include "isharedblob.h" #include -namespace vespalib { class Stash; } +namespace vespalib { + class Stash; + class DataBuffer; +} namespace vespalib::alloc { class Alloc; } namespace fnet { char * copyString(char *dst, const char *src, size_t len); @@ -218,6 +221,7 @@ public: FRT_StringValue *AddStringArray(uint32_t len); void AddSharedData(FRT_ISharedBlob *blob); void AddData(Alloc && buf, uint32_t len); + void AddData(vespalib::DataBuffer && buf); void AddData(const char *buf, uint32_t len); char *AddData(uint32_t len); FRT_DataValue *AddDataArray(uint32_t len); diff --git a/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp b/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp index 4c5b93048bf..f7303ece20f 100644 --- a/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp @@ -133,7 +133,7 @@ RPCSendV2::encodeRequest(FRT_RPCRequest &req, const Version &version, const Rout args.AddInt32(toCompress.size()); const auto bufferLength = buf.getDataLen(); assert(bufferLength <= INT32_MAX); - args.AddData(DataBuffer::stealBuffer(std::move(buf)), bufferLength); + args.AddData(std::move(buf).stealBuffer(), bufferLength); } namespace { @@ -261,9 +261,8 @@ RPCSendV2::createResponse(FRT_Values & ret, const string & version, Reply & repl ret.AddInt8(type); ret.AddInt32(toCompress.size()); - const auto bufferLength = buf.getDataLen(); - assert(bufferLength <= INT32_MAX); - ret.AddData(DataBuffer::stealBuffer(std::move(buf)), bufferLength); + assert(buf.getDataLen() <= INT32_MAX); + ret.AddData(std::move(buf)); } diff --git a/searchcore/src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.cpp b/searchcore/src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.cpp index f8952363509..c8d73444652 100644 --- a/searchcore/src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.cpp +++ b/searchcore/src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.cpp @@ -41,8 +41,7 @@ make_set_cluster_state_request() { auto* params = req->GetParams(); params->AddInt8(static_cast(encoded_bundle._compression_type)); params->AddInt32(encoded_bundle._uncompressed_length); - const auto buf_len = encoded_bundle._buffer->getDataLen(); - params->AddData(vespalib::DataBuffer::stealBuffer(std::move(*encoded_bundle._buffer)), buf_len); + params->AddData(std::move(*encoded_bundle._buffer)); req->SetMethodName("setdistributionstates"); return req; } diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp index e9689e36180..afa7abb9ef8 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp @@ -12,7 +12,6 @@ using vespalib::compression::compress; using vespalib::compression::decompress; using vespalib::compression::computeMaxCompressedsize; using vespalib::compression::CompressionConfig; -using vespalib::DataBuffer; ChunkException::ChunkException(const vespalib::string & msg, vespalib::stringref location) : Exception(make_string("Illegal chunk: %s", msg.c_str()), location) @@ -141,7 +140,7 @@ ChunkFormat::deserializeBody(vespalib::nbostream & is) assert(uncompressed.getData() == uncompressed.getDead()); if (uncompressed.getData() != data.c_str()) { const size_t sz(uncompressed.getDataLen()); - vespalib::nbostream(DataBuffer::stealBuffer(std::move(uncompressed)), sz).swap(_dataBuf); + vespalib::nbostream(std::move(uncompressed).stealBuffer(), sz).swap(_dataBuf); } else { _dataBuf = vespalib::nbostream(uncompressed.getData(), uncompressed.getDataLen()); } diff --git a/searchlib/src/vespa/searchlib/docstore/value.cpp b/searchlib/src/vespa/searchlib/docstore/value.cpp index 40d04b3d7fb..25cf93ac18b 100644 --- a/searchlib/src/vespa/searchlib/docstore/value.cpp +++ b/searchlib/src/vespa/searchlib/docstore/value.cpp @@ -6,7 +6,6 @@ using vespalib::compression::compress; using vespalib::compression::decompress; -using vespalib::DataBuffer; namespace search::docstore { @@ -67,8 +66,8 @@ Value::set(vespalib::DataBuffer &&buf, ssize_t len, const CompressionConfig &com _uncompressedSize = len; _uncompressedCrc = XXH64(input.c_str(), input.size(), 0); _buf = std::make_shared(compact(_compressedSize,(buf.getData() == compressed.getData()) - ? DataBuffer::stealBuffer(std::move(buf)) - : DataBuffer::stealBuffer(std::move(compressed)))); + ? std::move(buf).stealBuffer() + : std::move(compressed).stealBuffer())); assert(((type == CompressionConfig::NONE) && (len == ssize_t(_compressedSize))) || diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp index dada57c15b9..6990a0a3ed7 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp @@ -109,7 +109,7 @@ CompressedBlobSet::getBlobSet() const decompress(_compression, getBufferSize(_positions), ConstBufferRef(_buffer->c_str(), _buffer->size()), uncompressed, false); } - return BlobSet(_positions, DataBuffer::stealBuffer(std::move(uncompressed))); + return BlobSet(_positions, std::move(uncompressed).stealBuffer()); } size_t CompressedBlobSet::size() const { diff --git a/searchlib/src/vespa/searchlib/transactionlog/chunks.cpp b/searchlib/src/vespa/searchlib/transactionlog/chunks.cpp index b06ff9397c4..edb12b4bf36 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/chunks.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/chunks.cpp @@ -94,7 +94,7 @@ XXH64CompressedChunk::decompress(nbostream & is, uint32_t uncompressedLen) { ::decompress(_type, uncompressedLen, compressed, uncompressed, false); nbostream data(uncompressed.getData(), uncompressed.getDataLen()); deserializeEntries(data); - _backing = DataBuffer::stealBuffer(std::move(uncompressed)); + _backing = std::move(uncompressed).stealBuffer(); is.adjustReadPos(is.size()); } diff --git a/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp b/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp index a65e8678a72..8b009e02f28 100644 --- a/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp +++ b/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp @@ -15,7 +15,6 @@ #include #include -using vespalib::DataBuffer; namespace storage::rpc { using document::FixedBucketSpaces; @@ -83,8 +82,7 @@ struct SetStateFixture : FixtureBase { auto* params = bound_request->GetParams(); params->AddInt8(static_cast(encoded_bundle._compression_type)); params->AddInt32(uncompressed_length); - const auto buf_len = encoded_bundle._buffer->getDataLen(); - params->AddData(DataBuffer::stealBuffer(std::move(*encoded_bundle._buffer)), buf_len); + params->AddData(std::move(*encoded_bundle._buffer)); bound_request->SetDetachedPT(&request_is_detached); bound_request->SetReturnHandler(&return_handler); diff --git a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp index 42af872da9b..c234f623d45 100644 --- a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp @@ -117,8 +117,7 @@ void compress_and_add_payload_to_rpc_params(mbus::BlobRef payload, params.AddInt8(comp_type); params.AddInt32(static_cast(to_compress.size())); - auto buffer_len = buf.getDataLen(); - params.AddData(vespalib::DataBuffer::stealBuffer(std::move(buf)), buffer_len); + params.AddData(std::move(buf)); } } // anon ns diff --git a/vespalib/src/vespa/vespalib/data/databuffer.cpp b/vespalib/src/vespa/vespalib/data/databuffer.cpp index 39c0f3f7482..04c4b1e225b 100644 --- a/vespalib/src/vespa/vespalib/data/databuffer.cpp +++ b/vespalib/src/vespa/vespalib/data/databuffer.cpp @@ -155,7 +155,7 @@ DataBuffer::swap(DataBuffer &other) } vespalib::alloc::Alloc -DataBuffer::stealBuffer() +DataBuffer::stealBuffer() && { assert( ! referencesExternalData() ); _externalBuf = nullptr; diff --git a/vespalib/src/vespa/vespalib/data/databuffer.h b/vespalib/src/vespa/vespalib/data/databuffer.h index 3229921e09c..7c4cd63a7b1 100644 --- a/vespalib/src/vespa/vespalib/data/databuffer.h +++ b/vespalib/src/vespa/vespalib/data/databuffer.h @@ -40,7 +40,6 @@ private: char *_freept; Alloc _buffer; - Alloc stealBuffer(); public: typedef std::unique_ptr UP; DataBuffer(const DataBuffer &) = delete; @@ -607,9 +606,7 @@ public: **/ void swap(DataBuffer &other); - static Alloc stealBuffer(DataBuffer && buf) { - return buf.stealBuffer(); - } + Alloc stealBuffer() &&; }; } // namespace vespalib -- cgit v1.2.3 From ec50b1a6499d334d018015f7b6417a826164548d Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sat, 26 Sep 2020 19:23:42 +0000 Subject: Reinstate temporarily in order to for large revert to be smooth --- searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp | 1 + searchcore/src/vespa/searchcore/proton/server/feedhandler.h | 1 + 2 files changed, 2 insertions(+) diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 13427d0e3aa..209a35ce4a2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -403,6 +403,7 @@ FeedHandler::FeedHandler(IThreadingService &writeService, _serialNum(0), _prunedSerialNum(0), _numPendingCommit(0), + _numCommitsCompleted(0), _delayedPrune(false), _feedLock(), _feedState(make_shared(getDocTypeName())), diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index 395d11ba2fa..97629bfc018 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -77,6 +77,7 @@ private: SerialNum _serialNum; SerialNum _prunedSerialNum; size_t _numPendingCommit; + size_t _numCommitsCompleted; bool _delayedPrune; mutable std::shared_mutex _feedLock; FeedStateSP _feedState; -- cgit v1.2.3 From f6683a698da0a7478a00dcc57c9abf2dccc35f4a Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sat, 26 Sep 2020 21:30:47 +0200 Subject: Revert "Revert "Revert "Revert "Revert "Balder/group operations to tls and commit in batches""""" --- .../documentmetastore/lid_reuse_delayer_config.cpp | 2 - .../documentmetastore/lid_reuse_delayer_config.h | 2 - .../proton/documentmetastore/lidreusedelayer.cpp | 1 - .../proton/documentmetastore/lidreusedelayer.h | 4 +- .../proton/server/document_db_maintenance_config.h | 1 - .../vespa/searchcore/proton/server/feedhandler.cpp | 32 +---------- .../vespa/searchcore/proton/server/feedhandler.h | 5 -- .../searchcore/proton/server/i_operation_storer.h | 4 -- .../proton/server/lid_space_compaction_job.cpp | 2 +- .../proton/server/lid_space_compaction_job.h | 6 +- .../proton/server/maintenance_jobs_injector.cpp | 2 +- .../searchcore/proton/server/storeonlyfeedview.cpp | 26 ++++----- .../searchcore/proton/server/storeonlyfeedview.h | 2 +- .../src/vespa/searchlib/config/translogserver.def | 2 +- .../src/vespa/searchlib/transactionlog/domain.cpp | 66 ++-------------------- .../src/vespa/searchlib/transactionlog/domain.h | 5 -- .../vespa/searchlib/transactionlog/domainpart.cpp | 12 +--- .../searchlib/transactionlog/translogserver.cpp | 7 +-- 18 files changed, 32 insertions(+), 149 deletions(-) diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp index 1f979d1566c..b04bac5ef26 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp @@ -7,7 +7,6 @@ namespace proton::documentmetastore { LidReuseDelayerConfig::LidReuseDelayerConfig(const DocumentDBConfig & configSnapshot) : _visibilityDelay(configSnapshot.getMaintenanceConfigSP()->getVisibilityDelay()), - _allowEarlyAck(configSnapshot.getMaintenanceConfigSP()->allowEarlyAck()), _hasIndexedOrAttributeFields(configSnapshot.getSchemaSP()->getNumIndexFields() > 0 || configSnapshot.getSchemaSP()->getNumAttributeFields() > 0) { @@ -19,7 +18,6 @@ LidReuseDelayerConfig::LidReuseDelayerConfig() LidReuseDelayerConfig::LidReuseDelayerConfig(vespalib::duration visibilityDelay, bool hasIndexedOrAttributeFields_in) : _visibilityDelay(visibilityDelay), - _allowEarlyAck(visibilityDelay > 1ms), _hasIndexedOrAttributeFields(hasIndexedOrAttributeFields_in) { } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h index c81a2ff399f..82dab433a22 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h @@ -15,7 +15,6 @@ class LidReuseDelayerConfig { private: vespalib::duration _visibilityDelay; - bool _allowEarlyAck; bool _hasIndexedOrAttributeFields; public: LidReuseDelayerConfig(); @@ -23,7 +22,6 @@ public: explicit LidReuseDelayerConfig(const DocumentDBConfig &configSnapshot); vespalib::duration visibilityDelay() const { return _visibilityDelay; } bool hasIndexedOrAttributeFields() const { return _hasIndexedOrAttributeFields; } - bool allowEarlyAck() const { return _allowEarlyAck; } }; } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp index ed2202c830b..03dfd83a132 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp @@ -16,7 +16,6 @@ LidReuseDelayer::LidReuseDelayer(IThreadingService &writeService, IStore &docume : _writeService(writeService), _documentMetaStore(documentMetaStore), _immediateCommit(config.visibilityDelay() == vespalib::duration::zero()), - _allowEarlyAck(config.allowEarlyAck()), _config(config), _pendingLids() { diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h index ba407ab57f8..5f1de878b4a 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h @@ -27,7 +27,6 @@ class LidReuseDelayer searchcorespi::index::IThreadingService &_writeService; IStore &_documentMetaStore; const bool _immediateCommit; - const bool _allowEarlyAck; LidReuseDelayerConfig _config; std::vector _pendingLids; // lids waiting for commit @@ -39,8 +38,7 @@ public: bool delayReuse(const std::vector &lids); std::vector getReuseLids(); - bool needImmediateCommit() const { return _immediateCommit; } - bool allowEarlyAck() const { return _allowEarlyAck; } + bool getImmediateCommit() const { return _immediateCommit; } const LidReuseDelayerConfig & getConfig() const { return _config; } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h index 5ed0ad7492c..4c0485baec6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h @@ -135,7 +135,6 @@ public: } vespalib::duration getVisibilityDelay() const { return _visibilityDelay; } bool hasVisibilityDelay() const { return _visibilityDelay > vespalib::duration::zero(); } - bool allowEarlyAck() const { return _visibilityDelay > 1ms; } const DocumentDBLidSpaceCompactionConfig &getLidSpaceCompactionConfig() const { return _lidSpaceCompaction; } diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 209a35ce4a2..37dfddf0c2c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -402,8 +402,6 @@ FeedHandler::FeedHandler(IThreadingService &writeService, _tlsReplayProgress(), _serialNum(0), _prunedSerialNum(0), - _numPendingCommit(0), - _numCommitsCompleted(0), _delayedPrune(false), _feedLock(), _feedState(make_shared(getDocTypeName())), @@ -496,40 +494,12 @@ FeedHandler::getTransactionLogReplayDone() const { return _tlsMgr.getReplayDone(); } -void -FeedHandler::onCommitDone(uint64_t numPendingAtStart) { - assert(numPendingAtStart <= _numPendingCommit); - _numPendingCommit -= numPendingAtStart; - if (_numPendingCommit > 0) { - enqueCommitTask(); - } -} - -void FeedHandler::enqueCommitTask() { - _writeService.master().execute(makeLambdaTask([this]() { initiateCommit(); })); -} - -void -FeedHandler::initiateCommit() { - auto commitResult = _tlsWriter->startCommit(std::make_shared( - _writeService.master(), - makeLambdaTask([this, numPendingAtStart=_numPendingCommit]() { - onCommitDone(numPendingAtStart); - }))); - if (_activeFeedView) { - _activeFeedView->forceCommit(_serialNum, std::make_shared>(std::move(commitResult))); - } -} - void FeedHandler::appendOperation(const FeedOperation &op, TlsWriter::DoneCallback onDone) { if (!op.getSerialNum()) { const_cast(op).setSerialNum(incSerialNum()); } _tlsWriter->appendOperation(op, std::move(onDone)); - if (++_numPendingCommit == 1) { - enqueCommitTask(); - } } FeedHandler::CommitResult @@ -540,7 +510,7 @@ FeedHandler::startCommit(DoneCallback onDone) { void FeedHandler::storeOperationSync(const FeedOperation &op) { vespalib::Gate gate; - appendAndCommitOperation(op, make_shared(gate)); + appendOperation(op, make_shared(gate)); gate.await(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index 97629bfc018..4807c596130 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -76,8 +76,6 @@ private: // the serial num of the last message in the transaction log SerialNum _serialNum; SerialNum _prunedSerialNum; - size_t _numPendingCommit; - size_t _numCommitsCompleted; bool _delayedPrune; mutable std::shared_mutex _feedLock; FeedStateSP _feedState; @@ -127,9 +125,6 @@ private: FeedStateSP getFeedState() const; void changeFeedState(FeedStateSP newState); void doChangeFeedState(FeedStateSP newState); - void onCommitDone(uint64_t numPendingAtStart); - void initiateCommit(); - void enqueCommitTask(); public: FeedHandler(const FeedHandler &) = delete; FeedHandler & operator = (const FeedHandler &) = delete; diff --git a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h index c3b76a9db75..b276779c2ee 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h @@ -22,10 +22,6 @@ struct IOperationStorer */ virtual void appendOperation(const FeedOperation &op, DoneCallback onDone) = 0; [[nodiscard]] virtual CommitResult startCommit(DoneCallback onDone) = 0; - void appendAndCommitOperation(const FeedOperation &op, DoneCallback onDone) { - appendOperation(op, onDone); - (void) startCommit(std::move(onDone)); - } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp index 468850b4409..d423e095ad9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp @@ -82,7 +82,7 @@ LidSpaceCompactionJob::compactLidSpace(const LidUsageStats &stats) uint32_t wantedLidLimit = stats.getHighestUsedLid() + 1; CompactLidSpaceOperation op(_handler.getSubDbId(), wantedLidLimit); vespalib::Gate gate; - _opStorer.appendAndCommitOperation(op, std::make_shared(gate)); + _opStorer.appendOperation(op, std::make_shared(gate)); gate.await(); _handler.handleCompactLidSpace(op); EventLogger::lidSpaceCompactionComplete(_handler.getName(), wantedLidLimit); diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h index 35549f21471..37497eaa998 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h @@ -63,13 +63,13 @@ public: ~LidSpaceCompactionJob(); // Implements IDiskMemUsageListener - void notifyDiskMemUsage(DiskMemUsageState state) override; + virtual void notifyDiskMemUsage(DiskMemUsageState state) override; // Implements IClusterStateChangedNofifier - void notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) override; + virtual void notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) override; // Implements IMaintenanceJob - bool run() override; + virtual bool run() override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp index d4b542a0af8..d94bb2e3d03 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp @@ -105,7 +105,7 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller, AttributeUsageFilter &attributeUsageFilter) { controller.registerJobInMasterThread(std::make_unique(hbHandler, config.getHeartBeatConfig())); controller.registerJobInDefaultPool(std::make_unique(scPruner, config.getSessionCachePruneInterval())); - if (config.hasVisibilityDelay() && config.allowEarlyAck()) { + if (config.hasVisibilityDelay()) { controller.registerJobInMasterThread(std::make_unique(commit, config.getVisibilityDelay())); } const MaintenanceDocumentSubDB &mRemSubDB(controller.getRemSubDB()); diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index 9927a2d2ce0..217a3bb24d3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -210,11 +210,11 @@ moveMetaData(documentmetastore::IStore &meta_store, const DocumentId & doc_id, c } std::unique_ptr -createUncommitedLidTracker(bool allowEarlyAck) { - if (allowEarlyAck) { - return std::make_unique(); - } else { +createUncommitedLidTracker(bool needImmediateCommit) { + if (needImmediateCommit) { return std::make_unique(); + } else { + return std::make_unique(); } } @@ -229,7 +229,7 @@ StoreOnlyFeedView::StoreOnlyFeedView(const Context &ctx, const PersistentParams _docType(nullptr), _lidReuseDelayer(ctx._writeService, _documentMetaStoreContext->get(), ctx._lidReuseDelayerConfig), _pendingLidsForDocStore(), - _pendingLidsForCommit(createUncommitedLidTracker(_lidReuseDelayer.allowEarlyAck())), + _pendingLidsForCommit(createUncommitedLidTracker(_lidReuseDelayer.getImmediateCommit())), _schema(ctx._schema), _writeService(ctx._writeService), _params(params), @@ -275,7 +275,7 @@ StoreOnlyFeedView::internalForceCommit(SerialNum serialNum, OnForceCommitDoneTyp void StoreOnlyFeedView::considerEarlyAck(FeedToken & token) { - if ( _lidReuseDelayer.allowEarlyAck() && token) { + if ( ! needCommit() && token) { token.reset(); } } @@ -327,7 +327,7 @@ StoreOnlyFeedView::internalPut(FeedToken token, const PutOperation &putOp) bool docAlreadyExists = putOp.getValidPrevDbdId(_params._subDbId); if (putOp.getValidDbdId(_params._subDbId)) { - bool immediateCommit = needImmediateCommit(); + bool immediateCommit = needCommit(); const document::GlobalId &gid = docId.getGlobalId(); std::shared_ptr onWriteDone = createPutDoneContext(std::move(token), std::move(uncommitted), @@ -345,8 +345,8 @@ StoreOnlyFeedView::internalPut(FeedToken token, const PutOperation &putOp) } bool -StoreOnlyFeedView::needImmediateCommit() const { - return _lidReuseDelayer.needImmediateCommit(); +StoreOnlyFeedView::needCommit() const { + return _lidReuseDelayer.getImmediateCommit(); } void @@ -483,7 +483,7 @@ StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp) auto uncommitted = _pendingLidsForCommit->produce(updOp.getLid()); considerEarlyAck(token); - bool immediateCommit = needImmediateCommit(); + bool immediateCommit = needCommit(); auto onWriteDone = createUpdateDoneContext(std::move(token), std::move(uncommitted), updOp.getUpdate()); UpdateScope updateScope(*_schema, upd); updateAttributes(serialNum, lid, upd, immediateCommit, onWriteDone, updateScope); @@ -657,7 +657,7 @@ StoreOnlyFeedView::internalRemove(FeedToken token, IPendingLidTracker::Token unc std::move(pendingNotifyRemoveDone), (explicitReuseLid ? lid : 0u), std::move(moveDoneCtx)); removeSummary(serialNum, lid, onWriteDone); - bool immediateCommit = needImmediateCommit(); + bool immediateCommit = needCommit(); removeAttributes(serialNum, lid, immediateCommit, onWriteDone); removeIndexedFields(serialNum, lid, immediateCommit, onWriteDone); } @@ -770,7 +770,7 @@ StoreOnlyFeedView::handleDeleteBucket(const DeleteBucketOperation &delOp) void StoreOnlyFeedView::internalDeleteBucket(const DeleteBucketOperation &delOp) { - bool immediateCommit = needImmediateCommit(); + bool immediateCommit = needCommit(); size_t rm_count = removeDocuments(delOp, true, immediateCommit); LOG(debug, "internalDeleteBucket(): docType(%s), bucket(%s), lidsToRemove(%zu)", _params._docTypeName.toString().c_str(), delOp.getBucketId().toString().c_str(), rm_count); @@ -809,7 +809,7 @@ StoreOnlyFeedView::handleMove(const MoveOperation &moveOp, IDestructorCallback:: PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(moveOp, docId.getGlobalId(), docId); bool docAlreadyExists = moveOp.getValidPrevDbdId(_params._subDbId); if (moveOp.getValidDbdId(_params._subDbId)) { - bool immediateCommit = needImmediateCommit(); + bool immediateCommit = needCommit(); const document::GlobalId &gid = docId.getGlobalId(); std::shared_ptr onWriteDone = createPutDoneContext(FeedToken(), _pendingLidsForCommit->produce(moveOp.getLid()), diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h index 4569e01f9fd..7d91ea86a22 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h @@ -161,7 +161,7 @@ private: void putSummary(SerialNum serialNum, Lid lid, DocumentSP doc, OnOperationDoneType onDone); void removeSummary(SerialNum serialNum, Lid lid, OnWriteDoneType onDone); void heartBeatSummary(SerialNum serialNum); - bool needImmediateCommit() const; + bool needCommit() const; bool useDocumentStore(SerialNum replaySerialNum) const { diff --git a/searchlib/src/vespa/searchlib/config/translogserver.def b/searchlib/src/vespa/searchlib/config/translogserver.def index defce8c3421..540895b2404 100644 --- a/searchlib/src/vespa/searchlib/config/translogserver.def +++ b/searchlib/src/vespa/searchlib/config/translogserver.def @@ -15,7 +15,7 @@ basedir string default="tmp" restart ## Use fsync after each commit. ## If not the below interval is used. -usefsync bool default=false +usefsync bool default=false restart ##Number of threads available for visiting/subscription. maxthreads int default=4 restart diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index bd7feec0598..9e0f1a8a1aa 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -113,12 +113,7 @@ Domain::addPart(SerialNum partId, bool isLastPart) { } } -Domain::~Domain() { - MonitorGuard guard(_currentChunkMonitor); - guard.broadcast(); - commitChunk(grabCurrentChunk(guard), guard); - _singleCommitter->shutdown().sync(); -} +Domain::~Domain() { } DomainInfo Domain::getDomainInfo() const @@ -323,73 +318,22 @@ Domain::optionallyRotateFile(SerialNum serialNum) { return dp; } -void -Domain::append(const Packet & packet, Writer::DoneCallback onDone) { - vespalib::MonitorGuard guard(_currentChunkMonitor); - if (_lastSerial >= packet.range().from()) { - throw runtime_error(fmt("Incoming serial number(%" PRIu64 ") must be bigger than the last one (%" PRIu64 ").", - packet.range().from(), _lastSerial)); - } else { - _lastSerial = packet.range().to(); - } - _currentChunk->add(packet, std::move(onDone)); - commitIfFull(guard); -} - Domain::CommitResult Domain::startCommit(DoneCallback onDone) { - vespalib::MonitorGuard guard(_currentChunkMonitor); - if ( !_currentChunk->empty() ) { - auto completed = grabCurrentChunk(guard); - completed->setCommitDoneCallback(std::move(onDone)); - CommitResult result(completed->createCommitResult()); - commitChunk(std::move(completed), guard); - return result; - } + (void) onDone; return CommitResult(); } void -Domain::commitIfFull(const vespalib::MonitorGuard &guard) { - if (_currentChunk->sizeBytes() > _config.getChunkSizeLimit()) { - auto completed = std::move(_currentChunk); - _currentChunk = std::make_unique(_config.getChunkSizeLimit(), completed->stealCallbacks()); - commitChunk(std::move(completed), guard); - } -} - -std::unique_ptr -Domain::grabCurrentChunk(const vespalib::MonitorGuard & guard) { - assert(guard.monitors(_currentChunkMonitor)); - auto chunk = std::move(_currentChunk); - _currentChunk = createCommitChunk(_config); - return chunk; -} - -void -Domain::commitChunk(std::unique_ptr chunk, const vespalib::MonitorGuard & chunkOrderGuard) { - assert(chunkOrderGuard.monitors(_currentChunkMonitor)); - _singleCommitter->execute( makeLambdaTask([this, chunk = std::move(chunk)]() mutable { - doCommit(std::move(chunk)); - })); -} - -void -Domain::doCommit(std::unique_ptr chunk) { - const Packet & packet = chunk->getPacket(); - if (packet.empty()) return; - +Domain::append(const Packet & packet, Writer::DoneCallback onDone) +{ + (void) onDone; vespalib::nbostream_longlivedbuf is(packet.getHandle().data(), packet.getHandle().size()); Packet::Entry entry; entry.deserialize(is); DomainPart::SP dp = optionallyRotateFile(entry.serial()); dp->commit(entry.serial(), packet); - if (_config.getFSyncOnCommit()) { - dp->sync(); - } cleanSessions(); - LOG(debug, "Releasing %zu acks and %zu entries and %zu bytes.", - chunk->getNumCallBacks(), chunk->getPacket().size(), chunk->sizeBytes()); } bool diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h index 041ec27cf23..7e77e6ef0ef 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h @@ -56,11 +56,6 @@ public: uint64_t size() const; Domain & setConfig(const DomainConfig & cfg); private: - void commitIfFull(const vespalib::MonitorGuard & guard); - - std::unique_ptr grabCurrentChunk(const vespalib::MonitorGuard & guard); - void commitChunk(std::unique_ptr chunk, const vespalib::MonitorGuard & chunkOrderGuard); - void doCommit(std::unique_ptr chunk); SerialNum begin(const vespalib::LockGuard & guard) const; SerialNum end(const vespalib::LockGuard & guard) const; size_t byteSize(const vespalib::LockGuard & guard) const; diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp index b7e02894e6b..8855183226d 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp @@ -252,7 +252,7 @@ DomainPart::buildPacketMapping(bool allowTruncate) DomainPart::DomainPart(const string & name, const string & baseDir, SerialNum s, Encoding encoding, uint8_t compressionLevel, const FileHeaderContext &fileHeaderContext, bool allowTruncate) - : _encoding(encoding), + : _encoding(encoding.getCrc(), Encoding::Compression::none), //TODO We do not yet support compression _compressionLevel(compressionLevel), _lock(), _fileLock(), @@ -396,19 +396,16 @@ DomainPart::commit(SerialNum firstSerial, const Packet &packet) if (_range.from() == 0) { _range.from(firstSerial); } - IChunk::UP chunk = IChunk::create(_encoding, _compressionLevel); for (size_t i(0); h.size() > 0; i++) { //LOG(spam, //"Pos(%d) Len(%d), Lim(%d), Remaining(%d)", //h.getPos(), h.getLength(), h.getLimit(), h.getRemaining()); + IChunk::UP chunk = IChunk::create(_encoding, _compressionLevel); Packet::Entry entry; entry.deserialize(h); if (_range.to() < entry.serial()) { chunk->add(entry); - if (_encoding.getCompression() == Encoding::Compression::none) { - write(*_transLog, *chunk); - chunk = IChunk::create(_encoding, _compressionLevel); - } + write(*_transLog, *chunk); _sz++; _range.to(entry.serial()); } else { @@ -416,9 +413,6 @@ DomainPart::commit(SerialNum firstSerial, const Packet &packet) entry.serial(), _range.to())); } } - if ( ! chunk->getEntries().empty()) { - write(*_transLog, *chunk); - } bool merged(false); LockGuard guard(_lock); diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index 0c0c9186e12..7be3dd708a5 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -572,11 +572,8 @@ TransLogServer::domainCommit(FRT_RPCRequest *req) Packet packet(params[1]._data._buf, params[1]._data._len); try { vespalib::Gate gate; - { - // Need to scope in order to drain out all the callbacks. - domain->append(packet, make_shared(gate)); - auto keep = domain->startCommit(make_shared()); - } + domain->append(packet, make_shared(gate)); + auto keep = domain->startCommit(make_shared()); gate.await(); ret.AddInt32(0); ret.AddString("ok"); -- cgit v1.2.3 From 715f4d8faa49d7d87a4e8a7a8ecadb7c8de66ef7 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sat, 26 Sep 2020 20:01:49 +0000 Subject: Separate immediateCommit and allowEarlyAck --- .../documentmetastore/lid_reuse_delayer_config.cpp | 2 ++ .../documentmetastore/lid_reuse_delayer_config.h | 2 ++ .../proton/documentmetastore/lidreusedelayer.cpp | 1 + .../proton/documentmetastore/lidreusedelayer.h | 4 +++- .../proton/server/document_db_maintenance_config.h | 1 + .../proton/server/maintenance_jobs_injector.cpp | 2 +- .../searchcore/proton/server/storeonlyfeedview.cpp | 26 +++++++++++----------- .../searchcore/proton/server/storeonlyfeedview.h | 2 +- 8 files changed, 24 insertions(+), 16 deletions(-) diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp index b04bac5ef26..1f979d1566c 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp @@ -7,6 +7,7 @@ namespace proton::documentmetastore { LidReuseDelayerConfig::LidReuseDelayerConfig(const DocumentDBConfig & configSnapshot) : _visibilityDelay(configSnapshot.getMaintenanceConfigSP()->getVisibilityDelay()), + _allowEarlyAck(configSnapshot.getMaintenanceConfigSP()->allowEarlyAck()), _hasIndexedOrAttributeFields(configSnapshot.getSchemaSP()->getNumIndexFields() > 0 || configSnapshot.getSchemaSP()->getNumAttributeFields() > 0) { @@ -18,6 +19,7 @@ LidReuseDelayerConfig::LidReuseDelayerConfig() LidReuseDelayerConfig::LidReuseDelayerConfig(vespalib::duration visibilityDelay, bool hasIndexedOrAttributeFields_in) : _visibilityDelay(visibilityDelay), + _allowEarlyAck(visibilityDelay > 1ms), _hasIndexedOrAttributeFields(hasIndexedOrAttributeFields_in) { } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h index 82dab433a22..c81a2ff399f 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h @@ -15,6 +15,7 @@ class LidReuseDelayerConfig { private: vespalib::duration _visibilityDelay; + bool _allowEarlyAck; bool _hasIndexedOrAttributeFields; public: LidReuseDelayerConfig(); @@ -22,6 +23,7 @@ public: explicit LidReuseDelayerConfig(const DocumentDBConfig &configSnapshot); vespalib::duration visibilityDelay() const { return _visibilityDelay; } bool hasIndexedOrAttributeFields() const { return _hasIndexedOrAttributeFields; } + bool allowEarlyAck() const { return _allowEarlyAck; } }; } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp index 03dfd83a132..ed2202c830b 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp @@ -16,6 +16,7 @@ LidReuseDelayer::LidReuseDelayer(IThreadingService &writeService, IStore &docume : _writeService(writeService), _documentMetaStore(documentMetaStore), _immediateCommit(config.visibilityDelay() == vespalib::duration::zero()), + _allowEarlyAck(config.allowEarlyAck()), _config(config), _pendingLids() { diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h index 5f1de878b4a..ba407ab57f8 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h @@ -27,6 +27,7 @@ class LidReuseDelayer searchcorespi::index::IThreadingService &_writeService; IStore &_documentMetaStore; const bool _immediateCommit; + const bool _allowEarlyAck; LidReuseDelayerConfig _config; std::vector _pendingLids; // lids waiting for commit @@ -38,7 +39,8 @@ public: bool delayReuse(const std::vector &lids); std::vector getReuseLids(); - bool getImmediateCommit() const { return _immediateCommit; } + bool needImmediateCommit() const { return _immediateCommit; } + bool allowEarlyAck() const { return _allowEarlyAck; } const LidReuseDelayerConfig & getConfig() const { return _config; } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h index 4c0485baec6..388a1101dcf 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h @@ -135,6 +135,7 @@ public: } vespalib::duration getVisibilityDelay() const { return _visibilityDelay; } bool hasVisibilityDelay() const { return _visibilityDelay > vespalib::duration::zero(); } + bool allowEarlyAck() const { return hasVisibilityDelay(); } const DocumentDBLidSpaceCompactionConfig &getLidSpaceCompactionConfig() const { return _lidSpaceCompaction; } diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp index d94bb2e3d03..d4b542a0af8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp @@ -105,7 +105,7 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller, AttributeUsageFilter &attributeUsageFilter) { controller.registerJobInMasterThread(std::make_unique(hbHandler, config.getHeartBeatConfig())); controller.registerJobInDefaultPool(std::make_unique(scPruner, config.getSessionCachePruneInterval())); - if (config.hasVisibilityDelay()) { + if (config.hasVisibilityDelay() && config.allowEarlyAck()) { controller.registerJobInMasterThread(std::make_unique(commit, config.getVisibilityDelay())); } const MaintenanceDocumentSubDB &mRemSubDB(controller.getRemSubDB()); diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index 217a3bb24d3..9927a2d2ce0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -210,11 +210,11 @@ moveMetaData(documentmetastore::IStore &meta_store, const DocumentId & doc_id, c } std::unique_ptr -createUncommitedLidTracker(bool needImmediateCommit) { - if (needImmediateCommit) { - return std::make_unique(); - } else { +createUncommitedLidTracker(bool allowEarlyAck) { + if (allowEarlyAck) { return std::make_unique(); + } else { + return std::make_unique(); } } @@ -229,7 +229,7 @@ StoreOnlyFeedView::StoreOnlyFeedView(const Context &ctx, const PersistentParams _docType(nullptr), _lidReuseDelayer(ctx._writeService, _documentMetaStoreContext->get(), ctx._lidReuseDelayerConfig), _pendingLidsForDocStore(), - _pendingLidsForCommit(createUncommitedLidTracker(_lidReuseDelayer.getImmediateCommit())), + _pendingLidsForCommit(createUncommitedLidTracker(_lidReuseDelayer.allowEarlyAck())), _schema(ctx._schema), _writeService(ctx._writeService), _params(params), @@ -275,7 +275,7 @@ StoreOnlyFeedView::internalForceCommit(SerialNum serialNum, OnForceCommitDoneTyp void StoreOnlyFeedView::considerEarlyAck(FeedToken & token) { - if ( ! needCommit() && token) { + if ( _lidReuseDelayer.allowEarlyAck() && token) { token.reset(); } } @@ -327,7 +327,7 @@ StoreOnlyFeedView::internalPut(FeedToken token, const PutOperation &putOp) bool docAlreadyExists = putOp.getValidPrevDbdId(_params._subDbId); if (putOp.getValidDbdId(_params._subDbId)) { - bool immediateCommit = needCommit(); + bool immediateCommit = needImmediateCommit(); const document::GlobalId &gid = docId.getGlobalId(); std::shared_ptr onWriteDone = createPutDoneContext(std::move(token), std::move(uncommitted), @@ -345,8 +345,8 @@ StoreOnlyFeedView::internalPut(FeedToken token, const PutOperation &putOp) } bool -StoreOnlyFeedView::needCommit() const { - return _lidReuseDelayer.getImmediateCommit(); +StoreOnlyFeedView::needImmediateCommit() const { + return _lidReuseDelayer.needImmediateCommit(); } void @@ -483,7 +483,7 @@ StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp) auto uncommitted = _pendingLidsForCommit->produce(updOp.getLid()); considerEarlyAck(token); - bool immediateCommit = needCommit(); + bool immediateCommit = needImmediateCommit(); auto onWriteDone = createUpdateDoneContext(std::move(token), std::move(uncommitted), updOp.getUpdate()); UpdateScope updateScope(*_schema, upd); updateAttributes(serialNum, lid, upd, immediateCommit, onWriteDone, updateScope); @@ -657,7 +657,7 @@ StoreOnlyFeedView::internalRemove(FeedToken token, IPendingLidTracker::Token unc std::move(pendingNotifyRemoveDone), (explicitReuseLid ? lid : 0u), std::move(moveDoneCtx)); removeSummary(serialNum, lid, onWriteDone); - bool immediateCommit = needCommit(); + bool immediateCommit = needImmediateCommit(); removeAttributes(serialNum, lid, immediateCommit, onWriteDone); removeIndexedFields(serialNum, lid, immediateCommit, onWriteDone); } @@ -770,7 +770,7 @@ StoreOnlyFeedView::handleDeleteBucket(const DeleteBucketOperation &delOp) void StoreOnlyFeedView::internalDeleteBucket(const DeleteBucketOperation &delOp) { - bool immediateCommit = needCommit(); + bool immediateCommit = needImmediateCommit(); size_t rm_count = removeDocuments(delOp, true, immediateCommit); LOG(debug, "internalDeleteBucket(): docType(%s), bucket(%s), lidsToRemove(%zu)", _params._docTypeName.toString().c_str(), delOp.getBucketId().toString().c_str(), rm_count); @@ -809,7 +809,7 @@ StoreOnlyFeedView::handleMove(const MoveOperation &moveOp, IDestructorCallback:: PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(moveOp, docId.getGlobalId(), docId); bool docAlreadyExists = moveOp.getValidPrevDbdId(_params._subDbId); if (moveOp.getValidDbdId(_params._subDbId)) { - bool immediateCommit = needCommit(); + bool immediateCommit = needImmediateCommit(); const document::GlobalId &gid = docId.getGlobalId(); std::shared_ptr onWriteDone = createPutDoneContext(FeedToken(), _pendingLidsForCommit->produce(moveOp.getLid()), diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h index 7d91ea86a22..4569e01f9fd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h @@ -161,7 +161,7 @@ private: void putSummary(SerialNum serialNum, Lid lid, DocumentSP doc, OnOperationDoneType onDone); void removeSummary(SerialNum serialNum, Lid lid, OnWriteDoneType onDone); void heartBeatSummary(SerialNum serialNum); - bool needCommit() const; + bool needImmediateCommit() const; bool useDocumentStore(SerialNum replaySerialNum) const { -- cgit v1.2.3 From 76555827b7a0c507d544a6c2a280ab8ed474e73e Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sat, 26 Sep 2020 20:18:52 +0000 Subject: Allow early ack on non-zero visibility delay. --- .../searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp index 1f979d1566c..9db1fee58e1 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp @@ -19,7 +19,7 @@ LidReuseDelayerConfig::LidReuseDelayerConfig() LidReuseDelayerConfig::LidReuseDelayerConfig(vespalib::duration visibilityDelay, bool hasIndexedOrAttributeFields_in) : _visibilityDelay(visibilityDelay), - _allowEarlyAck(visibilityDelay > 1ms), + _allowEarlyAck(visibilityDelay > vespalib::duration::zero()), _hasIndexedOrAttributeFields(hasIndexedOrAttributeFields_in) { } -- cgit v1.2.3 From ec864947f2d63fa36c906c25a71a8fca74b17575 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sat, 26 Sep 2020 20:38:34 +0000 Subject: Drop virtual keyword on override. --- .../src/vespa/searchcore/proton/server/lid_space_compaction_job.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h index 37497eaa998..35549f21471 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h @@ -63,13 +63,13 @@ public: ~LidSpaceCompactionJob(); // Implements IDiskMemUsageListener - virtual void notifyDiskMemUsage(DiskMemUsageState state) override; + void notifyDiskMemUsage(DiskMemUsageState state) override; // Implements IClusterStateChangedNofifier - virtual void notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) override; + void notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) override; // Implements IMaintenanceJob - virtual bool run() override; + bool run() override; }; } // namespace proton -- cgit v1.2.3 From 5992b83214f95dd353932fdc46b5a5123ae63802 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sat, 26 Sep 2020 20:44:32 +0000 Subject: Ensure that contexts have the proper lifetime. --- searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp | 2 +- searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h | 4 ++++ .../vespa/searchcore/proton/server/lid_space_compaction_job.cpp | 2 +- searchlib/src/vespa/searchlib/config/translogserver.def | 2 +- searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp | 7 +++++-- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 37dfddf0c2c..282ec3c9ace 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -510,7 +510,7 @@ FeedHandler::startCommit(DoneCallback onDone) { void FeedHandler::storeOperationSync(const FeedOperation &op) { vespalib::Gate gate; - appendOperation(op, make_shared(gate)); + appendAndCommitOperation(op, make_shared(gate)); gate.await(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h index b276779c2ee..c3b76a9db75 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h @@ -22,6 +22,10 @@ struct IOperationStorer */ virtual void appendOperation(const FeedOperation &op, DoneCallback onDone) = 0; [[nodiscard]] virtual CommitResult startCommit(DoneCallback onDone) = 0; + void appendAndCommitOperation(const FeedOperation &op, DoneCallback onDone) { + appendOperation(op, onDone); + (void) startCommit(std::move(onDone)); + } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp index d423e095ad9..468850b4409 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp @@ -82,7 +82,7 @@ LidSpaceCompactionJob::compactLidSpace(const LidUsageStats &stats) uint32_t wantedLidLimit = stats.getHighestUsedLid() + 1; CompactLidSpaceOperation op(_handler.getSubDbId(), wantedLidLimit); vespalib::Gate gate; - _opStorer.appendOperation(op, std::make_shared(gate)); + _opStorer.appendAndCommitOperation(op, std::make_shared(gate)); gate.await(); _handler.handleCompactLidSpace(op); EventLogger::lidSpaceCompactionComplete(_handler.getName(), wantedLidLimit); diff --git a/searchlib/src/vespa/searchlib/config/translogserver.def b/searchlib/src/vespa/searchlib/config/translogserver.def index 540895b2404..defce8c3421 100644 --- a/searchlib/src/vespa/searchlib/config/translogserver.def +++ b/searchlib/src/vespa/searchlib/config/translogserver.def @@ -15,7 +15,7 @@ basedir string default="tmp" restart ## Use fsync after each commit. ## If not the below interval is used. -usefsync bool default=false restart +usefsync bool default=false ##Number of threads available for visiting/subscription. maxthreads int default=4 restart diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index 7be3dd708a5..0c0c9186e12 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -572,8 +572,11 @@ TransLogServer::domainCommit(FRT_RPCRequest *req) Packet packet(params[1]._data._buf, params[1]._data._len); try { vespalib::Gate gate; - domain->append(packet, make_shared(gate)); - auto keep = domain->startCommit(make_shared()); + { + // Need to scope in order to drain out all the callbacks. + domain->append(packet, make_shared(gate)); + auto keep = domain->startCommit(make_shared()); + } gate.await(); ret.AddInt32(0); ret.AddString("ok"); -- cgit v1.2.3 From 434ebddfeb45f8a769989447a66adfb88e1e3596 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sat, 26 Sep 2020 20:58:52 +0000 Subject: If compression type not != NONE put all operations in one chunk. --- searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp index 8855183226d..b7e02894e6b 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp @@ -252,7 +252,7 @@ DomainPart::buildPacketMapping(bool allowTruncate) DomainPart::DomainPart(const string & name, const string & baseDir, SerialNum s, Encoding encoding, uint8_t compressionLevel, const FileHeaderContext &fileHeaderContext, bool allowTruncate) - : _encoding(encoding.getCrc(), Encoding::Compression::none), //TODO We do not yet support compression + : _encoding(encoding), _compressionLevel(compressionLevel), _lock(), _fileLock(), @@ -396,16 +396,19 @@ DomainPart::commit(SerialNum firstSerial, const Packet &packet) if (_range.from() == 0) { _range.from(firstSerial); } + IChunk::UP chunk = IChunk::create(_encoding, _compressionLevel); for (size_t i(0); h.size() > 0; i++) { //LOG(spam, //"Pos(%d) Len(%d), Lim(%d), Remaining(%d)", //h.getPos(), h.getLength(), h.getLimit(), h.getRemaining()); - IChunk::UP chunk = IChunk::create(_encoding, _compressionLevel); Packet::Entry entry; entry.deserialize(h); if (_range.to() < entry.serial()) { chunk->add(entry); - write(*_transLog, *chunk); + if (_encoding.getCompression() == Encoding::Compression::none) { + write(*_transLog, *chunk); + chunk = IChunk::create(_encoding, _compressionLevel); + } _sz++; _range.to(entry.serial()); } else { @@ -413,6 +416,9 @@ DomainPart::commit(SerialNum firstSerial, const Packet &packet) entry.serial(), _range.to())); } } + if ( ! chunk->getEntries().empty()) { + write(*_transLog, *chunk); + } bool merged(false); LockGuard guard(_lock); -- cgit v1.2.3 From aed7313cce497e15a49435bc9330513f3aa7fdc1 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sun, 27 Sep 2020 08:00:34 +0000 Subject: Add logging of forcecommit calls. --- searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index 9927a2d2ce0..323c11c66ca 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -263,7 +263,7 @@ StoreOnlyFeedView::forceCommit(SerialNum serialNum, DoneCallback onDone) void StoreOnlyFeedView::internalForceCommit(SerialNum serialNum, OnForceCommitDoneType onCommitDone) { - (void) serialNum; + LOG(spam, "internalForceCommit: serial=%" PRIu64 ".", serialNum); _writeService.summary().execute(makeLambdaTask([onDone=onCommitDone]() {(void) onDone;})); std::vector lidsToReuse; lidsToReuse = _lidReuseDelayer.getReuseLids(); -- cgit v1.2.3 From e181ed562204e3ed03aadc7872351322653e76d7 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sun, 27 Sep 2020 21:05:35 +0000 Subject: Spam to debug --- searchcore/src/vespa/searchcore/proton/common/feeddebugger.cpp | 2 +- searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/searchcore/src/vespa/searchcore/proton/common/feeddebugger.cpp b/searchcore/src/vespa/searchcore/proton/common/feeddebugger.cpp index 1aa89b78966..73f909c115c 100644 --- a/searchcore/src/vespa/searchcore/proton/common/feeddebugger.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/feeddebugger.cpp @@ -41,7 +41,7 @@ FeedDebugger::FeedDebugger() : _enableDebugging = ! (_debugLidList.empty() && _debugDocIdList.empty()); } -FeedDebugger::~FeedDebugger() {} +FeedDebugger::~FeedDebugger() = default; ns_log::Logger::LogLevel FeedDebugger::getDebugDebuggerInternal(uint32_t lid, const document::DocumentId * docid) const diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index 323c11c66ca..bda94c22adc 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -263,7 +263,7 @@ StoreOnlyFeedView::forceCommit(SerialNum serialNum, DoneCallback onDone) void StoreOnlyFeedView::internalForceCommit(SerialNum serialNum, OnForceCommitDoneType onCommitDone) { - LOG(spam, "internalForceCommit: serial=%" PRIu64 ".", serialNum); + LOG(debug, "internalForceCommit: serial=%" PRIu64 ".", serialNum); _writeService.summary().execute(makeLambdaTask([onDone=onCommitDone]() {(void) onDone;})); std::vector lidsToReuse; lidsToReuse = _lidReuseDelayer.getReuseLids(); -- cgit v1.2.3 From 661a39d3cfb548bfdac313cd85d8230828fc1b52 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Mon, 28 Sep 2020 06:48:26 +0200 Subject: Revert "Reapply "Upgrade to Curator 4"" --- .../provision/persistence/CuratorDatabaseTest.java | 1 - parent/pom.xml | 4 +- zkfacade/pom.xml | 1 - .../main/java/com/yahoo/vespa/curator/Curator.java | 2 +- .../com/yahoo/vespa/curator/mock/MockCurator.java | 491 ++------------------- .../apache/curator/framework/api/package-info.java | 2 +- .../framework/api/transaction/package-info.java | 2 +- .../curator/framework/listen/package-info.java | 2 +- .../org/apache/curator/framework/package-info.java | 2 +- .../framework/recipes/atomic/package-info.java | 2 +- .../framework/recipes/barriers/package-info.java | 2 +- .../framework/recipes/cache/package-info.java | 2 +- .../framework/recipes/locks/package-info.java | 2 +- .../curator/framework/state/package-info.java | 2 +- .../main/java/org/apache/curator/package-info.java | 2 +- .../org/apache/curator/retry/package-info.java | 2 +- 16 files changed, 58 insertions(+), 463 deletions(-) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseTest.java index 44026f835dd..149510bdc97 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseTest.java @@ -149,7 +149,6 @@ public class CuratorDatabaseTest { this.task = task; } - @SuppressWarnings("deprecation") @Override public org.apache.curator.framework.api.transaction.CuratorTransaction and(org.apache.curator.framework.api.transaction.CuratorTransaction transaction) { task.run(); diff --git a/parent/pom.xml b/parent/pom.xml index da30e160d1b..a9998af3962 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -757,9 +757,9 @@ zkfacade/src/main/java/org/apache/curator/**/package-info.java using something like find zkfacade/src/main/java/org/apache/curator -name package-info.java | \ - xargs perl -pi -e 's/major = [0-9]+, minor = [0-9]+, micro = [0-9]+/major = 4, minor = 3, micro = 0/g' + xargs perl -pi -e 's/major = [0-9]+, minor = [0-9]+, micro = [0-9]+/major = 2, minor = 9, micro = 1/g' --> - 4.3.0 + 2.13.0 4.5.2 3.6.1 5.6.2 diff --git a/zkfacade/pom.xml b/zkfacade/pom.xml index c160d935788..de542b89b67 100644 --- a/zkfacade/pom.xml +++ b/zkfacade/pom.xml @@ -48,7 +48,6 @@ org.apache.curator curator-test - 4.2.0 test diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java index 51cb387b734..37b1fa1c9fb 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java @@ -279,7 +279,7 @@ public class Curator implements AutoCloseable { */ public void createAtomically(Path... paths) { try { - @SuppressWarnings("deprecation") CuratorTransaction transaction = framework().inTransaction(); + CuratorTransaction transaction = framework().inTransaction(); for (Path path : paths) { if ( ! exists(path)) { transaction = transaction.create().forPath(path.getAbsolute(), new byte[0]).and(); diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java index 1f583ada7a1..3da7678c44e 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java @@ -12,40 +12,31 @@ import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.recipes.CuratorLockException; import org.apache.curator.CuratorZookeeperClient; import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.WatcherRemoveCuratorFramework; import org.apache.curator.framework.api.ACLBackgroundPathAndBytesable; import org.apache.curator.framework.api.ACLCreateModeBackgroundPathAndBytesable; import org.apache.curator.framework.api.ACLCreateModePathAndBytesable; -import org.apache.curator.framework.api.ACLCreateModeStatBackgroundPathAndBytesable; import org.apache.curator.framework.api.ACLPathAndBytesable; -import org.apache.curator.framework.api.ACLableExistBuilderMain; import org.apache.curator.framework.api.BackgroundCallback; import org.apache.curator.framework.api.BackgroundPathAndBytesable; import org.apache.curator.framework.api.BackgroundPathable; import org.apache.curator.framework.api.BackgroundVersionable; import org.apache.curator.framework.api.ChildrenDeletable; -import org.apache.curator.framework.api.CreateBackgroundModeStatACLable; +import org.apache.curator.framework.api.CreateBackgroundModeACLable; import org.apache.curator.framework.api.CreateBuilder; -import org.apache.curator.framework.api.CreateBuilder2; -import org.apache.curator.framework.api.CreateBuilderMain; -import org.apache.curator.framework.api.CreateProtectACLCreateModePathAndBytesable; import org.apache.curator.framework.api.CuratorListener; import org.apache.curator.framework.api.CuratorWatcher; import org.apache.curator.framework.api.DeleteBuilder; -import org.apache.curator.framework.api.DeleteBuilderMain; import org.apache.curator.framework.api.ErrorListenerPathAndBytesable; import org.apache.curator.framework.api.ErrorListenerPathable; import org.apache.curator.framework.api.ExistsBuilder; +import org.apache.curator.framework.api.ExistsBuilderMain; import org.apache.curator.framework.api.GetACLBuilder; import org.apache.curator.framework.api.GetChildrenBuilder; -import org.apache.curator.framework.api.GetConfigBuilder; import org.apache.curator.framework.api.GetDataBuilder; import org.apache.curator.framework.api.GetDataWatchBackgroundStatable; import org.apache.curator.framework.api.PathAndBytesable; import org.apache.curator.framework.api.Pathable; -import org.apache.curator.framework.api.ProtectACLCreateModeStatPathAndBytesable; -import org.apache.curator.framework.api.ReconfigBuilder; -import org.apache.curator.framework.api.RemoveWatchesBuilder; +import org.apache.curator.framework.api.ProtectACLCreateModePathAndBytesable; import org.apache.curator.framework.api.SetACLBuilder; import org.apache.curator.framework.api.SetDataBackgroundVersionable; import org.apache.curator.framework.api.SetDataBuilder; @@ -54,16 +45,13 @@ import org.apache.curator.framework.api.UnhandledErrorListener; import org.apache.curator.framework.api.VersionPathAndBytesable; import org.apache.curator.framework.api.WatchPathable; import org.apache.curator.framework.api.Watchable; -import org.apache.curator.framework.api.transaction.CuratorMultiTransaction; import org.apache.curator.framework.api.transaction.CuratorTransaction; import org.apache.curator.framework.api.transaction.CuratorTransactionBridge; import org.apache.curator.framework.api.transaction.CuratorTransactionFinal; import org.apache.curator.framework.api.transaction.CuratorTransactionResult; import org.apache.curator.framework.api.transaction.TransactionCheckBuilder; import org.apache.curator.framework.api.transaction.TransactionCreateBuilder; -import org.apache.curator.framework.api.transaction.TransactionCreateBuilder2; import org.apache.curator.framework.api.transaction.TransactionDeleteBuilder; -import org.apache.curator.framework.api.transaction.TransactionOp; import org.apache.curator.framework.api.transaction.TransactionSetDataBuilder; import org.apache.curator.framework.imps.CuratorFrameworkState; import org.apache.curator.framework.listen.Listenable; @@ -76,8 +64,6 @@ import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; import org.apache.curator.framework.recipes.locks.InterProcessLock; import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreMutex; -import org.apache.curator.framework.schema.SchemaSet; -import org.apache.curator.framework.state.ConnectionStateErrorPolicy; import org.apache.curator.framework.state.ConnectionStateListener; import org.apache.curator.utils.EnsurePath; import org.apache.zookeeper.CreateMode; @@ -85,7 +71,6 @@ import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Stat; -import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; import java.nio.file.Paths; import java.time.Duration; @@ -97,7 +82,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; @@ -649,101 +633,64 @@ public class MockCurator extends Curator { // ----- file system methods above. ----- // ----- There's nothing to see unless you are interested in an illustration of ----- // ----- the folly of fluent API's or, more generally, mankind. ----- - private abstract static class MockProtectACLCreateModeStatPathAndBytesable - implements ProtectACLCreateModeStatPathAndBytesable { - public BackgroundPathAndBytesable withACL(List list) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } + private abstract class MockBackgroundACLPathAndBytesableBuilder implements PathAndBytesable, ProtectACLCreateModePathAndBytesable { - public BackgroundPathAndBytesable withACL(List list, boolean b) { + public BackgroundPathAndBytesable withACL(List list) { throw new UnsupportedOperationException("Not implemented in MockCurator"); } - public ProtectACLCreateModeStatPathAndBytesable withMode(CreateMode createMode) { + public ACLBackgroundPathAndBytesable withMode(CreateMode createMode) { throw new UnsupportedOperationException("Not implemented in MockCurator"); } @Override - public ACLCreateModeBackgroundPathAndBytesable withProtection() { - return null; - } - - @Override - public ErrorListenerPathAndBytesable inBackground() { - return null; - } - - @Override - public ErrorListenerPathAndBytesable inBackground(Object o) { - return null; - } - - @Override - public ErrorListenerPathAndBytesable inBackground(BackgroundCallback backgroundCallback) { - return null; - } - - @Override - public ErrorListenerPathAndBytesable inBackground(BackgroundCallback backgroundCallback, Object o) { - return null; - } - - @Override - public ErrorListenerPathAndBytesable inBackground(BackgroundCallback backgroundCallback, Executor executor) { - return null; + public ACLCreateModeBackgroundPathAndBytesable withProtection() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); } - @Override - public ErrorListenerPathAndBytesable inBackground(BackgroundCallback backgroundCallback, Object o, Executor executor) { - return null; + public T forPath(String s, byte[] bytes) throws Exception { + throw new UnsupportedOperationException("Not implemented in MockCurator"); } - @Override - public ACLBackgroundPathAndBytesable storingStatIn(Stat stat) { - return null; + public T forPath(String s) throws Exception { + throw new UnsupportedOperationException("Not implemented in MockCurator"); } } - private class MockCreateBuilder implements CreateBuilder { + private class MockCreateBuilder extends MockBackgroundACLPathAndBytesableBuilder implements CreateBuilder { private boolean createParents = false; private CreateMode createMode = CreateMode.PERSISTENT; @Override - public ProtectACLCreateModeStatPathAndBytesable creatingParentsIfNeeded() { + public ProtectACLCreateModePathAndBytesable creatingParentsIfNeeded() { createParents = true; - return new MockProtectACLCreateModeStatPathAndBytesable<>() { - - @Override - public String forPath(String s, byte[] bytes) throws Exception { - return createNode(s, bytes, createParents, createMode, fileSystem.root(), listeners); - } - - @Override - public String forPath(String s) throws Exception { - return createNode(s, new byte[0], createParents, createMode, fileSystem.root(), listeners); - } - - }; + return this; } @Override - public ProtectACLCreateModeStatPathAndBytesable creatingParentContainersIfNeeded() { - return new MockProtectACLCreateModeStatPathAndBytesable<>() { + public ACLCreateModeBackgroundPathAndBytesable withProtection() { + // Protection against the server crashing after creating the file but before returning to the client. + // Not relevant for an in-memory mock, obviously + return this; + } - @Override - public String forPath(String s, byte[] bytes) throws Exception { - return createNode(s, bytes, createParents, createMode, fileSystem.root(), listeners); - } + public ACLBackgroundPathAndBytesable withMode(CreateMode createMode) { + this.createMode = createMode; + return this; + } - @Override - public String forPath(String s) throws Exception { - return createNode(s, new byte[0], createParents, createMode, fileSystem.root(), listeners); - } + @Override + public CreateBackgroundModeACLable compressed() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } - }; + @Override + public ProtectACLCreateModePathAndBytesable creatingParentContainersIfNeeded() { + // TODO: Add proper support for container nodes, see https://issues.apache.org/jira/browse/ZOOKEEPER-2163. + return creatingParentsIfNeeded(); } @Override @@ -752,11 +699,6 @@ public class MockCurator extends Curator { throw new UnsupportedOperationException("Not implemented in MockCurator"); } - @Override - public ACLCreateModeStatBackgroundPathAndBytesable withProtection() { - return null; - } - public String forPath(String s) throws Exception { return createNode(s, new byte[0], createParents, createMode, fileSystem.root(), listeners); } @@ -794,50 +736,9 @@ public class MockCurator extends Curator { public ErrorListenerPathAndBytesable inBackground(BackgroundCallback backgroundCallback, Object o, Executor executor) { throw new UnsupportedOperationException("Not implemented in MockCurator"); } - - @Override - public CreateBuilderMain withTtl(long l) { - return null; - } - - @Override - public CreateBuilder2 orSetData() { - return null; - } - - @Override - public CreateBuilder2 orSetData(int i) { - return null; - } - - @Override - public CreateBackgroundModeStatACLable compressed() { - return null; - } - - @Override - public CreateProtectACLCreateModePathAndBytesable storingStatIn(Stat stat) { - return null; - } - - @Override - public BackgroundPathAndBytesable withACL(List list) { - return null; - } - - @Override - public ACLBackgroundPathAndBytesable withMode(CreateMode createMode) { - this.createMode = createMode; - return this; - } - - @Override - public BackgroundPathAndBytesable withACL(List list, boolean b) { - return null; - } } - private static class MockBackgroundPathableBuilder implements BackgroundPathable, Watchable> { + private class MockBackgroundPathableBuilder implements BackgroundPathable, Watchable> { @Override public ErrorListenerPathable inBackground() { @@ -920,12 +821,7 @@ public class MockCurator extends Curator { } @Override - public ACLableExistBuilderMain creatingParentsIfNeeded() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ACLableExistBuilderMain creatingParentContainersIfNeeded() { + public ExistsBuilderMain creatingParentContainersIfNeeded() { throw new UnsupportedOperationException("Not implemented in MockCurator"); } } @@ -955,10 +851,6 @@ public class MockCurator extends Curator { return null; } - @Override - public DeleteBuilderMain quietly() { - return this; - } } private class MockGetDataBuilder extends MockBackgroundPathableBuilder implements GetDataBuilder { @@ -968,48 +860,18 @@ public class MockCurator extends Curator { throw new UnsupportedOperationException("Not implemented in MockCurator"); } - public byte[] forPath(String path) throws Exception { - return getData(path, fileSystem.root()); - } - @Override - public ErrorListenerPathable inBackground() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathable inBackground(Object o) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathable inBackground(BackgroundCallback backgroundCallback) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathable inBackground(BackgroundCallback backgroundCallback, Object o) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathable inBackground(BackgroundCallback backgroundCallback, Executor executor) { + public WatchPathable storingStatIn(Stat stat) { throw new UnsupportedOperationException("Not implemented in MockCurator"); } - @Override - public ErrorListenerPathable inBackground(BackgroundCallback backgroundCallback, Object o, Executor executor) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); + public byte[] forPath(String path) throws Exception { + return getData(path, fileSystem.root()); } - @Override - public WatchPathable storingStatIn(Stat stat) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } } - // extends MockBackgroundACLPathAndBytesableBuilder - private class MockSetDataBuilder implements SetDataBuilder { + private class MockSetDataBuilder extends MockBackgroundACLPathAndBytesableBuilder implements SetDataBuilder { @Override public SetDataBackgroundVersionable compressed() { @@ -1027,11 +889,6 @@ public class MockCurator extends Curator { return null; } - @Override - public Stat forPath(String s) throws Exception { - return null; - } - @Override public ErrorListenerPathAndBytesable inBackground() { throw new UnsupportedOperationException("Not implemented in MockCurator"); @@ -1133,6 +990,11 @@ public class MockCurator extends Curator { private CreateMode createMode = CreateMode.PERSISTENT; + @Override + public PathAndBytesable withACL(List list) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + @Override public ACLCreateModePathAndBytesable compressed() { throw new UnsupportedOperationException("Not implemented in MockCurator"); @@ -1156,20 +1018,6 @@ public class MockCurator extends Curator { return new MockCuratorTransactionBridge(); } - @Override - public TransactionCreateBuilder2 withTtl(long l) { - return this; - } - - @Override - public Object withACL(List list, boolean b) { - return this; - } - - @Override - public Object withACL(List list) { - return this; - } } private class MockTransactionDeleteBuilder implements TransactionDeleteBuilder { @@ -1306,31 +1154,11 @@ public class MockCurator extends Curator { throw new UnsupportedOperationException("Not implemented in MockCurator"); } - @Override - public ReconfigBuilder reconfig() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public GetConfigBuilder getConfig() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - @Override public CuratorTransaction inTransaction() { return new MockCuratorTransactionFinal(); } - @Override - public CuratorMultiTransaction transaction() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public TransactionOp transactionOp() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - @Override @Deprecated public void sync(String path, Object backgroundContextObject) { @@ -1399,242 +1227,11 @@ public class MockCurator extends Curator { } - @Override - public WatcherRemoveCuratorFramework newWatcherRemoveCuratorFramework() { - return new WatcherRemoveCuratorFramework() { - @Override - public void removeWatchers() { - - } - - @Override - public void start() { - - } - - @Override - public void close() { - - } - - @Override - public CuratorFrameworkState getState() { - return null; - } - - @Override - public boolean isStarted() { - return false; - } - - @Override - public CreateBuilder create() { - return null; - } - - @Override - public DeleteBuilder delete() { - return null; - } - - @Override - public ExistsBuilder checkExists() { - return null; - } - - @Override - public GetDataBuilder getData() { - return null; - } - - @Override - public SetDataBuilder setData() { - return null; - } - - @Override - public GetChildrenBuilder getChildren() { - return null; - } - - @Override - public GetACLBuilder getACL() { - return null; - } - - @Override - public SetACLBuilder setACL() { - return null; - } - - @Override - public ReconfigBuilder reconfig() { - return null; - } - - @Override - public GetConfigBuilder getConfig() { - return null; - } - - @Override - public CuratorTransaction inTransaction() { - return null; - } - - @Override - public CuratorMultiTransaction transaction() { - return null; - } - - @Override - public TransactionOp transactionOp() { - return null; - } - - @Override - public void sync(String s, Object o) { - - } - - @Override - public void createContainers(String s) throws Exception { - - } - - @Override - public SyncBuilder sync() { - return null; - } - - @Override - public RemoveWatchesBuilder watches() { - return null; - } - - @Override - public Listenable getConnectionStateListenable() { - return null; - } - - @Override - public Listenable getCuratorListenable() { - return null; - } - - @Override - public Listenable getUnhandledErrorListenable() { - return null; - } - - @Override - public CuratorFramework nonNamespaceView() { - return null; - } - - @Override - public CuratorFramework usingNamespace(String s) { - return null; - } - - @Override - public String getNamespace() { - return null; - } - - @Override - public CuratorZookeeperClient getZookeeperClient() { - return null; - } - - @Override - public EnsurePath newNamespaceAwareEnsurePath(String s) { - return null; - } - - @Override - public void clearWatcherReferences(Watcher watcher) { - - } - - @Override - public boolean blockUntilConnected(int i, TimeUnit timeUnit) throws InterruptedException { - return false; - } - - @Override - public void blockUntilConnected() throws InterruptedException { - - } - - @Override - public WatcherRemoveCuratorFramework newWatcherRemoveCuratorFramework() { - return null; - } - - @Override - public ConnectionStateErrorPolicy getConnectionStateErrorPolicy() { - return null; - } - - @Override - public QuorumVerifier getCurrentConfig() { - return null; - } - - @Override - public SchemaSet getSchemaSet() { - return null; - } - - @Override - public boolean isZk34CompatibilityMode() { - return false; - } - - @Override - public CompletableFuture runSafe(Runnable runnable) { - return null; - } - }; - - } - - @Override - public ConnectionStateErrorPolicy getConnectionStateErrorPolicy() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public QuorumVerifier getCurrentConfig() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public SchemaSet getSchemaSet() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public boolean isZk34CompatibilityMode() { - return false; - } - - @Override - public CompletableFuture runSafe(Runnable runnable) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - @Override public SyncBuilder sync() { throw new UnsupportedOperationException("Not implemented in MockCurator"); } - - @Override - public RemoveWatchesBuilder watches() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - + } } diff --git a/zkfacade/src/main/java/org/apache/curator/framework/api/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/api/package-info.java index e3da4ab3efa..be9e84013c1 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/api/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/api/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 0)) +@ExportPackage(version = @Version(major = 2, minor = 13, micro = 0)) package org.apache.curator.framework.api; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/api/transaction/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/api/transaction/package-info.java index 94f8b12894e..be3ece0357b 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/api/transaction/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/api/transaction/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 0)) +@ExportPackage(version = @Version(major = 2, minor = 13, micro = 0)) package org.apache.curator.framework.api.transaction; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/listen/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/listen/package-info.java index 71ee8ccfff0..79c67cedf75 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/listen/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/listen/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 0)) +@ExportPackage(version = @Version(major = 2, minor = 13, micro = 0)) package org.apache.curator.framework.listen; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/package-info.java index 2999456bc9d..3e3b8433556 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 0)) +@ExportPackage(version = @Version(major = 2, minor = 13, micro = 0)) package org.apache.curator.framework; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/recipes/atomic/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/recipes/atomic/package-info.java index dd1dd7a1899..a607d5dcda5 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/recipes/atomic/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/recipes/atomic/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 0)) +@ExportPackage(version = @Version(major = 2, minor = 13, micro = 0)) package org.apache.curator.framework.recipes.atomic; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/recipes/barriers/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/recipes/barriers/package-info.java index 4e2aea367de..2db4beef75f 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/recipes/barriers/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/recipes/barriers/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 0)) +@ExportPackage(version = @Version(major = 2, minor = 13, micro = 0)) package org.apache.curator.framework.recipes.barriers; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/recipes/cache/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/recipes/cache/package-info.java index ad6913d6381..0465bbf2039 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/recipes/cache/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/recipes/cache/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 0)) +@ExportPackage(version = @Version(major = 2, minor = 13, micro = 0)) package org.apache.curator.framework.recipes.cache; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/recipes/locks/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/recipes/locks/package-info.java index 4307c09e30a..63b067bcffc 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/recipes/locks/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/recipes/locks/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 0)) +@ExportPackage(version = @Version(major = 2, minor = 13, micro = 0)) package org.apache.curator.framework.recipes.locks; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/framework/state/package-info.java b/zkfacade/src/main/java/org/apache/curator/framework/state/package-info.java index 4a10e20318d..eec4f00ddb4 100644 --- a/zkfacade/src/main/java/org/apache/curator/framework/state/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/framework/state/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 0)) +@ExportPackage(version = @Version(major = 2, minor = 13, micro = 0)) package org.apache.curator.framework.state; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/package-info.java b/zkfacade/src/main/java/org/apache/curator/package-info.java index 232a5fd46f3..120aa4558d2 100644 --- a/zkfacade/src/main/java/org/apache/curator/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 0)) +@ExportPackage(version = @Version(major = 2, minor = 13, micro = 0)) package org.apache.curator; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; diff --git a/zkfacade/src/main/java/org/apache/curator/retry/package-info.java b/zkfacade/src/main/java/org/apache/curator/retry/package-info.java index f45a0d927a5..98130481c4c 100644 --- a/zkfacade/src/main/java/org/apache/curator/retry/package-info.java +++ b/zkfacade/src/main/java/org/apache/curator/retry/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 0)) +@ExportPackage(version = @Version(major = 2, minor = 13, micro = 0)) package org.apache.curator.retry; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; -- cgit v1.2.3 From 044494d83632d11b96e9319c7184a3a973b77f67 Mon Sep 17 00:00:00 2001 From: gjoranv Date: Mon, 28 Sep 2020 08:49:25 +0200 Subject: Remove bundles.def - No live config models refer to it anymore. --- .../src/main/resources/configdefinitions/container.bundles.def | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 container-di/src/main/resources/configdefinitions/container.bundles.def diff --git a/container-di/src/main/resources/configdefinitions/container.bundles.def b/container-di/src/main/resources/configdefinitions/container.bundles.def deleted file mode 100644 index 79e24742398..00000000000 --- a/container-di/src/main/resources/configdefinitions/container.bundles.def +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=container - -# References to both application and platform bundles to install. -bundle[] file -- cgit v1.2.3 From 87af53c51538340c699aec33e011a92a5c95089f Mon Sep 17 00:00:00 2001 From: gjoranv Date: Mon, 28 Sep 2020 08:50:44 +0200 Subject: Remove the special rule that allowed pre-installing bundles .. for old versions of the config-model-fat-amended bundle. --- .../java/com/yahoo/container/core/config/ApplicationBundleLoader.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/container-core/src/main/java/com/yahoo/container/core/config/ApplicationBundleLoader.java b/container-core/src/main/java/com/yahoo/container/core/config/ApplicationBundleLoader.java index f87dd3f42d2..6ecb6c75f90 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/ApplicationBundleLoader.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/ApplicationBundleLoader.java @@ -103,9 +103,7 @@ public class ApplicationBundleLoader { // it means that the X-JDisc-Preinstall-Bundle header was used. // However, test osgi frameworks may return multiple bundles when installing a single bundle. if (bundles.size() > 1 && osgi.hasFelixFramework()) { - // TODO: remove if-statement below when the last model with preinstall has rolled out of hosted - if (! bundles.get(0).getSymbolicName().equals("config-model-fat-amended")) - throw new RuntimeException("Bundle '" + bundles.get(0).getSymbolicName() + "' tried to pre-install bundles from disk."); + throw new RuntimeException("Bundle '" + bundles.get(0).getSymbolicName() + "' tried to pre-install bundles from disk."); } reference2Bundle.put(reference, bundles.get(0)); } -- cgit v1.2.3 From e118b80bd8641e320677b4130c5d09526ae1a26f Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Mon, 28 Sep 2020 09:34:39 +0200 Subject: Use feature flag for skipping maintenance for some apps --- .../provision/maintenance/NodeRepositoryMaintenance.java | 3 ++- .../maintenance/PeriodicApplicationMaintainer.java | 15 ++++++++++----- .../maintenance/PeriodicApplicationMaintainerTest.java | 5 ++--- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 452c3b1c6ca..0dd7cfe47f0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -75,7 +75,8 @@ public class NodeRepositoryMaintenance extends AbstractComponent { nodeFailer = new NodeFailer(deployer, hostLivenessTracker, serviceMonitor, nodeRepository, defaults.failGrace, defaults.nodeFailerInterval, clock, orchestrator, throttlePolicyFromEnv().orElse(defaults.throttlePolicy), metric); - periodicApplicationMaintainer = new PeriodicApplicationMaintainer(deployer, metric, nodeRepository, defaults.redeployMaintainerInterval, defaults.periodicRedeployInterval); + periodicApplicationMaintainer = new PeriodicApplicationMaintainer(deployer, metric, nodeRepository, + defaults.redeployMaintainerInterval, defaults.periodicRedeployInterval, flagSource); operatorChangeApplicationMaintainer = new OperatorChangeApplicationMaintainer(deployer, metric, nodeRepository, defaults.operatorChangeRedeployInterval); reservationExpirer = new ReservationExpirer(nodeRepository, clock, defaults.reservationExpiry, metric); retiredExpirer = new RetiredExpirer(nodeRepository, orchestrator, deployer, metric, clock, defaults.retiredInterval, defaults.retiredExpiry); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java index e3128dfc8e9..289d5a6742a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java @@ -4,6 +4,10 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Deployer; import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -26,11 +30,13 @@ import java.util.stream.Collectors; public class PeriodicApplicationMaintainer extends ApplicationMaintainer { private final Duration minTimeBetweenRedeployments; + private final FlagSource flagSource; PeriodicApplicationMaintainer(Deployer deployer, Metric metric, NodeRepository nodeRepository, - Duration interval, Duration minTimeBetweenRedeployments) { + Duration interval, Duration minTimeBetweenRedeployments, FlagSource flagSource) { super(deployer, metric, nodeRepository, interval); this.minTimeBetweenRedeployments = minTimeBetweenRedeployments; + this.flagSource = flagSource; } @Override @@ -64,10 +70,9 @@ public class PeriodicApplicationMaintainer extends ApplicationMaintainer { } private boolean shouldMaintain(ApplicationId id) { - if (id.tenant().value().equals("stream") && id.application().value().equals("stream-ranking")) return false; - if (id.tenant().value().equals("stream") && id.application().value().equals("stream-ranking-canary")) return false; - if (id.tenant().value().equals("stream") && id.application().value().equals("stream-ranking-rhel7")) return false; - return true; + BooleanFlag skipMaintenanceDeployment = Flags.SKIP_MAINTENANCE_DEPLOYMENT.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, id.serializedForm()); + return ! skipMaintenanceDeployment.value(); } protected List nodesNeedingMaintenance() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java index ec830a7dc31..0ab6d4ddaa7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java @@ -316,14 +316,13 @@ public class PeriodicApplicationMaintainerTest { private List overriddenNodesNeedingMaintenance; - TestablePeriodicApplicationMaintainer setOverriddenNodesNeedingMaintenance(List overriddenNodesNeedingMaintenance) { + void setOverriddenNodesNeedingMaintenance(List overriddenNodesNeedingMaintenance) { this.overriddenNodesNeedingMaintenance = overriddenNodesNeedingMaintenance; - return this; } TestablePeriodicApplicationMaintainer(Deployer deployer, NodeRepository nodeRepository, Duration interval, Duration minTimeBetweenRedeployments) { - super(deployer, new TestMetric(), nodeRepository, interval, minTimeBetweenRedeployments); + super(deployer, new TestMetric(), nodeRepository, interval, minTimeBetweenRedeployments, new InMemoryFlagSource()); } @Override -- cgit v1.2.3 From 6e93e683e7cf85174f081e5fd2737be178766c00 Mon Sep 17 00:00:00 2001 From: Frode Lundgren Date: Mon, 28 Sep 2020 11:44:09 +0200 Subject: Revert "Revert "Support multiple levels of directories for flags"" --- .../api/systemflags/v1/SystemFlagsDataArchive.java | 18 +++++++++++++++++- .../api/systemflags/v1/SystemFlagsDataArchiveTest.java | 13 +++++++++++++ .../flags/group-1/my-test-flag/default.json | 8 ++++++++ .../flags/group-2/my-test-flag/default.json | 8 ++++++++ .../flags/group-1/my-test-flag/default.json | 8 ++++++++ .../flags/group-2/my-other-test-flag/default.json | 8 ++++++++ 6 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-1/my-test-flag/default.json create mode 100644 controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-2/my-test-flag/default.json create mode 100644 controller-api/src/test/resources/system-flags-multi-level/flags/group-1/my-test-flag/default.json create mode 100644 controller-api/src/test/resources/system-flags-multi-level/flags/group-2/my-other-test-flag/default.json diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java index e6310cc6432..a00992da815 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java @@ -45,6 +45,11 @@ import static com.yahoo.yolean.Exceptions.uncheck; * The flag files must reside in a 'flags/' root directory containing a directory for each flag name: * {@code ./flags//*.json} * + * Optionally, there can be an arbitrary number of directories "between" 'flags/' root directory and + * the flag name directory: + * {@code ./flags/onelevel//*.json} + * {@code ./flags/onelevel/anotherlevel//*.json} + * * @author bjorncs */ public class SystemFlagsDataArchive { @@ -155,7 +160,7 @@ public class SystemFlagsDataArchive { if (!filename.endsWith(".json")) { throw new IllegalArgumentException(String.format("Only JSON files are allowed in 'flags/' directory (found '%s')", filePath.toString())); } - FlagId directoryDeducedFlagId = new FlagId(filePath.getName(1).toString()); + FlagId directoryDeducedFlagId = new FlagId(filePath.getName(filePath.getNameCount()-2).toString()); FlagData flagData; if (rawData.isBlank()) { flagData = new FlagData(directoryDeducedFlagId); @@ -178,6 +183,13 @@ public class SystemFlagsDataArchive { "\nSee https://git.ouroath.com/vespa/hosted-feature-flags for more info on the JSON syntax"); } } + + if (builder.hasFile(filename, flagData)) { + throw new IllegalArgumentException( + String.format("Flag data file in '%s' contains redundant flag data for id '%s' already set in another directory!", + filePath, flagData.id())); + } + builder.addFile(filename, flagData); } @@ -236,6 +248,10 @@ public class SystemFlagsDataArchive { return this; } + public boolean hasFile(String filename, FlagData data) { + return files.containsKey(data.id()) && files.get(data.id()).containsKey(filename); + } + public SystemFlagsDataArchive build() { Map> copy = new TreeMap<>(); files.forEach((flagId, map) -> copy.put(flagId, new TreeMap<>(map))); diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java index 4cdbe5241bc..21e89cb5ea3 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java @@ -83,6 +83,19 @@ public class SystemFlagsDataArchiveTest { assertArchiveReturnsCorrectTestFlagDataForTarget(archive); } + @Test + public void supports_multi_level_flags_directory() { + var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags-multi-level/")); + assertFlagDataHasValue(archive, MY_TEST_FLAG, mainControllerTarget, "default"); + } + + @Test + public void duplicated_flagdata_is_detected() { + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Flag data file in 'flags/group-1/my-test-flag/default.json' contains redundant flag data for id 'my-test-flag' already set in another directory!"); + var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags-multi-level-with-duplicated-flagdata/")); + } + @Test public void empty_files_are_handled_as_no_flag_data_for_target() { var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags/")); diff --git a/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-1/my-test-flag/default.json b/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-1/my-test-flag/default.json new file mode 100644 index 00000000000..5924eb860c0 --- /dev/null +++ b/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-1/my-test-flag/default.json @@ -0,0 +1,8 @@ +{ + "id" : "my-test-flag", + "rules" : [ + { + "value" : "default" + } + ] +} \ No newline at end of file diff --git a/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-2/my-test-flag/default.json b/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-2/my-test-flag/default.json new file mode 100644 index 00000000000..5924eb860c0 --- /dev/null +++ b/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-2/my-test-flag/default.json @@ -0,0 +1,8 @@ +{ + "id" : "my-test-flag", + "rules" : [ + { + "value" : "default" + } + ] +} \ No newline at end of file diff --git a/controller-api/src/test/resources/system-flags-multi-level/flags/group-1/my-test-flag/default.json b/controller-api/src/test/resources/system-flags-multi-level/flags/group-1/my-test-flag/default.json new file mode 100644 index 00000000000..5924eb860c0 --- /dev/null +++ b/controller-api/src/test/resources/system-flags-multi-level/flags/group-1/my-test-flag/default.json @@ -0,0 +1,8 @@ +{ + "id" : "my-test-flag", + "rules" : [ + { + "value" : "default" + } + ] +} \ No newline at end of file diff --git a/controller-api/src/test/resources/system-flags-multi-level/flags/group-2/my-other-test-flag/default.json b/controller-api/src/test/resources/system-flags-multi-level/flags/group-2/my-other-test-flag/default.json new file mode 100644 index 00000000000..e30485b755c --- /dev/null +++ b/controller-api/src/test/resources/system-flags-multi-level/flags/group-2/my-other-test-flag/default.json @@ -0,0 +1,8 @@ +{ + "id" : "my-other-test-flag", + "rules" : [ + { + "value" : "default" + } + ] +} \ No newline at end of file -- cgit v1.2.3 From fe3cf9af1e3ab1c8d76a4718642ee14b264096de Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Mon, 28 Sep 2020 11:52:57 +0200 Subject: Synchronize when adding session Also confirm upload when adding session and status is NEW --- .../config/server/session/SessionRepository.java | 34 ++++++++-------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 8de196ce7c0..78bbe8b5438 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -47,7 +47,6 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -266,11 +265,6 @@ public class SessionRepository { return getSessionList(curator.getChildren(sessionsPath)); } - public void addRemoteSession(RemoteSession session) { - remoteSessionCache.put(session.getSessionId(), session); - metrics.incAddedSessions(); - } - public int deleteExpiredRemoteSessions(Clock clock, Duration expiryTime) { int deleted = 0; for (long sessionId : getRemoteSessions()) { @@ -344,12 +338,18 @@ public class SessionRepository { * * @param sessionId session id for the new session */ - public void sessionAdded(long sessionId) { + public synchronized void sessionAdded(long sessionId) { SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); if (sessionZKClient.readStatus().equals(Session.Status.DELETE)) return; log.log(Level.FINE, () -> "Adding remote session " + sessionId); - createRemoteSession(sessionId); + RemoteSession session = createRemoteSession(sessionId); + if (session.getStatus() == Session.Status.NEW) { + log.log(Level.FINE, () -> session.logPre() + "Confirming upload for session " + sessionId); + session.confirmUpload(); + } else { + log.log(Level.WARNING, () -> session.logPre() + "Session " + sessionId + " added, but with unexpected status " + session.getStatus()); + } if (distributeApplicationPackage()) createLocalSessionUsingDistributedApplicationPackage(sessionId); } @@ -376,7 +376,7 @@ public class SessionRepository { long sessionId = remoteSession.getSessionId(); try (Lock lock = lock(sessionId)) { // TODO: Change log level to FINE when debugging is finished - log.log(Level.INFO, "Deactivating and deleting remote session " + sessionId); + log.log(Level.INFO, () -> remoteSession.logPre() + "Deactivating and deleting remote session " + sessionId); remoteSession.deactivate(); remoteSession.delete(); remoteSessionCache.remove(sessionId); @@ -384,7 +384,7 @@ public class SessionRepository { LocalSession localSession = getLocalSession(sessionId); if (localSession != null) { // TODO: Change log level to FINE when debugging is finished - log.log(Level.INFO, "Deleting local session " + sessionId); + log.log(Level.INFO, () -> localSession.logPre() + "Deleting local session " + sessionId); deleteLocalSession(localSession); } } @@ -434,26 +434,16 @@ public class SessionRepository { log.log(Level.FINE, () -> "Got child event: " + event); switch (event.getType()) { case CHILD_ADDED: - sessionsChanged(); - synchronizeOnNew(getSessionListFromDirectoryCache(Collections.singletonList(event.getData()))); - break; case CHILD_REMOVED: case CONNECTION_RECONNECTED: sessionsChanged(); break; + default: + break; } }); } - private void synchronizeOnNew(List sessionList) { - for (long sessionId : sessionList) { - RemoteSession session = remoteSessionCache.get(sessionId); - if (session == null) continue; // session might have been deleted after getting session list - log.log(Level.FINE, () -> session.logPre() + "Confirming upload for session " + sessionId); - session.confirmUpload(); - } - } - /** * Creates a new deployment session from an application package. * -- cgit v1.2.3 From 6e284ef06a2e6dafbaeae8e486b6f68fa53d5d48 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Mon, 28 Sep 2020 12:10:38 +0200 Subject: Use deque as stack --- .../main/java/com/yahoo/vespa/curator/Lock.java | 4 ---- .../yahoo/vespa/curator/stats/ThreadLockInfo.java | 19 +++++++++--------- .../com/yahoo/vespa/curator/stats/LockTest.java | 23 ++++++++++++++++++++++ 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java index 4239d325ba5..b630995d6b4 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java @@ -56,10 +56,6 @@ public class Lock implements Mutex { @Override public void close() { - release(); - } - - private void release() { ThreadLockInfo.getCurrentThreadLockInfo().lockReleased(lockPath); try { mutex.release(); diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java index 5280a049ceb..92e6eb453b6 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java @@ -6,9 +6,8 @@ import com.yahoo.vespa.curator.Lock; import java.time.Duration; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentLinkedDeque; import java.util.function.Consumer; /** @@ -30,7 +29,7 @@ public class ThreadLockInfo { private final Thread thread; /** The locks are reentrant so there may be more than 1 lock for this thread. */ - private final ConcurrentLinkedQueue lockInfos = new ConcurrentLinkedQueue<>(); + private final ConcurrentLinkedDeque lockInfos = new ConcurrentLinkedDeque<>(); public static Map getLockCountersByPath() { return Map.copyOf(countersByLockPath); } @@ -83,7 +82,7 @@ public class ThreadLockInfo { LockCounters lockCounters = getLockCounters(lockPath); lockCounters.invokeAcquireCount.incrementAndGet(); lockCounters.inCriticalRegionCount.incrementAndGet(); - lockInfos.add(LockInfo.invokingAcquire(this, lockPath, timeout)); + lockInfos.addLast(LockInfo.invokingAcquire(this, lockPath, timeout)); } /** Mutable method (see class doc) */ @@ -107,7 +106,11 @@ public class ThreadLockInfo { /** Mutable method (see class doc) */ public void lockAcquired(String lockPath) { getLockCounters(lockPath).lockAcquiredCount.incrementAndGet(); - getLastLockInfo().ifPresent(LockInfo::lockAcquired); + LockInfo lastLockInfo = lockInfos.peekLast(); + if (lastLockInfo == null) { + throw new IllegalStateException("lockAcquired invoked without lockInfos"); + } + lastLockInfo.lockAcquired(); } /** Mutable method (see class doc) */ @@ -121,10 +124,6 @@ public class ThreadLockInfo { return countersByLockPath.computeIfAbsent(lockPath, __ -> new LockCounters()); } - private Optional getLastLockInfo() { - return lockInfos.isEmpty() ? Optional.empty() : Optional.of(lockInfos.peek()); - } - private void removeLastLockInfo(LockCounters lockCounters, Consumer completeLockInfo) { lockCounters.inCriticalRegionCount.decrementAndGet(); @@ -133,7 +132,7 @@ public class ThreadLockInfo { return; } - LockInfo lockInfo = lockInfos.poll(); + LockInfo lockInfo = lockInfos.pollLast(); completeLockInfo.accept(lockInfo); completedLockInfoSamples.maybeSample(lockInfo); } diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java index 4b9b6a4429b..984a04c6d4e 100644 --- a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java +++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java @@ -110,4 +110,27 @@ public class LockTest { expectedCounters.locksReleasedCount.set(2); assertEquals(Map.of(lockPath, expectedCounters), ThreadLockInfo.getLockCountersByPath()); } + + @Test + public void nestedLocks() throws Exception { + when(mutex.acquire(anyLong(), any())).thenReturn(true); + + String lockPath2 = "/lock/path/2"; + Lock lock2 = new Lock(lockPath2, mutex); + + lock.acquire(acquireTimeout); + lock2.acquire(acquireTimeout); + + List threadLockInfos = ThreadLockInfo.getThreadLockInfos(); + assertEquals(1, threadLockInfos.size()); + List lockInfos = threadLockInfos.get(0).getLockInfos(); + assertEquals(2, lockInfos.size()); + assertEquals(lockPath, lockInfos.get(0).getLockPath()); + assertEquals(LockInfo.LockState.ACQUIRED, lockInfos.get(0).getLockState()); + assertEquals(lockPath2, lockInfos.get(1).getLockPath()); + assertEquals(LockInfo.LockState.ACQUIRED, lockInfos.get(1).getLockState()); + + lock.close(); + lock.close(); + } } -- cgit v1.2.3 From e9dfc402f1e900a37d90cd62bdad704ebe395703 Mon Sep 17 00:00:00 2001 From: Håvard Pettersen Date: Mon, 28 Sep 2020 10:05:42 +0000 Subject: more generic tensor spec generation --- .../instruction_benchmark.cpp | 43 +++++++++++----------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp index 8bb227a7e85..9db7bbae4e8 100644 --- a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp +++ b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp @@ -185,44 +185,45 @@ struct D { return ValueType::Dimension(name, size); } } - std::pair operator()(size_t idx) const { + TensorSpec::Label operator()(size_t idx) const { if (mapped) { - return std::make_pair(name, TensorSpec::Label(fmt("label_%zu", idx))); + return TensorSpec::Label(fmt("label_%zu", idx)); } else { - return std::make_pair(name, TensorSpec::Label(idx)); + return TensorSpec::Label(idx); } } }; -TensorSpec make_vector(const D &d1, double seq) { - auto type = ValueType::tensor_type({d1}, ValueType::CellType::FLOAT); - TensorSpec spec(type.to_spec()); - for (size_t i = 0, idx1 = 0; i < d1.size; ++i, idx1 += d1.stride, seq += 1.0) { - spec.add({d1(idx1)}, seq); - } - return spec; +void add_cells(TensorSpec &spec, double &seq, TensorSpec::Address addr) { + spec.add(addr, seq); + seq += 1.0; } -TensorSpec make_cube(const D &d1, const D &d2, const D &d3, double seq) { - auto type = ValueType::tensor_type({d1, d2, d3}, ValueType::CellType::FLOAT); - TensorSpec spec(type.to_spec()); - for (size_t i = 0, idx1 = 0; i < d1.size; ++i, idx1 += d1.stride) { - for (size_t j = 0, idx2 = 0; j < d2.size; ++j, idx2 += d2.stride) { - for (size_t k = 0, idx3 = 0; k < d3.size; ++k, idx3 += d3.stride, seq += 1.0) { - spec.add({d1(idx1), d2(idx2), d3(idx3)}, seq); - } - } +template void add_cells(TensorSpec &spec, double &seq, TensorSpec::Address addr, const D &d, const Ds &...ds) { + for (size_t i = 0, idx = 0; i < d.size; ++i, idx += d.stride) { + addr.insert_or_assign(d.name, d(idx)); + add_cells(spec, seq, addr, ds...); } +} + +template TensorSpec make_spec(double seq, const Ds &...ds) { + TensorSpec spec(ValueType::tensor_type({ds...}, ValueType::CellType::FLOAT).to_spec()); + add_cells(spec, seq, TensorSpec::Address(), ds...); return spec; } +TensorSpec make_vector(const D &d1, double seq) { return make_spec(seq, d1); } +TensorSpec make_cube(const D &d1, const D &d2, const D &d3, double seq) { return make_spec(seq, d1, d2, d3); } + //----------------------------------------------------------------------------- TEST(MakeInputTest, print_some_test_input) { + auto number = make_spec(5.0); auto sparse = make_vector(D::map("x", 5, 3), 1.0); auto dense = make_vector(D::idx("x", 5), 10.0); auto mixed = make_cube(D::map("x", 3, 7), D::idx("y", 2), D::idx("z", 2), 100.0); fprintf(stderr, "--------------------------------------------------------\n"); + fprintf(stderr, "simple number: %s\n", number.to_string().c_str()); fprintf(stderr, "sparse vector: %s\n", sparse.to_string().c_str()); fprintf(stderr, "dense vector: %s\n", dense.to_string().c_str()); fprintf(stderr, "mixed cube: %s\n", mixed.to_string().c_str()); @@ -232,8 +233,8 @@ TEST(MakeInputTest, print_some_test_input) { //----------------------------------------------------------------------------- TEST(NumberJoin, plain_op2) { - auto lhs = TensorSpec("double").add({}, 2.0); - auto rhs = TensorSpec("double").add({}, 3.0); + auto lhs = make_spec(2.0); + auto rhs = make_spec(3.0); benchmark_join("simple numbers multiply", lhs, rhs, operation::Mul::f); } -- cgit v1.2.3 From 905cbe0bee949379fd8573cbd298e96acfa25906 Mon Sep 17 00:00:00 2001 From: Håvard Pettersen Date: Mon, 28 Sep 2020 10:30:00 +0000 Subject: implement reference join without using SimpleTensorEngine --- .../instruction/generic_join/generic_join_test.cpp | 35 +++++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/eval/src/tests/instruction/generic_join/generic_join_test.cpp b/eval/src/tests/instruction/generic_join/generic_join_test.cpp index 53df23d77be..4821bf092da 100644 --- a/eval/src/tests/instruction/generic_join/generic_join_test.cpp +++ b/eval/src/tests/instruction/generic_join/generic_join_test.cpp @@ -41,13 +41,32 @@ std::vector join_layouts = { float_cells({x({"a","b","c"}),y(5)}), float_cells({y(5),z({"i","j","k","l"})}) }; -TensorSpec simple_tensor_join(const TensorSpec &a, const TensorSpec &b, join_fun_t function) { - Stash stash; - const auto &engine = SimpleTensorEngine::ref(); - auto lhs = engine.from_spec(a); - auto rhs = engine.from_spec(b); - const auto &result = engine.join(*lhs, *rhs, function, stash); - return engine.to_spec(result); +bool join_address(const TensorSpec::Address &a, const TensorSpec::Address &b, TensorSpec::Address &addr) { + for (const auto &dim_a: a) { + auto pos_b = b.find(dim_a.first); + if ((pos_b != b.end()) && !(pos_b->second == dim_a.second)) { + return false; + } + addr.insert_or_assign(dim_a.first, dim_a.second); + } + return true; +} + +TensorSpec reference_join(const TensorSpec &a, const TensorSpec &b, join_fun_t function) { + ValueType res_type = ValueType::join(ValueType::from_spec(a.type()), ValueType::from_spec(b.type())); + EXPECT_FALSE(res_type.is_error()); + TensorSpec result(res_type.to_spec()); + for (const auto &cell_a: a.cells()) { + for (const auto &cell_b: b.cells()) { + TensorSpec::Address addr; + if (join_address(cell_a.first, cell_b.first, addr) && + join_address(cell_b.first, cell_a.first, addr)) + { + result.add(addr, function(cell_a.second, cell_b.second)); + } + } + } + return result; } TensorSpec perform_generic_join(const TensorSpec &a, const TensorSpec &b, join_fun_t function) { @@ -109,7 +128,7 @@ TEST(GenericJoinTest, generic_join_works_for_simple_values) { TensorSpec rhs = spec(join_layouts[i + 1], Div16(N())); for (auto fun: {operation::Add::f, operation::Sub::f, operation::Mul::f, operation::Div::f}) { SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); - auto expect = simple_tensor_join(lhs, rhs, fun); + auto expect = reference_join(lhs, rhs, fun); auto actual = perform_generic_join(lhs, rhs, fun); EXPECT_EQ(actual, expect); } -- cgit v1.2.3 From 87e537733efd9c435a6c2382f8b42262a687cb23 Mon Sep 17 00:00:00 2001 From: Håvard Pettersen Date: Mon, 28 Sep 2020 10:43:18 +0000 Subject: implement reference rename without using SimpleTensorEngine --- .../generic_rename/generic_rename_test.cpp | 30 +++++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp b/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp index 1139d35d847..f61899e4dda 100644 --- a/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp +++ b/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp @@ -87,12 +87,28 @@ TEST(GenericRenameTest, sparse_rename_plan_can_be_created) { EXPECT_EQ(plan.output_dimensions, expect); } -TensorSpec simple_tensor_rename(const TensorSpec &a, const FromTo &ft) { - Stash stash; - const auto &engine = SimpleTensorEngine::ref(); - auto lhs = engine.from_spec(a); - const auto &result = engine.rename(*lhs, ft.from, ft.to, stash); - return engine.to_spec(result); +vespalib::string rename_dimension(const vespalib::string &name, const FromTo &ft) { + assert(ft.from.size() == ft.to.size()); + for (size_t i = 0; i < ft.from.size(); ++i) { + if (name == ft.from[i]) { + return ft.to[i]; + } + } + return name; +} + +TensorSpec reference_rename(const TensorSpec &a, const FromTo &ft) { + ValueType res_type = ValueType::from_spec(a.type()).rename(ft.from, ft.to); + EXPECT_FALSE(res_type.is_error()); + TensorSpec result(res_type.to_spec()); + for (const auto &cell: a.cells()) { + TensorSpec::Address addr; + for (const auto &dim: cell.first) { + addr.insert_or_assign(rename_dimension(dim.first, ft), dim.second); + } + result.add(addr, cell.second); + } + return result; } TensorSpec perform_generic_rename(const TensorSpec &a, const ValueType &res_type, @@ -115,7 +131,7 @@ void test_generic_rename(const ValueBuilderFactory &factory) { if (renamed_type.is_error()) continue; // printf("type %s -> %s\n", lhs_type.to_spec().c_str(), renamed_type.to_spec().c_str()); SCOPED_TRACE(fmt("\n===\nLHS: %s\n===\n", lhs.to_string().c_str())); - auto expect = simple_tensor_rename(lhs, from_to); + auto expect = reference_rename(lhs, from_to); auto actual = perform_generic_rename(lhs, renamed_type, from_to, factory); EXPECT_EQ(actual, expect); } -- cgit v1.2.3 From 646120d32e342f8ce25ca72fafd216978ca7412d Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Mon, 28 Sep 2020 12:56:18 +0200 Subject: Use context to keep track of extra storagelink instance used to inject benchmark feed. --- .../storage_api_chain_bm_feed_handler.cpp | 37 ++++++++++++++++------ .../storage_api_chain_bm_feed_handler.h | 9 ++++-- .../src/apps/vespa-feed-bm/vespa_feed_bm.cpp | 18 +++++++---- 3 files changed, 46 insertions(+), 18 deletions(-) diff --git a/searchcore/src/apps/vespa-feed-bm/storage_api_chain_bm_feed_handler.cpp b/searchcore/src/apps/vespa-feed-bm/storage_api_chain_bm_feed_handler.cpp index 6ff8f0a2159..6f1acd10fe4 100644 --- a/searchcore/src/apps/vespa-feed-bm/storage_api_chain_bm_feed_handler.cpp +++ b/searchcore/src/apps/vespa-feed-bm/storage_api_chain_bm_feed_handler.cpp @@ -89,19 +89,28 @@ BmStorageLink::onUp(const std::shared_ptr& msg) return release(msg->getMsgId()); } +struct StorageApiChainBmFeedHandler::Context { + BmStorageLink* bm_link; + Context() + : bm_link(nullptr) + { + } + ~Context() = default; +}; + class MyStorageChainBuilder : public storage::StorageChainBuilder { using Parent = storage::StorageChainBuilder; + std::shared_ptr _context; public: - MyStorageChainBuilder(); + MyStorageChainBuilder(std::shared_ptr context); ~MyStorageChainBuilder() override; void add(std::unique_ptr link) override; }; -BmStorageLink *bm_link = nullptr; - -MyStorageChainBuilder::MyStorageChainBuilder() - : storage::StorageChainBuilder() +MyStorageChainBuilder::MyStorageChainBuilder(std::shared_ptr context) + : storage::StorageChainBuilder(), + _context(std::move(context)) { } @@ -114,13 +123,14 @@ MyStorageChainBuilder::add(std::unique_ptr link) Parent::add(std::move(link)); if (name == "Communication manager") { auto my_link = std::make_unique(); - bm_link = my_link.get(); + _context->bm_link = my_link.get(); Parent::add(std::move(my_link)); } } -StorageApiChainBmFeedHandler::StorageApiChainBmFeedHandler() - : IBmFeedHandler() +StorageApiChainBmFeedHandler::StorageApiChainBmFeedHandler(std::shared_ptr context) + : IBmFeedHandler(), + _context(std::move(context)) { auto cmd = make_set_cluster_state_cmd(); PendingTracker tracker(1); @@ -134,6 +144,7 @@ void StorageApiChainBmFeedHandler::send_msg(std::shared_ptr cmd, PendingTracker& pending_tracker) { cmd->setSourceIndex(0); + auto bm_link = _context->bm_link; bm_link->retain(cmd->getMsgId(), pending_tracker); bm_link->sendDown(std::move(cmd)); } @@ -159,10 +170,16 @@ StorageApiChainBmFeedHandler::remove(const document::Bucket& bucket, const Docum send_msg(std::move(cmd), tracker); } +std::shared_ptr +StorageApiChainBmFeedHandler::get_context() +{ + return std::make_shared(); +} + std::unique_ptr -StorageApiChainBmFeedHandler::get_storage_chain_builder() +StorageApiChainBmFeedHandler::get_storage_chain_builder(std::shared_ptr context) { - return std::make_unique(); + return std::make_unique(std::move(context)); } } diff --git a/searchcore/src/apps/vespa-feed-bm/storage_api_chain_bm_feed_handler.h b/searchcore/src/apps/vespa-feed-bm/storage_api_chain_bm_feed_handler.h index 79c04c98de9..521deddd19e 100644 --- a/searchcore/src/apps/vespa-feed-bm/storage_api_chain_bm_feed_handler.h +++ b/searchcore/src/apps/vespa-feed-bm/storage_api_chain_bm_feed_handler.h @@ -15,15 +15,20 @@ namespace feedbm { */ class StorageApiChainBmFeedHandler : public IBmFeedHandler { +public: + struct Context; +private: + std::shared_ptr _context; void send_msg(std::shared_ptr cmd, PendingTracker& tracker); public: - StorageApiChainBmFeedHandler(); + StorageApiChainBmFeedHandler(std::shared_ptr context); ~StorageApiChainBmFeedHandler(); void put(const document::Bucket& bucket, std::unique_ptr document, uint64_t timestamp, PendingTracker& tracker) override; void update(const document::Bucket& bucket, std::unique_ptr document_update, uint64_t timestamp, PendingTracker& tracker) override; void remove(const document::Bucket& bucket, const document::DocumentId& document_id, uint64_t timestamp, PendingTracker& tracker) override; - static std::unique_ptr get_storage_chain_builder(); + static std::shared_ptr get_context(); + static std::unique_ptr get_storage_chain_builder(std::shared_ptr context); }; } diff --git a/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp b/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp index 2c143c7ab23..73d741c3fee 100644 --- a/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp +++ b/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp @@ -282,7 +282,7 @@ class MyServiceLayerProcess : public storage::ServiceLayerProcess { public: MyServiceLayerProcess(const config::ConfigUri & configUri, PersistenceProvider &provider, - bool use_storage_chain); + std::unique_ptr chain_builder); ~MyServiceLayerProcess() override { shutdown(); } void shutdown() override; @@ -292,12 +292,12 @@ public: MyServiceLayerProcess::MyServiceLayerProcess(const config::ConfigUri & configUri, PersistenceProvider &provider, - bool use_storage_chain) + std::unique_ptr chain_builder) : ServiceLayerProcess(configUri), _provider(provider) { - if (use_storage_chain) { - set_storage_chain_builder(StorageApiChainBmFeedHandler::get_storage_chain_builder()); + if (chain_builder) { + set_storage_chain_builder(std::move(chain_builder)); } } @@ -632,9 +632,15 @@ PersistenceProviderFixture::start_service_layer(bool use_storage_chain) _slobrok = std::make_unique(_slobrok_port); LOG(info, "start service layer"); config::ConfigUri config_uri("bm-servicelayer", _config_context); + std::unique_ptr chain_builder; + std::shared_ptr context; + if (use_storage_chain) { + context = StorageApiChainBmFeedHandler::get_context(); + chain_builder = StorageApiChainBmFeedHandler::get_storage_chain_builder(context); + } _service_layer = std::make_unique(config_uri, *_persistence_engine, - use_storage_chain); + std::move(chain_builder)); _service_layer->setupConfig(100ms); _service_layer->createNode(); _service_layer->getNode().waitUntilInitialized(); @@ -643,7 +649,7 @@ PersistenceProviderFixture::start_service_layer(bool use_storage_chain) _rpc_client_shared_rpc_resources = std::make_unique(client_config_uri, _rpc_client_port, 100); _rpc_client_shared_rpc_resources->start_server_and_register_slobrok("bm-rpc-client"); if (use_storage_chain) { - _feed_handler = std::make_unique(); + _feed_handler = std::make_unique(std::move(context)); } else { _feed_handler = std::make_unique(*_rpc_client_shared_rpc_resources, _repo); } -- cgit v1.2.3 From 4fd2db62039bf5fed95dcb71bd3ff59267812c7c Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Mon, 28 Sep 2020 13:49:49 +0200 Subject: LockInfo -> LockAttempt, ThreadLockInfo -> ThreadLockStats, and more --- .../hosted/provision/restapi/LocksResponse.java | 46 +++---- .../main/java/com/yahoo/vespa/curator/Lock.java | 14 +-- .../com/yahoo/vespa/curator/stats/LockAttempt.java | 99 +++++++++++++++ .../vespa/curator/stats/LockAttemptSamples.java | 106 ++++++++++++++++ .../com/yahoo/vespa/curator/stats/LockInfo.java | 99 --------------- .../yahoo/vespa/curator/stats/LockInfoSamples.java | 106 ---------------- .../yahoo/vespa/curator/stats/ThreadLockInfo.java | 139 --------------------- .../yahoo/vespa/curator/stats/ThreadLockStats.java | 139 +++++++++++++++++++++ .../curator/stats/LockAttemptSamplesTest.java | 62 +++++++++ .../vespa/curator/stats/LockInfoSamplesTest.java | 58 --------- .../com/yahoo/vespa/curator/stats/LockTest.java | 56 +++++---- 11 files changed, 466 insertions(+), 458 deletions(-) create mode 100644 zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttempt.java create mode 100644 zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttemptSamples.java delete mode 100644 zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java delete mode 100644 zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfoSamples.java delete mode 100644 zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java create mode 100644 zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java create mode 100644 zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockAttemptSamplesTest.java delete mode 100644 zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockInfoSamplesTest.java diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java index 7a110f4c415..59050106cba 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java @@ -5,9 +5,9 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; +import com.yahoo.vespa.curator.stats.LockAttempt; import com.yahoo.vespa.curator.stats.LockCounters; -import com.yahoo.vespa.curator.stats.LockInfo; -import com.yahoo.vespa.curator.stats.ThreadLockInfo; +import com.yahoo.vespa.curator.stats.ThreadLockStats; import java.io.IOException; import java.io.OutputStream; @@ -25,15 +25,15 @@ public class LocksResponse extends HttpResponse { private final Slime slime = new Slime(); public LocksResponse() { - this(new TreeMap<>(ThreadLockInfo.getLockCountersByPath()), - ThreadLockInfo.getThreadLockInfos(), - ThreadLockInfo.getLockInfoSamples()); + this(new TreeMap<>(ThreadLockStats.getLockCountersByPath()), + ThreadLockStats.getThreadLockInfos(), + ThreadLockStats.getLockInfoSamples()); } /** For testing */ LocksResponse(TreeMap lockCountersByPath, - List threadLockInfos, - List historicSamples) { + List threadLockStats, + List historicSamples) { super(200); Cursor root = slime.setObject(); @@ -53,14 +53,14 @@ public class LocksResponse extends HttpResponse { }); Cursor threadsCursor = root.setArray("threads"); - for (var threadLockInfo : threadLockInfos) { - List lockInfos = threadLockInfo.getLockInfos(); - if (!lockInfos.isEmpty()) { + for (var threadLockInfo : threadLockStats) { + List lockAttempts = threadLockInfo.getLockAttempts(); + if (!lockAttempts.isEmpty()) { Cursor threadLockInfoCursor = threadsCursor.addObject(); threadLockInfoCursor.setString("thread-name", threadLockInfo.getThreadName()); Cursor lockInfosCursor = threadLockInfoCursor.setArray("active-locks"); - for (var lockInfo : lockInfos) { + for (var lockInfo : lockAttempts) { setLockInfo(lockInfosCursor.addObject(), lockInfo, false); } @@ -82,21 +82,21 @@ public class LocksResponse extends HttpResponse { return "application/json"; } - private void setLockInfo(Cursor lockInfoCursor, LockInfo lockInfo, boolean includeThreadInfo) { + private void setLockInfo(Cursor lockInfoCursor, LockAttempt lockAttempt, boolean includeThreadInfo) { if (includeThreadInfo) { - lockInfoCursor.setString("thread-name", lockInfo.getThreadName()); + lockInfoCursor.setString("thread-name", lockAttempt.getThreadName()); } - lockInfoCursor.setString("lock-path", lockInfo.getLockPath()); - lockInfoCursor.setString("invoke-acquire-time", toString(lockInfo.getTimeAcquiredWasInvoked())); - lockInfoCursor.setString("acquire-timeout", lockInfo.getAcquireTimeout().toString()); - lockInfo.getTimeLockWasAcquired().ifPresent(instant -> lockInfoCursor.setString("lock-acquired-time", toString(instant))); - lockInfoCursor.setString("lock-state", lockInfo.getLockState().name()); - lockInfo.getTimeTerminalStateWasReached().ifPresent(instant -> lockInfoCursor.setString("terminal-state-time", toString(instant))); - lockInfoCursor.setString("acquire-duration", lockInfo.getDurationOfAcquire().toString()); - lockInfoCursor.setString("locked-duration", lockInfo.getDurationWithLock().toString()); - lockInfoCursor.setString("total-duration", lockInfo.getDuration().toString()); + lockInfoCursor.setString("lock-path", lockAttempt.getLockPath()); + lockInfoCursor.setString("invoke-acquire-time", toString(lockAttempt.getTimeAcquiredWasInvoked())); + lockInfoCursor.setString("acquire-timeout", lockAttempt.getAcquireTimeout().toString()); + lockAttempt.getTimeLockWasAcquired().ifPresent(instant -> lockInfoCursor.setString("lock-acquired-time", toString(instant))); + lockInfoCursor.setString("lock-state", lockAttempt.getLockState().name()); + lockAttempt.getTimeTerminalStateWasReached().ifPresent(instant -> lockInfoCursor.setString("terminal-state-time", toString(instant))); + lockInfoCursor.setString("acquire-duration", lockAttempt.getDurationOfAcquire().toString()); + lockInfoCursor.setString("locked-duration", lockAttempt.getDurationWithLock().toString()); + lockInfoCursor.setString("total-duration", lockAttempt.getDuration().toString()); if (includeThreadInfo) { - lockInfo.getStackTrace().ifPresent(stackTrace -> lockInfoCursor.setString("stack-trace", stackTrace)); + lockAttempt.getStackTrace().ifPresent(stackTrace -> lockInfoCursor.setString("stack-trace", stackTrace)); } } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java index b630995d6b4..6e3b79ec5ce 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.curator; import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.path.Path; import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.curator.stats.ThreadLockInfo; +import com.yahoo.vespa.curator.stats.ThreadLockStats; import org.apache.curator.framework.recipes.locks.InterProcessLock; import java.time.Duration; @@ -35,28 +35,28 @@ public class Lock implements Mutex { /** Take the lock with the given timeout. This may be called multiple times from the same thread - each matched by a close */ public void acquire(Duration timeout) throws UncheckedTimeoutException { - ThreadLockInfo threadLockInfo = ThreadLockInfo.getCurrentThreadLockInfo(); - threadLockInfo.invokingAcquire(lockPath, timeout); + ThreadLockStats threadLockStats = ThreadLockStats.getCurrentThreadLockInfo(); + threadLockStats.invokingAcquire(lockPath, timeout); final boolean acquired; try { acquired = mutex.acquire(timeout.toMillis(), TimeUnit.MILLISECONDS); } catch (Exception e) { - threadLockInfo.acquireFailed(lockPath); + threadLockStats.acquireFailed(lockPath); throw new RuntimeException("Exception acquiring lock '" + lockPath + "'", e); } if (!acquired) { - threadLockInfo.acquireTimedOut(lockPath); + threadLockStats.acquireTimedOut(lockPath); throw new UncheckedTimeoutException("Timed out after waiting " + timeout + " to acquire lock '" + lockPath + "'"); } - threadLockInfo.lockAcquired(lockPath); + threadLockStats.lockAcquired(lockPath); } @Override public void close() { - ThreadLockInfo.getCurrentThreadLockInfo().lockReleased(lockPath); + ThreadLockStats.getCurrentThreadLockInfo().lockReleased(lockPath); try { mutex.release(); } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttempt.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttempt.java new file mode 100644 index 00000000000..c092fb6c289 --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttempt.java @@ -0,0 +1,99 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator.stats; + +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; + +/** + * Information about a lock. + * + *

Should be mutated by a single thread, except {@link #fillStackTrace()} which can be + * invoked by any threads. Other threads may see an inconsistent state of this instance.

+ * + * @author hakon + */ +public class LockAttempt { + + private final ThreadLockStats threadLockStats; + private final String lockPath; + private final Instant callAcquireInstant; + private final Duration timeout; + + private volatile Optional lockAcquiredInstant = Optional.empty(); + private volatile Optional terminalStateInstant = Optional.empty(); + private volatile Optional stackTrace = Optional.empty(); + + public static LockAttempt invokingAcquire(ThreadLockStats threadLockStats, String lockPath, Duration timeout) { + return new LockAttempt(threadLockStats, lockPath, timeout, Instant.now()); + } + + public enum LockState { + ACQUIRING(false), ACQUIRE_FAILED(true), TIMED_OUT(true), ACQUIRED(false), RELEASED(true); + + private final boolean terminal; + + LockState(boolean terminal) { this.terminal = terminal; } + + public boolean isTerminal() { return terminal; } + } + + private volatile LockState lockState = LockState.ACQUIRING; + + private LockAttempt(ThreadLockStats threadLockStats, String lockPath, Duration timeout, Instant callAcquireInstant) { + this.threadLockStats = threadLockStats; + this.lockPath = lockPath; + this.callAcquireInstant = callAcquireInstant; + this.timeout = timeout; + } + + public String getThreadName() { return threadLockStats.getThreadName(); } + public String getLockPath() { return lockPath; } + public Instant getTimeAcquiredWasInvoked() { return callAcquireInstant; } + public Duration getAcquireTimeout() { return timeout; } + public LockState getLockState() { return lockState; } + public Optional getTimeLockWasAcquired() { return lockAcquiredInstant; } + public Optional getTimeTerminalStateWasReached() { return terminalStateInstant; } + public Optional getStackTrace() { return stackTrace; } + + public Duration getDurationOfAcquire() { + return Duration.between(callAcquireInstant, lockAcquiredInstant.orElseGet(Instant::now)); + } + + public Duration getDurationWithLock() { + return lockAcquiredInstant + .map(start -> Duration.between(start, terminalStateInstant.orElseGet(Instant::now))) + .orElse(Duration.ZERO); + } + + public Duration getDuration() { return Duration.between(callAcquireInstant, terminalStateInstant.orElseGet(Instant::now)); } + + /** Get time from just before trying to acquire lock to the time the terminal state was reached, or ZERO. */ + public Duration getStableTotalDuration() { + return terminalStateInstant.map(instant -> Duration.between(callAcquireInstant, instant)).orElse(Duration.ZERO); + } + + /** Fill in the stack trace starting at the caller's stack frame. */ + public void fillStackTrace() { + // This method is public. If invoked concurrently, the this.stackTrace may be updated twice, + // which is fine. + + this.stackTrace = Optional.of(threadLockStats.getStackTrace()); + } + + void acquireFailed() { setTerminalState(LockState.ACQUIRE_FAILED); } + void timedOut() { setTerminalState(LockState.TIMED_OUT); } + void released() { setTerminalState(LockState.RELEASED); } + + void lockAcquired() { + lockState = LockState.ACQUIRED; + lockAcquiredInstant = Optional.of(Instant.now()); + } + + void setTerminalState(LockState terminalState) { setTerminalState(terminalState, Instant.now()); } + + void setTerminalState(LockState terminalState, Instant instant) { + lockState = terminalState; + terminalStateInstant = Optional.of(instant); + } +} diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttemptSamples.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttemptSamples.java new file mode 100644 index 00000000000..54cb82ebc1e --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttemptSamples.java @@ -0,0 +1,106 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator.stats; + +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.PriorityQueue; + +/** + * Collection containing "interesting" {@code LockAttempt}s. + * + * @author hakon + */ +// @ThreadSafe +public class LockAttemptSamples { + private final int maxSamples; + + /** Ensure atomic operations on this collection. */ + private final Object monitor = new Object(); + + /** Keep at most one sample for each lock path. */ + private final Map byLockPath; + + /** + * Priority queue containing all samples. The head of this queue (peek()/poll()) + * returns the LockAttempt with the smallest duration. + */ + private final PriorityQueue priorityQueue = + new PriorityQueue<>(Comparator.comparing(LockAttempt::getStableTotalDuration)); + + LockAttemptSamples() { this(10); } + + LockAttemptSamples(int maxSamples) { + this.maxSamples = maxSamples; + this.byLockPath = new HashMap<>(maxSamples); + } + + int size() { return byLockPath.size(); } + + boolean maybeSample(LockAttempt lockAttempt) { + final boolean added; + synchronized (monitor) { + if (shouldAdd(lockAttempt)) { + byLockPath.put(lockAttempt.getLockPath(), lockAttempt); + priorityQueue.add(lockAttempt); + added = true; + } else { + added = false; + } + } + + if (added) { + // Unnecessary to invoke under synchronized, although it means that some samples + // may be without stack trace (just retry if that happens). + lockAttempt.fillStackTrace(); + } + + return added; + } + + private boolean shouldAdd(LockAttempt lockAttempt) { + LockAttempt existingLockAttempt = byLockPath.get(lockAttempt.getLockPath()); + if (existingLockAttempt != null) { + if (hasLongerDurationThan(lockAttempt, existingLockAttempt)) { + byLockPath.remove(existingLockAttempt.getLockPath()); + priorityQueue.remove(existingLockAttempt); + return true; + } + + return false; + } + + if (size() < maxSamples) { + return true; + } + + // peek() and poll() retrieves the smallest element. + existingLockAttempt = priorityQueue.peek(); // cannot be null + if (hasLongerDurationThan(lockAttempt, existingLockAttempt)) { + priorityQueue.poll(); + byLockPath.remove(existingLockAttempt.getLockPath()); + return true; + } + + return false; + } + + List asList() { + synchronized (monitor) { + return List.copyOf(byLockPath.values()); + } + } + + void clear() { + synchronized (monitor) { + byLockPath.clear(); + priorityQueue.clear(); + } + } + + private static boolean hasLongerDurationThan(LockAttempt lockAttempt, LockAttempt otherLockAttempt) { + // Use stable total duration to avoid messing up priority queue. + return lockAttempt.getStableTotalDuration().compareTo(otherLockAttempt.getStableTotalDuration()) > 0; + } +} diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java deleted file mode 100644 index f40c6f0498f..00000000000 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfo.java +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.curator.stats; - -import java.time.Duration; -import java.time.Instant; -import java.util.Optional; - -/** - * Information about a lock. - * - *

Should be mutated by a single thread, except {@link #fillStackTrace()} which can be - * invoked by any threads. Other threads may see an inconsistent state of this instance.

- * - * @author hakon - */ -public class LockInfo { - - private final ThreadLockInfo threadLockInfo; - private final String lockPath; - private final Instant callAcquireInstant; - private final Duration timeout; - - private volatile Optional lockAcquiredInstant = Optional.empty(); - private volatile Optional terminalStateInstant = Optional.empty(); - private volatile Optional stackTrace = Optional.empty(); - - public static LockInfo invokingAcquire(ThreadLockInfo threadLockInfo, String lockPath, Duration timeout) { - return new LockInfo(threadLockInfo, lockPath, timeout, Instant.now()); - } - - public enum LockState { - ACQUIRING(false), ACQUIRE_FAILED(true), TIMED_OUT(true), ACQUIRED(false), RELEASED(true); - - private final boolean terminal; - - LockState(boolean terminal) { this.terminal = terminal; } - - public boolean isTerminal() { return terminal; } - } - - private volatile LockState lockState = LockState.ACQUIRING; - - private LockInfo(ThreadLockInfo threadLockInfo, String lockPath, Duration timeout, Instant callAcquireInstant) { - this.threadLockInfo = threadLockInfo; - this.lockPath = lockPath; - this.callAcquireInstant = callAcquireInstant; - this.timeout = timeout; - } - - public String getThreadName() { return threadLockInfo.getThreadName(); } - public String getLockPath() { return lockPath; } - public Instant getTimeAcquiredWasInvoked() { return callAcquireInstant; } - public Duration getAcquireTimeout() { return timeout; } - public LockState getLockState() { return lockState; } - public Optional getTimeLockWasAcquired() { return lockAcquiredInstant; } - public Optional getTimeTerminalStateWasReached() { return terminalStateInstant; } - public Optional getStackTrace() { return stackTrace; } - - public Duration getDurationOfAcquire() { - return Duration.between(callAcquireInstant, lockAcquiredInstant.orElseGet(Instant::now)); - } - - public Duration getDurationWithLock() { - return lockAcquiredInstant - .map(start -> Duration.between(start, terminalStateInstant.orElseGet(Instant::now))) - .orElse(Duration.ZERO); - } - - public Duration getDuration() { return Duration.between(callAcquireInstant, terminalStateInstant.orElseGet(Instant::now)); } - - /** Get time from just before trying to acquire lock to the time the terminal state was reached, or ZERO. */ - public Duration getStableTotalDuration() { - return terminalStateInstant.map(instant -> Duration.between(callAcquireInstant, instant)).orElse(Duration.ZERO); - } - - /** Fill in the stack trace starting at the caller's stack frame. */ - public void fillStackTrace() { - // This method is public. If invoked concurrently, the this.stackTrace may be updated twice, - // which is fine. - - this.stackTrace = Optional.of(threadLockInfo.getStackTrace()); - } - - void acquireFailed() { setTerminalState(LockState.ACQUIRE_FAILED); } - void timedOut() { setTerminalState(LockState.TIMED_OUT); } - void released() { setTerminalState(LockState.RELEASED); } - - void lockAcquired() { - lockState = LockState.ACQUIRED; - lockAcquiredInstant = Optional.of(Instant.now()); - } - - void setTerminalState(LockState terminalState) { setTerminalState(terminalState, Instant.now()); } - - void setTerminalState(LockState terminalState, Instant instant) { - lockState = terminalState; - terminalStateInstant = Optional.of(instant); - } -} diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfoSamples.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfoSamples.java deleted file mode 100644 index ae84c5c984a..00000000000 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockInfoSamples.java +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.curator.stats; - -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.PriorityQueue; - -/** - * Collection containing "interesting" {@code LockInfo}s. - * - * @author hakon - */ -// @ThreadSafe -public class LockInfoSamples { - private final int maxSamples; - - /** Ensure atomic operations on this collection. */ - private final Object monitor = new Object(); - - /** Keep at most one sample for each lock path. */ - private final Map byLockPath; - - /** - * Priority queue containing all samples. The head of this queue (peek()/poll()) - * returns the LockInfo with the smallest duration. - */ - private final PriorityQueue priorityQueue = - new PriorityQueue<>(Comparator.comparing(LockInfo::getStableTotalDuration)); - - LockInfoSamples() { this(10); } - - LockInfoSamples(int maxSamples) { - this.maxSamples = maxSamples; - this.byLockPath = new HashMap<>(maxSamples); - } - - int size() { return byLockPath.size(); } - - boolean maybeSample(LockInfo lockInfo) { - final boolean added; - synchronized (monitor) { - if (shouldAdd(lockInfo)) { - byLockPath.put(lockInfo.getLockPath(), lockInfo); - priorityQueue.add(lockInfo); - added = true; - } else { - added = false; - } - } - - if (added) { - // Unnecessary to invoke under synchronized, although it means that some samples - // may be without stack trace (just retry if that happens). - lockInfo.fillStackTrace(); - } - - return added; - } - - private boolean shouldAdd(LockInfo lockInfo) { - LockInfo existingLockInfo = byLockPath.get(lockInfo.getLockPath()); - if (existingLockInfo != null) { - if (hasLongerDurationThan(lockInfo, existingLockInfo)) { - byLockPath.remove(existingLockInfo.getLockPath()); - priorityQueue.remove(existingLockInfo); - return true; - } - - return false; - } - - if (size() < maxSamples) { - return true; - } - - // peek() and poll() retrieves the smallest element. - existingLockInfo = priorityQueue.peek(); // cannot be null - if (hasLongerDurationThan(lockInfo, existingLockInfo)) { - priorityQueue.poll(); - byLockPath.remove(existingLockInfo.getLockPath()); - return true; - } - - return false; - } - - List asList() { - synchronized (monitor) { - return List.copyOf(byLockPath.values()); - } - } - - void clear() { - synchronized (monitor) { - byLockPath.clear(); - priorityQueue.clear(); - } - } - - private static boolean hasLongerDurationThan(LockInfo lockInfo, LockInfo otherLockInfo) { - // Use stable total duration to avoid messing up priority queue. - return lockInfo.getStableTotalDuration().compareTo(otherLockInfo.getStableTotalDuration()) > 0; - } -} diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java deleted file mode 100644 index 92e6eb453b6..00000000000 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockInfo.java +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.curator.stats; - -import com.yahoo.vespa.curator.Lock; - -import java.time.Duration; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.function.Consumer; - -/** - * This class contains process-wide statistics and information related to acquiring and releasing - * {@link Lock}. Instances of this class contain information tied to a specific thread and lock path. - * - *

Instances of this class are thread-safe as long as foreign threads (!= this.thread) avoid mutable methods.

- * - * @author hakon - */ -public class ThreadLockInfo { - - private static final ConcurrentHashMap locks = new ConcurrentHashMap<>(); - - private static final LockInfoSamples completedLockInfoSamples = new LockInfoSamples(); - - private static final ConcurrentHashMap countersByLockPath = new ConcurrentHashMap<>(); - - private final Thread thread; - - /** The locks are reentrant so there may be more than 1 lock for this thread. */ - private final ConcurrentLinkedDeque lockInfos = new ConcurrentLinkedDeque<>(); - - public static Map getLockCountersByPath() { return Map.copyOf(countersByLockPath); } - - public static List getThreadLockInfos() { return List.copyOf(locks.values()); } - - public static List getLockInfoSamples() { - return completedLockInfoSamples.asList(); - } - - /** Returns the per-thread singleton ThreadLockInfo. */ - public static ThreadLockInfo getCurrentThreadLockInfo() { - return locks.computeIfAbsent(Thread.currentThread(), ThreadLockInfo::new); - } - - static void clearStaticDataForTesting() { - locks.clear(); - completedLockInfoSamples.clear(); - countersByLockPath.clear(); - } - - ThreadLockInfo(Thread currentThread) { - this.thread = currentThread; - } - - public String getThreadName() { return thread.getName(); } - - public String getStackTrace() { - var stackTrace = new StringBuilder(); - - StackTraceElement[] elements = thread.getStackTrace(); - for (int i = 0; i < elements.length; ++i) { - var element = elements[i]; - stackTrace.append(element.getClassName()) - .append('.') - .append(element.getMethodName()) - .append('(') - .append(element.getFileName()) - .append(':') - .append(element.getLineNumber()) - .append(")\n"); - } - - return stackTrace.toString(); - } - - public List getLockInfos() { return List.copyOf(lockInfos); } - - /** Mutable method (see class doc) */ - public void invokingAcquire(String lockPath, Duration timeout) { - LockCounters lockCounters = getLockCounters(lockPath); - lockCounters.invokeAcquireCount.incrementAndGet(); - lockCounters.inCriticalRegionCount.incrementAndGet(); - lockInfos.addLast(LockInfo.invokingAcquire(this, lockPath, timeout)); - } - - /** Mutable method (see class doc) */ - public void acquireFailed(String lockPath) { - LockCounters lockCounters = getLockCounters(lockPath); - lockCounters.acquireFailedCount.incrementAndGet(); - removeLastLockInfo(lockCounters, LockInfo::acquireFailed); - } - - /** Mutable method (see class doc) */ - public void acquireTimedOut(String lockPath) { - LockCounters lockCounters = getLockCounters(lockPath); - if (lockInfos.size() > 1) { - lockCounters.timeoutOnReentrancyErrorCount.incrementAndGet(); - } - - lockCounters.acquireTimedOutCount.incrementAndGet(); - removeLastLockInfo(lockCounters, LockInfo::timedOut); - } - - /** Mutable method (see class doc) */ - public void lockAcquired(String lockPath) { - getLockCounters(lockPath).lockAcquiredCount.incrementAndGet(); - LockInfo lastLockInfo = lockInfos.peekLast(); - if (lastLockInfo == null) { - throw new IllegalStateException("lockAcquired invoked without lockInfos"); - } - lastLockInfo.lockAcquired(); - } - - /** Mutable method (see class doc) */ - public void lockReleased(String lockPath) { - LockCounters lockCounters = getLockCounters(lockPath); - lockCounters.locksReleasedCount.incrementAndGet(); - removeLastLockInfo(lockCounters, LockInfo::released); - } - - private LockCounters getLockCounters(String lockPath) { - return countersByLockPath.computeIfAbsent(lockPath, __ -> new LockCounters()); - } - - private void removeLastLockInfo(LockCounters lockCounters, Consumer completeLockInfo) { - lockCounters.inCriticalRegionCount.decrementAndGet(); - - if (lockInfos.isEmpty()) { - lockCounters.noLocksErrorCount.incrementAndGet(); - return; - } - - LockInfo lockInfo = lockInfos.pollLast(); - completeLockInfo.accept(lockInfo); - completedLockInfoSamples.maybeSample(lockInfo); - } -} diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java new file mode 100644 index 00000000000..117844e17ee --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java @@ -0,0 +1,139 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator.stats; + +import com.yahoo.vespa.curator.Lock; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.function.Consumer; + +/** + * This class contains process-wide statistics and information related to acquiring and releasing + * {@link Lock}. Instances of this class contain information tied to a specific thread and lock path. + * + *

Instances of this class are thread-safe as long as foreign threads (!= this.thread) avoid mutable methods.

+ * + * @author hakon + */ +public class ThreadLockStats { + + private static final ConcurrentHashMap locks = new ConcurrentHashMap<>(); + + private static final LockAttemptSamples COMPLETED_LOCK_ATTEMPT_SAMPLES = new LockAttemptSamples(); + + private static final ConcurrentHashMap countersByLockPath = new ConcurrentHashMap<>(); + + private final Thread thread; + + /** The locks are reentrant so there may be more than 1 lock for this thread. */ + private final ConcurrentLinkedDeque lockAttempts = new ConcurrentLinkedDeque<>(); + + public static Map getLockCountersByPath() { return Map.copyOf(countersByLockPath); } + + public static List getThreadLockInfos() { return List.copyOf(locks.values()); } + + public static List getLockInfoSamples() { + return COMPLETED_LOCK_ATTEMPT_SAMPLES.asList(); + } + + /** Returns the per-thread singleton ThreadLockStats. */ + public static ThreadLockStats getCurrentThreadLockInfo() { + return locks.computeIfAbsent(Thread.currentThread(), ThreadLockStats::new); + } + + static void clearStaticDataForTesting() { + locks.clear(); + COMPLETED_LOCK_ATTEMPT_SAMPLES.clear(); + countersByLockPath.clear(); + } + + ThreadLockStats(Thread currentThread) { + this.thread = currentThread; + } + + public String getThreadName() { return thread.getName(); } + + public String getStackTrace() { + var stackTrace = new StringBuilder(); + + StackTraceElement[] elements = thread.getStackTrace(); + for (int i = 0; i < elements.length; ++i) { + var element = elements[i]; + stackTrace.append(element.getClassName()) + .append('.') + .append(element.getMethodName()) + .append('(') + .append(element.getFileName()) + .append(':') + .append(element.getLineNumber()) + .append(")\n"); + } + + return stackTrace.toString(); + } + + public List getLockAttempts() { return List.copyOf(lockAttempts); } + + /** Mutable method (see class doc) */ + public void invokingAcquire(String lockPath, Duration timeout) { + LockCounters lockCounters = getLockCounters(lockPath); + lockCounters.invokeAcquireCount.incrementAndGet(); + lockCounters.inCriticalRegionCount.incrementAndGet(); + lockAttempts.addLast(LockAttempt.invokingAcquire(this, lockPath, timeout)); + } + + /** Mutable method (see class doc) */ + public void acquireFailed(String lockPath) { + LockCounters lockCounters = getLockCounters(lockPath); + lockCounters.acquireFailedCount.incrementAndGet(); + removeLastLockInfo(lockCounters, LockAttempt::acquireFailed); + } + + /** Mutable method (see class doc) */ + public void acquireTimedOut(String lockPath) { + LockCounters lockCounters = getLockCounters(lockPath); + if (lockAttempts.size() > 1) { + lockCounters.timeoutOnReentrancyErrorCount.incrementAndGet(); + } + + lockCounters.acquireTimedOutCount.incrementAndGet(); + removeLastLockInfo(lockCounters, LockAttempt::timedOut); + } + + /** Mutable method (see class doc) */ + public void lockAcquired(String lockPath) { + getLockCounters(lockPath).lockAcquiredCount.incrementAndGet(); + LockAttempt lastLockAttempt = lockAttempts.peekLast(); + if (lastLockAttempt == null) { + throw new IllegalStateException("lockAcquired invoked without lockAttempts"); + } + lastLockAttempt.lockAcquired(); + } + + /** Mutable method (see class doc) */ + public void lockReleased(String lockPath) { + LockCounters lockCounters = getLockCounters(lockPath); + lockCounters.locksReleasedCount.incrementAndGet(); + removeLastLockInfo(lockCounters, LockAttempt::released); + } + + private LockCounters getLockCounters(String lockPath) { + return countersByLockPath.computeIfAbsent(lockPath, __ -> new LockCounters()); + } + + private void removeLastLockInfo(LockCounters lockCounters, Consumer completeLockInfo) { + lockCounters.inCriticalRegionCount.decrementAndGet(); + + if (lockAttempts.isEmpty()) { + lockCounters.noLocksErrorCount.incrementAndGet(); + return; + } + + LockAttempt lockAttempt = lockAttempts.pollLast(); + completeLockInfo.accept(lockAttempt); + COMPLETED_LOCK_ATTEMPT_SAMPLES.maybeSample(lockAttempt); + } +} diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockAttemptSamplesTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockAttemptSamplesTest.java new file mode 100644 index 00000000000..6f877631b7b --- /dev/null +++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockAttemptSamplesTest.java @@ -0,0 +1,62 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator.stats; + +import org.junit.Test; + +import java.time.Duration; +import java.time.Instant; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author hakon + */ +public class LockAttemptSamplesTest { + private final LockAttemptSamples samples = new LockAttemptSamples(2); + private ThreadLockStats threadLockStats; + + @Test + public void test() { + threadLockStats = new ThreadLockStats(Thread.currentThread()); + + assertTrue(maybeSample("1", 10)); + + // new sample has longer duration + assertTrue(maybeSample("1", 11)); + + // new sample has shorter duration + assertFalse(maybeSample("1", 10)); + + // new path, will be added + assertTrue(maybeSample("2", 5)); + + // new path, too low duration be added + assertFalse(maybeSample("3", 4)); + + // new path, expels "2" + assertTrue(maybeSample("4", 6)); + + Map lockInfos = samples.asList().stream().collect(Collectors.toMap( + lockInfo -> lockInfo.getLockPath(), + lockInfo -> lockInfo)); + assertEquals(2, lockInfos.size()); + + assertTrue(lockInfos.containsKey("1")); + assertEquals(Duration.ofSeconds(11), lockInfos.get("1").getStableTotalDuration()); + + assertTrue(lockInfos.containsKey("4")); + assertEquals(Duration.ofSeconds(6), lockInfos.get("4").getStableTotalDuration()); + } + + private boolean maybeSample(String lockPath, int secondsDuration) { + LockAttempt lockAttempt = LockAttempt.invokingAcquire(threadLockStats, lockPath, Duration.ofSeconds(1)); + Instant instant = lockAttempt.getTimeAcquiredWasInvoked().plus(Duration.ofSeconds(secondsDuration)); + lockAttempt.setTerminalState(LockAttempt.LockState.RELEASED, instant); + return samples.maybeSample(lockAttempt); + } + +} \ No newline at end of file diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockInfoSamplesTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockInfoSamplesTest.java deleted file mode 100644 index 4a14b0cc1b2..00000000000 --- a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockInfoSamplesTest.java +++ /dev/null @@ -1,58 +0,0 @@ -package com.yahoo.vespa.curator.stats;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -import org.junit.Test; - -import java.time.Duration; -import java.time.Instant; -import java.util.Map; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -public class LockInfoSamplesTest { - private final LockInfoSamples samples = new LockInfoSamples(2); - private ThreadLockInfo threadLockInfo; - - @Test - public void test() { - threadLockInfo = new ThreadLockInfo(Thread.currentThread()); - - assertTrue(maybeSample("1", 10)); - - // new sample has longer duration - assertTrue(maybeSample("1", 11)); - - // new sample has shorter duration - assertFalse(maybeSample("1", 10)); - - // new path, will be added - assertTrue(maybeSample("2", 5)); - - // new path, too low duration be added - assertFalse(maybeSample("3", 4)); - - // new path, expels "2" - assertTrue(maybeSample("4", 6)); - - Map lockInfos = samples.asList().stream().collect(Collectors.toMap( - lockInfo -> lockInfo.getLockPath(), - lockInfo -> lockInfo)); - assertEquals(2, lockInfos.size()); - - assertTrue(lockInfos.containsKey("1")); - assertEquals(Duration.ofSeconds(11), lockInfos.get("1").getStableTotalDuration()); - - assertTrue(lockInfos.containsKey("4")); - assertEquals(Duration.ofSeconds(6), lockInfos.get("4").getStableTotalDuration()); - } - - private boolean maybeSample(String lockPath, int secondsDuration) { - LockInfo lockInfo = LockInfo.invokingAcquire(threadLockInfo, lockPath, Duration.ofSeconds(1)); - Instant instant = lockInfo.getTimeAcquiredWasInvoked().plus(Duration.ofSeconds(secondsDuration)); - lockInfo.setTerminalState(LockInfo.LockState.RELEASED, instant); - return samples.maybeSample(lockInfo); - } - -} \ No newline at end of file diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java index 984a04c6d4e..477fe650bb5 100644 --- a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java +++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java @@ -1,4 +1,5 @@ -package com.yahoo.vespa.curator.stats;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator.stats; import com.yahoo.vespa.curator.Lock; import org.apache.curator.framework.recipes.locks.InterProcessLock; @@ -19,6 +20,9 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +/** + * @author hakon + */ public class LockTest { private final InterProcessLock mutex = mock(InterProcessLock.class); private final String lockPath = "/lock/path"; @@ -27,7 +31,7 @@ public class LockTest { @Before public void setUp() { - ThreadLockInfo.clearStaticDataForTesting(); + ThreadLockStats.clearStaticDataForTesting(); } @Test @@ -45,22 +49,22 @@ public class LockTest { var expectedCounters = new LockCounters(); expectedCounters.invokeAcquireCount.set(1); expectedCounters.acquireFailedCount.set(1); - assertEquals(Map.of(lockPath, expectedCounters), ThreadLockInfo.getLockCountersByPath()); + assertEquals(Map.of(lockPath, expectedCounters), ThreadLockStats.getLockCountersByPath()); - List slowLockInfos = ThreadLockInfo.getLockInfoSamples(); - assertEquals(1, slowLockInfos.size()); - LockInfo slowLockInfo = slowLockInfos.get(0); - assertEquals(acquireTimeout, slowLockInfo.getAcquireTimeout()); - Optional stackTrace = slowLockInfo.getStackTrace(); + List slowLockAttempts = ThreadLockStats.getLockInfoSamples(); + assertEquals(1, slowLockAttempts.size()); + LockAttempt slowLockAttempt = slowLockAttempts.get(0); + assertEquals(acquireTimeout, slowLockAttempt.getAcquireTimeout()); + Optional stackTrace = slowLockAttempt.getStackTrace(); assertTrue(stackTrace.isPresent()); assertTrue("bad stacktrace: " + stackTrace.get(), stackTrace.get().contains(".Lock.acquire(Lock.java")); - assertEquals(LockInfo.LockState.ACQUIRE_FAILED, slowLockInfo.getLockState()); - assertTrue(slowLockInfo.getTimeTerminalStateWasReached().isPresent()); + assertEquals(LockAttempt.LockState.ACQUIRE_FAILED, slowLockAttempt.getLockState()); + assertTrue(slowLockAttempt.getTimeTerminalStateWasReached().isPresent()); - List threadLockInfos = ThreadLockInfo.getThreadLockInfos(); - assertEquals(1, threadLockInfos.size()); - ThreadLockInfo threadLockInfo = threadLockInfos.get(0); - assertEquals(0, threadLockInfo.getLockInfos().size()); + List threadLockStatsList = ThreadLockStats.getThreadLockInfos(); + assertEquals(1, threadLockStatsList.size()); + ThreadLockStats threadLockStats = threadLockStatsList.get(0); + assertEquals(0, threadLockStats.getLockAttempts().size()); } @Test @@ -77,7 +81,7 @@ public class LockTest { var expectedCounters = new LockCounters(); expectedCounters.invokeAcquireCount.set(1); expectedCounters.acquireTimedOutCount.set(1); - assertEquals(Map.of(lockPath, expectedCounters), ThreadLockInfo.getLockCountersByPath()); + assertEquals(Map.of(lockPath, expectedCounters), ThreadLockStats.getLockCountersByPath()); } @Test @@ -90,7 +94,7 @@ public class LockTest { expectedCounters.invokeAcquireCount.set(1); expectedCounters.lockAcquiredCount.set(1); expectedCounters.inCriticalRegionCount.set(1); - assertEquals(Map.of(lockPath, expectedCounters), ThreadLockInfo.getLockCountersByPath()); + assertEquals(Map.of(lockPath, expectedCounters), ThreadLockStats.getLockCountersByPath()); // reenter lock.acquire(acquireTimeout); @@ -102,13 +106,13 @@ public class LockTest { lock.close(); expectedCounters.inCriticalRegionCount.set(1); expectedCounters.locksReleasedCount.set(1); - assertEquals(Map.of(lockPath, expectedCounters), ThreadLockInfo.getLockCountersByPath()); + assertEquals(Map.of(lockPath, expectedCounters), ThreadLockStats.getLockCountersByPath()); // outer-most closes lock.close(); expectedCounters.inCriticalRegionCount.set(0); expectedCounters.locksReleasedCount.set(2); - assertEquals(Map.of(lockPath, expectedCounters), ThreadLockInfo.getLockCountersByPath()); + assertEquals(Map.of(lockPath, expectedCounters), ThreadLockStats.getLockCountersByPath()); } @Test @@ -121,14 +125,14 @@ public class LockTest { lock.acquire(acquireTimeout); lock2.acquire(acquireTimeout); - List threadLockInfos = ThreadLockInfo.getThreadLockInfos(); - assertEquals(1, threadLockInfos.size()); - List lockInfos = threadLockInfos.get(0).getLockInfos(); - assertEquals(2, lockInfos.size()); - assertEquals(lockPath, lockInfos.get(0).getLockPath()); - assertEquals(LockInfo.LockState.ACQUIRED, lockInfos.get(0).getLockState()); - assertEquals(lockPath2, lockInfos.get(1).getLockPath()); - assertEquals(LockInfo.LockState.ACQUIRED, lockInfos.get(1).getLockState()); + List threadLockStats = ThreadLockStats.getThreadLockInfos(); + assertEquals(1, threadLockStats.size()); + List lockAttempts = threadLockStats.get(0).getLockAttempts(); + assertEquals(2, lockAttempts.size()); + assertEquals(lockPath, lockAttempts.get(0).getLockPath()); + assertEquals(LockAttempt.LockState.ACQUIRED, lockAttempts.get(0).getLockState()); + assertEquals(lockPath2, lockAttempts.get(1).getLockPath()); + assertEquals(LockAttempt.LockState.ACQUIRED, lockAttempts.get(1).getLockState()); lock.close(); lock.close(); -- cgit v1.2.3 From 489cde8360e4552d44b36ee5fd0b0e5c631068df Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Mon, 28 Sep 2020 13:50:08 +0200 Subject: Remove useless mock --- .../config/server/configchange/MockRestartAction.java | 17 ----------------- .../vespa/config/server/deploy/HostedDeployTest.java | 6 +++--- 2 files changed, 3 insertions(+), 20 deletions(-) delete mode 100644 configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRestartAction.java diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRestartAction.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRestartAction.java deleted file mode 100644 index b1183f91282..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRestartAction.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.configchange; - -import com.yahoo.config.model.api.ConfigChangeRestartAction; -import com.yahoo.config.model.api.ServiceInfo; - -import java.util.List; - -/** - * @author geirst - * @since 5.44 - */ -public class MockRestartAction extends MockConfigChangeAction implements ConfigChangeRestartAction { - public MockRestartAction(String message, List services) { - super(message, services); - } -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index 04d018b71c0..18ca54e8c11 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -20,12 +20,12 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; -import com.yahoo.vespa.config.server.configchange.MockRestartAction; import com.yahoo.vespa.config.server.configchange.RestartActions; import com.yahoo.vespa.config.server.http.InvalidApplicationException; import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.model.TestModelFactory; import com.yahoo.vespa.config.server.session.PrepareParams; +import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -377,8 +377,8 @@ public class HostedDeployTest { new ServiceInfo("serviceName", "serviceType", null, new HashMap<>(), "configId", "hostName")); List modelFactories = List.of( - new ConfigChangeActionsModelFactory(Version.fromString("6.1.0"), new MockRestartAction("change", services)), - new ConfigChangeActionsModelFactory(Version.fromString("6.2.0"), new MockRestartAction("other change", services))); + new ConfigChangeActionsModelFactory(Version.fromString("6.1.0"), new VespaRestartAction("change", services)), + new ConfigChangeActionsModelFactory(Version.fromString("6.2.0"), new VespaRestartAction("other change", services))); DeployTester tester = createTester(hosts, modelFactories, prodZone); PrepareResult prepareResult = tester.deployApp("src/test/apps/hosted/", "6.2.0"); -- cgit v1.2.3 From c888ea9295ea84216a920d9ceca3883f75f1ce32 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Mon, 28 Sep 2020 13:52:23 +0200 Subject: More info -> attempt renames --- .../vespa/hosted/provision/restapi/LocksResponse.java | 4 ++-- zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java | 4 ++-- .../com/yahoo/vespa/curator/stats/ThreadLockStats.java | 16 ++++++++-------- .../vespa/curator/stats/LockAttemptSamplesTest.java | 16 ++++++++-------- .../java/com/yahoo/vespa/curator/stats/LockTest.java | 6 +++--- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java index 59050106cba..1d4efd05619 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java @@ -26,8 +26,8 @@ public class LocksResponse extends HttpResponse { public LocksResponse() { this(new TreeMap<>(ThreadLockStats.getLockCountersByPath()), - ThreadLockStats.getThreadLockInfos(), - ThreadLockStats.getLockInfoSamples()); + ThreadLockStats.getThreadLockStats(), + ThreadLockStats.getLockAttemptSamples()); } /** For testing */ diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java index 6e3b79ec5ce..60e497683a6 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java @@ -35,7 +35,7 @@ public class Lock implements Mutex { /** Take the lock with the given timeout. This may be called multiple times from the same thread - each matched by a close */ public void acquire(Duration timeout) throws UncheckedTimeoutException { - ThreadLockStats threadLockStats = ThreadLockStats.getCurrentThreadLockInfo(); + ThreadLockStats threadLockStats = ThreadLockStats.getCurrentThreadLockStats(); threadLockStats.invokingAcquire(lockPath, timeout); final boolean acquired; @@ -56,7 +56,7 @@ public class Lock implements Mutex { @Override public void close() { - ThreadLockStats.getCurrentThreadLockInfo().lockReleased(lockPath); + ThreadLockStats.getCurrentThreadLockStats().lockReleased(lockPath); try { mutex.release(); } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java index 117844e17ee..077c3e2fd3d 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java @@ -33,14 +33,14 @@ public class ThreadLockStats { public static Map getLockCountersByPath() { return Map.copyOf(countersByLockPath); } - public static List getThreadLockInfos() { return List.copyOf(locks.values()); } + public static List getThreadLockStats() { return List.copyOf(locks.values()); } - public static List getLockInfoSamples() { + public static List getLockAttemptSamples() { return COMPLETED_LOCK_ATTEMPT_SAMPLES.asList(); } /** Returns the per-thread singleton ThreadLockStats. */ - public static ThreadLockStats getCurrentThreadLockInfo() { + public static ThreadLockStats getCurrentThreadLockStats() { return locks.computeIfAbsent(Thread.currentThread(), ThreadLockStats::new); } @@ -89,7 +89,7 @@ public class ThreadLockStats { public void acquireFailed(String lockPath) { LockCounters lockCounters = getLockCounters(lockPath); lockCounters.acquireFailedCount.incrementAndGet(); - removeLastLockInfo(lockCounters, LockAttempt::acquireFailed); + removeLastLockAttempt(lockCounters, LockAttempt::acquireFailed); } /** Mutable method (see class doc) */ @@ -100,7 +100,7 @@ public class ThreadLockStats { } lockCounters.acquireTimedOutCount.incrementAndGet(); - removeLastLockInfo(lockCounters, LockAttempt::timedOut); + removeLastLockAttempt(lockCounters, LockAttempt::timedOut); } /** Mutable method (see class doc) */ @@ -117,14 +117,14 @@ public class ThreadLockStats { public void lockReleased(String lockPath) { LockCounters lockCounters = getLockCounters(lockPath); lockCounters.locksReleasedCount.incrementAndGet(); - removeLastLockInfo(lockCounters, LockAttempt::released); + removeLastLockAttempt(lockCounters, LockAttempt::released); } private LockCounters getLockCounters(String lockPath) { return countersByLockPath.computeIfAbsent(lockPath, __ -> new LockCounters()); } - private void removeLastLockInfo(LockCounters lockCounters, Consumer completeLockInfo) { + private void removeLastLockAttempt(LockCounters lockCounters, Consumer completeLockAttempt) { lockCounters.inCriticalRegionCount.decrementAndGet(); if (lockAttempts.isEmpty()) { @@ -133,7 +133,7 @@ public class ThreadLockStats { } LockAttempt lockAttempt = lockAttempts.pollLast(); - completeLockInfo.accept(lockAttempt); + completeLockAttempt.accept(lockAttempt); COMPLETED_LOCK_ATTEMPT_SAMPLES.maybeSample(lockAttempt); } } diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockAttemptSamplesTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockAttemptSamplesTest.java index 6f877631b7b..14dbbad56ba 100644 --- a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockAttemptSamplesTest.java +++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockAttemptSamplesTest.java @@ -40,16 +40,16 @@ public class LockAttemptSamplesTest { // new path, expels "2" assertTrue(maybeSample("4", 6)); - Map lockInfos = samples.asList().stream().collect(Collectors.toMap( - lockInfo -> lockInfo.getLockPath(), - lockInfo -> lockInfo)); - assertEquals(2, lockInfos.size()); + Map lockAttempts = samples.asList().stream().collect(Collectors.toMap( + lockAttempt -> lockAttempt.getLockPath(), + lockAttempt -> lockAttempt)); + assertEquals(2, lockAttempts.size()); - assertTrue(lockInfos.containsKey("1")); - assertEquals(Duration.ofSeconds(11), lockInfos.get("1").getStableTotalDuration()); + assertTrue(lockAttempts.containsKey("1")); + assertEquals(Duration.ofSeconds(11), lockAttempts.get("1").getStableTotalDuration()); - assertTrue(lockInfos.containsKey("4")); - assertEquals(Duration.ofSeconds(6), lockInfos.get("4").getStableTotalDuration()); + assertTrue(lockAttempts.containsKey("4")); + assertEquals(Duration.ofSeconds(6), lockAttempts.get("4").getStableTotalDuration()); } private boolean maybeSample(String lockPath, int secondsDuration) { diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java index 477fe650bb5..92911b0dadf 100644 --- a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java +++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java @@ -51,7 +51,7 @@ public class LockTest { expectedCounters.acquireFailedCount.set(1); assertEquals(Map.of(lockPath, expectedCounters), ThreadLockStats.getLockCountersByPath()); - List slowLockAttempts = ThreadLockStats.getLockInfoSamples(); + List slowLockAttempts = ThreadLockStats.getLockAttemptSamples(); assertEquals(1, slowLockAttempts.size()); LockAttempt slowLockAttempt = slowLockAttempts.get(0); assertEquals(acquireTimeout, slowLockAttempt.getAcquireTimeout()); @@ -61,7 +61,7 @@ public class LockTest { assertEquals(LockAttempt.LockState.ACQUIRE_FAILED, slowLockAttempt.getLockState()); assertTrue(slowLockAttempt.getTimeTerminalStateWasReached().isPresent()); - List threadLockStatsList = ThreadLockStats.getThreadLockInfos(); + List threadLockStatsList = ThreadLockStats.getThreadLockStats(); assertEquals(1, threadLockStatsList.size()); ThreadLockStats threadLockStats = threadLockStatsList.get(0); assertEquals(0, threadLockStats.getLockAttempts().size()); @@ -125,7 +125,7 @@ public class LockTest { lock.acquire(acquireTimeout); lock2.acquire(acquireTimeout); - List threadLockStats = ThreadLockStats.getThreadLockInfos(); + List threadLockStats = ThreadLockStats.getThreadLockStats(); assertEquals(1, threadLockStats.size()); List lockAttempts = threadLockStats.get(0).getLockAttempts(); assertEquals(2, lockAttempts.size()); -- cgit v1.2.3 From 24b812dc562a2727dda0b0f20759d153f0283ee9 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sat, 26 Sep 2020 21:52:19 +0200 Subject: - Amortise write cost by grouping mulptiple operations together when writing to TLS. - Commit memorystructures only then persisting to disk. - Ack operations back to user when both are completed. - Do not schedule a new commit task until both the tls and the memory structures have been comitted. --- .../documentmetastore/lid_reuse_delayer_config.cpp | 2 +- .../proton/server/document_db_maintenance_config.h | 2 +- .../vespa/searchcore/proton/server/feedhandler.cpp | 46 +++++++++++++-- .../vespa/searchcore/proton/server/feedhandler.h | 6 ++ .../src/vespa/searchlib/transactionlog/domain.cpp | 66 ++++++++++++++++++++-- .../src/vespa/searchlib/transactionlog/domain.h | 5 ++ 6 files changed, 116 insertions(+), 11 deletions(-) diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp index 9db1fee58e1..1f979d1566c 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp @@ -19,7 +19,7 @@ LidReuseDelayerConfig::LidReuseDelayerConfig() LidReuseDelayerConfig::LidReuseDelayerConfig(vespalib::duration visibilityDelay, bool hasIndexedOrAttributeFields_in) : _visibilityDelay(visibilityDelay), - _allowEarlyAck(visibilityDelay > vespalib::duration::zero()), + _allowEarlyAck(visibilityDelay > 1ms), _hasIndexedOrAttributeFields(hasIndexedOrAttributeFields_in) { } diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h index 388a1101dcf..5ed0ad7492c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h @@ -135,7 +135,7 @@ public: } vespalib::duration getVisibilityDelay() const { return _visibilityDelay; } bool hasVisibilityDelay() const { return _visibilityDelay > vespalib::duration::zero(); } - bool allowEarlyAck() const { return hasVisibilityDelay(); } + bool allowEarlyAck() const { return _visibilityDelay > 1ms; } const DocumentDBLidSpaceCompactionConfig &getLidSpaceCompactionConfig() const { return _lidSpaceCompaction; } diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 282ec3c9ace..75414052ef5 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -93,18 +93,18 @@ TlsMgrWriter::sync(SerialNum syncTo) { for (int retryCount = 0; retryCount < 10; ++retryCount) { SerialNum syncedTo(0); - LOG(spam, "Trying tls sync(%" PRIu64 ")", syncTo); + LOG(debug, "Trying tls sync(%" PRIu64 ")", syncTo); bool res = _tls_mgr.getSession()->sync(syncTo, syncedTo); if (!res) { - LOG(spam, "Tls sync failed, retrying"); + LOG(debug, "Tls sync failed, retrying"); sleep(1); continue; } if (syncedTo >= syncTo) { - LOG(spam, "Tls sync complete, reached %" PRIu64", returning", syncedTo); + LOG(debug, "Tls sync complete, reached %" PRIu64", returning", syncedTo); return syncedTo; } - LOG(spam, "Tls sync incomplete, reached %" PRIu64 ", retrying", syncedTo); + LOG(debug, "Tls sync incomplete, reached %" PRIu64 ", retrying", syncedTo); } throw IllegalStateException(make_string("Failed to sync TLS to token %" PRIu64 ".", syncTo)); } @@ -402,6 +402,9 @@ FeedHandler::FeedHandler(IThreadingService &writeService, _tlsReplayProgress(), _serialNum(0), _prunedSerialNum(0), + _numOperationsPendingCommit(0), + _numOperationsCompleted(0), + _numCommitsCompleted(0), _delayedPrune(false), _feedLock(), _feedState(make_shared(getDocTypeName())), @@ -494,12 +497,47 @@ FeedHandler::getTransactionLogReplayDone() const { return _tlsMgr.getReplayDone(); } +void +FeedHandler::onCommitDone(size_t numPendingAtStart) { + assert(numPendingAtStart <= _numOperationsPendingCommit); + _numOperationsPendingCommit -= numPendingAtStart; + _numOperationsCompleted += numPendingAtStart; + _numCommitsCompleted++; + if (_numOperationsPendingCommit > 0) { + enqueCommitTask(); + } + LOG(spam, "%zu: onCommitDone(%zu) total=%zu left=%zu", + _numCommitsCompleted, numPendingAtStart, _numOperationsCompleted, _numOperationsPendingCommit); +} + +void FeedHandler::enqueCommitTask() { + _writeService.master().execute(makeLambdaTask([this]() { initiateCommit(); })); +} + +void +FeedHandler::initiateCommit() { + auto onCommitDoneContext = std::make_shared( + _writeService.master(), + makeLambdaTask([this, numPendingAtStart=_numOperationsPendingCommit]() { + onCommitDone(numPendingAtStart); + })); + auto commitResult = _tlsWriter->startCommit(onCommitDoneContext); + if (_activeFeedView) { + using KeepAlivePair = KeepAlive>; + auto pair = std::make_pair(std::move(commitResult), std::move(onCommitDoneContext)); + _activeFeedView->forceCommit(_serialNum, std::make_shared(std::move(pair))); + } +} + void FeedHandler::appendOperation(const FeedOperation &op, TlsWriter::DoneCallback onDone) { if (!op.getSerialNum()) { const_cast(op).setSerialNum(incSerialNum()); } _tlsWriter->appendOperation(op, std::move(onDone)); + if (++_numOperationsPendingCommit == 1) { + enqueCommitTask(); + } } FeedHandler::CommitResult diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index 4807c596130..c295b26a759 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -76,6 +76,9 @@ private: // the serial num of the last message in the transaction log SerialNum _serialNum; SerialNum _prunedSerialNum; + size_t _numOperationsPendingCommit; + size_t _numOperationsCompleted; + size_t _numCommitsCompleted; bool _delayedPrune; mutable std::shared_mutex _feedLock; FeedStateSP _feedState; @@ -125,6 +128,9 @@ private: FeedStateSP getFeedState() const; void changeFeedState(FeedStateSP newState); void doChangeFeedState(FeedStateSP newState); + void onCommitDone(size_t numPendingAtStart); + void initiateCommit(); + void enqueCommitTask(); public: FeedHandler(const FeedHandler &) = delete; FeedHandler & operator = (const FeedHandler &) = delete; diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index 9e0f1a8a1aa..bd7feec0598 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -113,7 +113,12 @@ Domain::addPart(SerialNum partId, bool isLastPart) { } } -Domain::~Domain() { } +Domain::~Domain() { + MonitorGuard guard(_currentChunkMonitor); + guard.broadcast(); + commitChunk(grabCurrentChunk(guard), guard); + _singleCommitter->shutdown().sync(); +} DomainInfo Domain::getDomainInfo() const @@ -318,22 +323,73 @@ Domain::optionallyRotateFile(SerialNum serialNum) { return dp; } +void +Domain::append(const Packet & packet, Writer::DoneCallback onDone) { + vespalib::MonitorGuard guard(_currentChunkMonitor); + if (_lastSerial >= packet.range().from()) { + throw runtime_error(fmt("Incoming serial number(%" PRIu64 ") must be bigger than the last one (%" PRIu64 ").", + packet.range().from(), _lastSerial)); + } else { + _lastSerial = packet.range().to(); + } + _currentChunk->add(packet, std::move(onDone)); + commitIfFull(guard); +} + Domain::CommitResult Domain::startCommit(DoneCallback onDone) { - (void) onDone; + vespalib::MonitorGuard guard(_currentChunkMonitor); + if ( !_currentChunk->empty() ) { + auto completed = grabCurrentChunk(guard); + completed->setCommitDoneCallback(std::move(onDone)); + CommitResult result(completed->createCommitResult()); + commitChunk(std::move(completed), guard); + return result; + } return CommitResult(); } void -Domain::append(const Packet & packet, Writer::DoneCallback onDone) -{ - (void) onDone; +Domain::commitIfFull(const vespalib::MonitorGuard &guard) { + if (_currentChunk->sizeBytes() > _config.getChunkSizeLimit()) { + auto completed = std::move(_currentChunk); + _currentChunk = std::make_unique(_config.getChunkSizeLimit(), completed->stealCallbacks()); + commitChunk(std::move(completed), guard); + } +} + +std::unique_ptr +Domain::grabCurrentChunk(const vespalib::MonitorGuard & guard) { + assert(guard.monitors(_currentChunkMonitor)); + auto chunk = std::move(_currentChunk); + _currentChunk = createCommitChunk(_config); + return chunk; +} + +void +Domain::commitChunk(std::unique_ptr chunk, const vespalib::MonitorGuard & chunkOrderGuard) { + assert(chunkOrderGuard.monitors(_currentChunkMonitor)); + _singleCommitter->execute( makeLambdaTask([this, chunk = std::move(chunk)]() mutable { + doCommit(std::move(chunk)); + })); +} + +void +Domain::doCommit(std::unique_ptr chunk) { + const Packet & packet = chunk->getPacket(); + if (packet.empty()) return; + vespalib::nbostream_longlivedbuf is(packet.getHandle().data(), packet.getHandle().size()); Packet::Entry entry; entry.deserialize(is); DomainPart::SP dp = optionallyRotateFile(entry.serial()); dp->commit(entry.serial(), packet); + if (_config.getFSyncOnCommit()) { + dp->sync(); + } cleanSessions(); + LOG(debug, "Releasing %zu acks and %zu entries and %zu bytes.", + chunk->getNumCallBacks(), chunk->getPacket().size(), chunk->sizeBytes()); } bool diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h index 7e77e6ef0ef..041ec27cf23 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h @@ -56,6 +56,11 @@ public: uint64_t size() const; Domain & setConfig(const DomainConfig & cfg); private: + void commitIfFull(const vespalib::MonitorGuard & guard); + + std::unique_ptr grabCurrentChunk(const vespalib::MonitorGuard & guard); + void commitChunk(std::unique_ptr chunk, const vespalib::MonitorGuard & chunkOrderGuard); + void doCommit(std::unique_ptr chunk); SerialNum begin(const vespalib::LockGuard & guard) const; SerialNum end(const vespalib::LockGuard & guard) const; size_t byteSize(const vespalib::LockGuard & guard) const; -- cgit v1.2.3 From aca1fd385447a49175a09b301619aa99bee20aa9 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Mon, 28 Sep 2020 14:11:41 +0200 Subject: Remove hardcoding, use feature flag when deciding to use config sever VIP --- .../vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java | 8 +++----- .../vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java index 36a46f72d34..4a245fb3555 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java @@ -116,11 +116,9 @@ public class ZoneApiHandler extends AuditLoggingRequestHandler { boolean useConfigServerVip = Flags.USE_CONFIG_SERVER_VIP.bindTo(flagSource) .with(FetchVector.Dimension.ZONE_ID, zoneId.value()).value(); - // TODO: Still need to hardcode AWS since flag cannot be set until flag has been rolled out - if (zoneId.region().value().startsWith("aws-") || useConfigServerVip) { - return ProxyRequest.tryOne(zoneRegistry.getConfigServerVipUri(zoneId), path, request); - } - return ProxyRequest.tryAll(zoneRegistry.getConfigServerUris(zoneId), path, request); + return useConfigServerVip + ? ProxyRequest.tryOne(zoneRegistry.getConfigServerVipUri(zoneId), path, request) + : ProxyRequest.tryAll(zoneRegistry.getConfigServerUris(zoneId), path, request); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java index a7adac7f89d..61915860d7c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java @@ -81,7 +81,7 @@ public class ZoneApiTest extends ControllerContainerTest { tester.assertResponse(operatorRequest("http://localhost:8080/zone/v2/dev/aws-us-north-2/nodes/v2/node/node1", "{\"currentRestartGeneration\": 1}", Method.PATCH), "ok"); - assertLastRequest(ZoneId.from("dev", "aws-us-north-2"), 1, "PATCH"); + assertLastRequest(ZoneId.from("dev", "aws-us-north-2"), 2, "PATCH"); assertEquals("{\"currentRestartGeneration\": 1}", proxy.lastRequestBody().get()); assertFalse("Actions are logged to audit log", tester.controller().auditLogger().readLog().entries().isEmpty()); -- cgit v1.2.3 From 0cf64bbf1ce25a077b82d8262c7e0fe89bd4b172 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Mon, 28 Sep 2020 14:18:02 +0200 Subject: Provide underlying executor through getter --- container-core/abi-spec.json | 3 ++- .../main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index 28aeb5155c2..ceaf1426ef2 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -632,7 +632,8 @@ "public final com.yahoo.jdisc.handler.ContentChannel handleRequest(com.yahoo.jdisc.Request, com.yahoo.jdisc.handler.ResponseHandler)", "public java.time.Duration getTimeout()", "protected abstract void handleRequest(com.yahoo.jdisc.Request, com.yahoo.jdisc.handler.BufferedContentChannel, com.yahoo.jdisc.handler.ResponseHandler)", - "protected void writeErrorResponseOnOverload(com.yahoo.jdisc.Request, com.yahoo.jdisc.handler.ResponseHandler)" + "protected void writeErrorResponseOnOverload(com.yahoo.jdisc.Request, com.yahoo.jdisc.handler.ResponseHandler)", + "protected java.util.concurrent.Executor executor()" ], "fields": [ "protected final com.yahoo.jdisc.Metric metric" diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java index 0e8865538ee..ab768dba0d2 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java @@ -169,6 +169,8 @@ public abstract class ThreadedRequestHandler extends AbstractRequestHandler { ResponseDispatch.newInstance(Response.Status.SERVICE_UNAVAILABLE).dispatch(responseHandler); } + protected Executor executor() { return executor; } + private class RequestTask implements ResponseHandler, Runnable { final Request request; -- cgit v1.2.3 From 314aa652acd2409def51ac051093a99c8bdf0ce0 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Mon, 28 Sep 2020 14:20:36 +0200 Subject: Add count of failed releases --- .../hosted/provision/restapi/LocksResponse.java | 46 +++++++++++----------- .../main/java/com/yahoo/vespa/curator/Lock.java | 3 +- .../com/yahoo/vespa/curator/stats/LockAttempt.java | 4 +- .../yahoo/vespa/curator/stats/LockCounters.java | 11 +++--- .../yahoo/vespa/curator/stats/ThreadLockStats.java | 10 +++-- 5 files changed, 41 insertions(+), 33 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java index 1d4efd05619..8692be2b322 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java @@ -32,7 +32,7 @@ public class LocksResponse extends HttpResponse { /** For testing */ LocksResponse(TreeMap lockCountersByPath, - List threadLockStats, + List threadLockStatsList, List historicSamples) { super(200); @@ -49,27 +49,27 @@ public class LocksResponse extends HttpResponse { lockPathCursor.setLong("lock-acquired", lockCounters.lockAcquiredCount()); lockPathCursor.setLong("locks-released", lockCounters.locksReleasedCount()); lockPathCursor.setLong("no-locks-errors", lockCounters.noLocksErrorCount()); - lockPathCursor.setLong("timeout-on-reentrancy-errors", lockCounters.timeoutOnReentrancyErrorCount()); + lockPathCursor.setLong("lock-release-errors", lockCounters.lockReleaseErrorCount()); }); Cursor threadsCursor = root.setArray("threads"); - for (var threadLockInfo : threadLockStats) { - List lockAttempts = threadLockInfo.getLockAttempts(); + for (var threadLockStats : threadLockStatsList) { + List lockAttempts = threadLockStats.getLockAttempts(); if (!lockAttempts.isEmpty()) { - Cursor threadLockInfoCursor = threadsCursor.addObject(); - threadLockInfoCursor.setString("thread-name", threadLockInfo.getThreadName()); + Cursor threadLockStatsCursor = threadsCursor.addObject(); + threadLockStatsCursor.setString("thread-name", threadLockStats.getThreadName()); - Cursor lockInfosCursor = threadLockInfoCursor.setArray("active-locks"); - for (var lockInfo : lockAttempts) { - setLockInfo(lockInfosCursor.addObject(), lockInfo, false); + Cursor lockAttemptsCursor = threadLockStatsCursor.setArray("active-locks"); + for (var lockAttempt : lockAttempts) { + setLockAttempt(lockAttemptsCursor.addObject(), lockAttempt, false); } - threadLockInfoCursor.setString("stack-trace", threadLockInfo.getStackTrace()); + threadLockStatsCursor.setString("stack-trace", threadLockStats.getStackTrace()); } } Cursor historicSamplesCursor = root.setArray("historic-samples"); - historicSamples.forEach(lockInfo -> setLockInfo(historicSamplesCursor.addObject(), lockInfo, true)); + historicSamples.forEach(lockAttempt -> setLockAttempt(historicSamplesCursor.addObject(), lockAttempt, true)); } @Override @@ -82,21 +82,21 @@ public class LocksResponse extends HttpResponse { return "application/json"; } - private void setLockInfo(Cursor lockInfoCursor, LockAttempt lockAttempt, boolean includeThreadInfo) { + private void setLockAttempt(Cursor lockAttemptCursor, LockAttempt lockAttempt, boolean includeThreadInfo) { if (includeThreadInfo) { - lockInfoCursor.setString("thread-name", lockAttempt.getThreadName()); + lockAttemptCursor.setString("thread-name", lockAttempt.getThreadName()); } - lockInfoCursor.setString("lock-path", lockAttempt.getLockPath()); - lockInfoCursor.setString("invoke-acquire-time", toString(lockAttempt.getTimeAcquiredWasInvoked())); - lockInfoCursor.setString("acquire-timeout", lockAttempt.getAcquireTimeout().toString()); - lockAttempt.getTimeLockWasAcquired().ifPresent(instant -> lockInfoCursor.setString("lock-acquired-time", toString(instant))); - lockInfoCursor.setString("lock-state", lockAttempt.getLockState().name()); - lockAttempt.getTimeTerminalStateWasReached().ifPresent(instant -> lockInfoCursor.setString("terminal-state-time", toString(instant))); - lockInfoCursor.setString("acquire-duration", lockAttempt.getDurationOfAcquire().toString()); - lockInfoCursor.setString("locked-duration", lockAttempt.getDurationWithLock().toString()); - lockInfoCursor.setString("total-duration", lockAttempt.getDuration().toString()); + lockAttemptCursor.setString("lock-path", lockAttempt.getLockPath()); + lockAttemptCursor.setString("invoke-acquire-time", toString(lockAttempt.getTimeAcquiredWasInvoked())); + lockAttemptCursor.setString("acquire-timeout", lockAttempt.getAcquireTimeout().toString()); + lockAttempt.getTimeLockWasAcquired().ifPresent(instant -> lockAttemptCursor.setString("lock-acquired-time", toString(instant))); + lockAttemptCursor.setString("lock-state", lockAttempt.getLockState().name()); + lockAttempt.getTimeTerminalStateWasReached().ifPresent(instant -> lockAttemptCursor.setString("terminal-state-time", toString(instant))); + lockAttemptCursor.setString("acquire-duration", lockAttempt.getDurationOfAcquire().toString()); + lockAttemptCursor.setString("locked-duration", lockAttempt.getDurationWithLock().toString()); + lockAttemptCursor.setString("total-duration", lockAttempt.getDuration().toString()); if (includeThreadInfo) { - lockAttempt.getStackTrace().ifPresent(stackTrace -> lockInfoCursor.setString("stack-trace", stackTrace)); + lockAttempt.getStackTrace().ifPresent(stackTrace -> lockAttemptCursor.setString("stack-trace", stackTrace)); } } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java index 60e497683a6..920bba22804 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java @@ -56,11 +56,12 @@ public class Lock implements Mutex { @Override public void close() { - ThreadLockStats.getCurrentThreadLockStats().lockReleased(lockPath); try { mutex.release(); + ThreadLockStats.getCurrentThreadLockStats().lockReleased(lockPath); } catch (Exception e) { + ThreadLockStats.getCurrentThreadLockStats().lockReleaseFailed(lockPath); throw new RuntimeException("Exception releasing lock '" + lockPath + "'"); } } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttempt.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttempt.java index c092fb6c289..3b06377ccf7 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttempt.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttempt.java @@ -29,7 +29,8 @@ public class LockAttempt { } public enum LockState { - ACQUIRING(false), ACQUIRE_FAILED(true), TIMED_OUT(true), ACQUIRED(false), RELEASED(true); + ACQUIRING(false), ACQUIRE_FAILED(true), TIMED_OUT(true), ACQUIRED(false), RELEASED(true), + RELEASED_WITH_ERROR(true); private final boolean terminal; @@ -84,6 +85,7 @@ public class LockAttempt { void acquireFailed() { setTerminalState(LockState.ACQUIRE_FAILED); } void timedOut() { setTerminalState(LockState.TIMED_OUT); } void released() { setTerminalState(LockState.RELEASED); } + void releasedWithError() { setTerminalState(LockState.RELEASED_WITH_ERROR); } void lockAcquired() { lockState = LockState.ACQUIRED; diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java index 5309bf755d7..561ea9a7ed2 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java @@ -18,7 +18,7 @@ public class LockCounters { final AtomicInteger locksReleasedCount = new AtomicInteger(0); final AtomicInteger noLocksErrorCount = new AtomicInteger(0); - final AtomicInteger timeoutOnReentrancyErrorCount = new AtomicInteger(0); + final AtomicInteger lockReleaseErrorCount = new AtomicInteger(0); public int invokeAcquireCount() { return invokeAcquireCount.get(); } public int inCriticalRegionCount() { return inCriticalRegionCount.get(); } @@ -27,7 +27,7 @@ public class LockCounters { public int lockAcquiredCount() { return lockAcquiredCount.get(); } public int locksReleasedCount() { return locksReleasedCount.get(); } public int noLocksErrorCount() { return noLocksErrorCount.get(); } - public int timeoutOnReentrancyErrorCount() { return timeoutOnReentrancyErrorCount.get(); } + public int lockReleaseErrorCount() { return lockReleaseErrorCount.get(); } @Override public String toString() { @@ -39,7 +39,7 @@ public class LockCounters { ", lockAcquiredCount=" + lockAcquiredCount + ", locksReleasedCount=" + locksReleasedCount + ", noLocksErrorCount=" + noLocksErrorCount + - ", timeoutOnReentrancyErrorCount=" + timeoutOnReentrancyErrorCount + + ", locksReleaseErrorCount=" + lockReleaseErrorCount + '}'; } @@ -55,11 +55,12 @@ public class LockCounters { lockAcquiredCount.get() == that.lockAcquiredCount.get() && locksReleasedCount.get() == that.locksReleasedCount.get() && noLocksErrorCount.get() == that.noLocksErrorCount.get() && - timeoutOnReentrancyErrorCount.get() == that.timeoutOnReentrancyErrorCount.get(); + lockReleaseErrorCount.get() == that.lockReleaseErrorCount.get(); } @Override public int hashCode() { - return Objects.hash(invokeAcquireCount, inCriticalRegionCount, acquireFailedCount, acquireTimedOutCount, lockAcquiredCount, locksReleasedCount, noLocksErrorCount, timeoutOnReentrancyErrorCount); + return Objects.hash(invokeAcquireCount, inCriticalRegionCount, acquireFailedCount, acquireTimedOutCount, + lockAcquiredCount, locksReleasedCount, noLocksErrorCount, lockReleaseErrorCount); } } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java index 077c3e2fd3d..db26523ec37 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java @@ -95,9 +95,6 @@ public class ThreadLockStats { /** Mutable method (see class doc) */ public void acquireTimedOut(String lockPath) { LockCounters lockCounters = getLockCounters(lockPath); - if (lockAttempts.size() > 1) { - lockCounters.timeoutOnReentrancyErrorCount.incrementAndGet(); - } lockCounters.acquireTimedOutCount.incrementAndGet(); removeLastLockAttempt(lockCounters, LockAttempt::timedOut); @@ -120,6 +117,13 @@ public class ThreadLockStats { removeLastLockAttempt(lockCounters, LockAttempt::released); } + /** Mutable method (see class doc) */ + public void lockReleaseFailed(String lockPath) { + LockCounters lockCounters = getLockCounters(lockPath); + lockCounters.lockReleaseErrorCount.incrementAndGet(); + removeLastLockAttempt(lockCounters, LockAttempt::releasedWithError); + } + private LockCounters getLockCounters(String lockPath) { return countersByLockPath.computeIfAbsent(lockPath, __ -> new LockCounters()); } -- cgit v1.2.3 From 664ee879879f17fcd7fe63167dd9b2a7ded59efc Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Mon, 28 Sep 2020 14:35:56 +0200 Subject: Increase default node object cache size --- flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 2776c374051..291011f91d9 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -406,7 +406,7 @@ public class Flags { public static final UnboundLongFlag NODE_OBJECT_CACHE_SIZE = defineLongFlag( "node-object-cache-size", - 1000, + 2000, "The number of deserialized Node objects to store in-memory.", "Takes effect on config server restart"); -- cgit v1.2.3 From 9aaebc79a7835a17f5d1ad3cec3713a20b36014a Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Mon, 28 Sep 2020 14:38:15 +0200 Subject: Revert "Add maintainer for debugging zookeeper locks" --- .../maintenance/ConfigServerMaintenance.java | 3 - .../config/server/maintenance/LocksMaintainer.java | 65 ---------------------- 2 files changed, 68 deletions(-) delete mode 100644 configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/LocksMaintainer.java diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java index 751e983770a..0e75d683478 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java @@ -27,7 +27,6 @@ public class ConfigServerMaintenance extends AbstractComponent { private final FileDistributionMaintainer fileDistributionMaintainer; private final SessionsMaintainer sessionsMaintainer; private final ApplicationPackageMaintainer applicationPackageMaintainer; - private final LocksMaintainer locksMaintainer; @Inject public ConfigServerMaintenance(ConfigServerBootstrap configServerBootstrap, @@ -41,7 +40,6 @@ public class ConfigServerMaintenance extends AbstractComponent { fileDistributionMaintainer = new FileDistributionMaintainer(applicationRepository, curator, defaults.defaultInterval, flagSource); sessionsMaintainer = new SessionsMaintainer(applicationRepository, curator, Duration.ofSeconds(30), flagSource); applicationPackageMaintainer = new ApplicationPackageMaintainer(applicationRepository, curator, Duration.ofSeconds(30), flagSource); - locksMaintainer = new LocksMaintainer(applicationRepository, curator, Duration.ofSeconds(10), flagSource); } @Override @@ -50,7 +48,6 @@ public class ConfigServerMaintenance extends AbstractComponent { sessionsMaintainer.close(); applicationPackageMaintainer.close(); tenantsMaintainer.close(); - locksMaintainer.close(); } /* diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/LocksMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/LocksMaintainer.java deleted file mode 100644 index b43d65d4c0a..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/LocksMaintainer.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.maintenance; - -import com.yahoo.path.Path; -import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.flags.FlagSource; -import org.apache.zookeeper.data.Stat; - -import java.time.Duration; -import java.time.Instant; -import java.util.List; -import java.util.Optional; -import java.util.logging.Level; - -/** - * Debug maintainer for logging info about node repository locks. Logs with level FINE, so to actually log something - * one needs to change log levels with vespa-logctl - * - * @author hmusum - */ -public class LocksMaintainer extends ConfigServerMaintainer { - - private final boolean hostedVespa; - private final Curator curator; - - LocksMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval, FlagSource flagSource) { - super(applicationRepository, curator, flagSource, Duration.ZERO, interval); - this.hostedVespa = applicationRepository.configserverConfig().hostedVespa(); - this.curator = curator; - } - - @Override - protected boolean maintain() { - if (! hostedVespa) return true; - - Path unallocatedLockPath = Path.fromString("/provision/v1/locks/unallocatedLock"); - logLockInfo(unallocatedLockPath); - - applicationRepository.listApplications().forEach(applicationId -> { - Path applicationLockPath = Path.fromString("/provision/v1/locks").append(applicationId.tenant().value()) - .append(applicationId.application().value()) - .append(applicationId.instance().value()); - logLockInfo(applicationLockPath); - }); - - return true; - } - - private void logLockInfo(Path path) { - List children = curator.getChildren(path); - if (children.size() > 0) - log.log(Level.FINE, path + " has " + children.size() + " locks "); - children.forEach(lockString -> { - Optional createTime = Optional.empty(); - Path lockPath = path.append(lockString); - Optional stat = curator.getStat(lockPath); - if (stat.isPresent()) { - createTime = Optional.of(Instant.ofEpochMilli(stat.get().getCtime())); - } - log.log(Level.FINE, "Lock /" + lockPath + " created " + createTime.map(Instant::toString).orElse(" at unknown time")); - }); - } - -} -- cgit v1.2.3 From 90c2c465a15a817c5579623c7efc60df3ff73e3c Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sun, 27 Sep 2020 21:03:13 +0000 Subject: Drop commit of memory structures if high visibility-delay --- .../src/vespa/searchcore/proton/server/combiningfeedview.cpp | 10 ++++++++++ .../src/vespa/searchcore/proton/server/combiningfeedview.h | 1 + searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp | 2 +- searchcore/src/vespa/searchcore/proton/server/ifeedview.h | 1 + .../src/vespa/searchcore/proton/server/storeonlyfeedview.cpp | 7 ++++++- .../src/vespa/searchcore/proton/server/storeonlyfeedview.h | 1 + searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h | 1 + 7 files changed, 21 insertions(+), 2 deletions(-) diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp index 066e135741e..ea5d46f02ad 100644 --- a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp @@ -218,6 +218,16 @@ CombiningFeedView::heartBeat(search::SerialNum serialNum) } } +bool +CombiningFeedView::allowEarlyAck() const { + for (const auto &view : _views) { + if ( ! view->allowEarlyAck() ) { + return false; + } + } + return true; +} + void CombiningFeedView::sync() { diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h index d1da0408318..3a37fdc37cb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h @@ -83,6 +83,7 @@ public: // Called by document db executor void setCalculator(const IBucketStateCalculator::SP &newCalc); + bool allowEarlyAck() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 75414052ef5..8b82478c1a4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -522,7 +522,7 @@ FeedHandler::initiateCommit() { onCommitDone(numPendingAtStart); })); auto commitResult = _tlsWriter->startCommit(onCommitDoneContext); - if (_activeFeedView) { + if (_activeFeedView && ! _activeFeedView->allowEarlyAck()) { using KeepAlivePair = KeepAlive>; auto pair = std::make_pair(std::move(commitResult), std::move(onCommitDoneContext)); _activeFeedView->forceCommit(_serialNum, std::make_shared(std::move(pair))); diff --git a/searchcore/src/vespa/searchcore/proton/server/ifeedview.h b/searchcore/src/vespa/searchcore/proton/server/ifeedview.h index 4b028a289a9..8dd7ae8474e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/ifeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/ifeedview.h @@ -65,6 +65,7 @@ public: virtual void handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation & pruneOp) = 0; virtual void handleCompactLidSpace(const CompactLidSpaceOperation &op) = 0; virtual ILidCommitState & getUncommittedLidsTracker() = 0; + virtual bool allowEarlyAck() const = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index bda94c22adc..90875aa8591 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -275,7 +275,7 @@ StoreOnlyFeedView::internalForceCommit(SerialNum serialNum, OnForceCommitDoneTyp void StoreOnlyFeedView::considerEarlyAck(FeedToken & token) { - if ( _lidReuseDelayer.allowEarlyAck() && token) { + if (allowEarlyAck() && token) { token.reset(); } } @@ -349,6 +349,11 @@ StoreOnlyFeedView::needImmediateCommit() const { return _lidReuseDelayer.needImmediateCommit(); } +bool +StoreOnlyFeedView::allowEarlyAck() const { + return _lidReuseDelayer.allowEarlyAck(); +} + void StoreOnlyFeedView::heartBeatIndexedFields(SerialNum ) {} diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h index 4569e01f9fd..20942423995 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h @@ -264,6 +264,7 @@ public: void handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation &pruneOp) override; void handleCompactLidSpace(const CompactLidSpaceOperation &op) override; ILidCommitState & getUncommittedLidsTracker() override; + bool allowEarlyAck() const final override; }; } diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h b/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h index adfc911c8df..b96fd77409c 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h @@ -34,6 +34,7 @@ struct DummyFeedView : public IFeedView void handleCompactLidSpace(const CompactLidSpaceOperation &) override {} void forceCommit(search::SerialNum, DoneCallback) override { } ILidCommitState & getUncommittedLidsTracker() override; + bool allowEarlyAck() const override { return false; } }; } -- cgit v1.2.3 From 84124f5a88bed7d247b1f6fc2d01dca8624cbdf8 Mon Sep 17 00:00:00 2001 From: Frode Lundgren Date: Mon, 28 Sep 2020 14:55:29 +0200 Subject: Only match end of string since the order of how files are tested varies. --- .../controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java index 21e89cb5ea3..aca991ec637 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java @@ -92,7 +92,7 @@ public class SystemFlagsDataArchiveTest { @Test public void duplicated_flagdata_is_detected() { expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("Flag data file in 'flags/group-1/my-test-flag/default.json' contains redundant flag data for id 'my-test-flag' already set in another directory!"); + expectedException.expectMessage("contains redundant flag data for id 'my-test-flag' already set in another directory!"); var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags-multi-level-with-duplicated-flagdata/")); } -- cgit v1.2.3 From 1d3182f6e4eac5f2c07691a79f855ab8bd5a8b03 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Mon, 28 Sep 2020 15:05:29 +0200 Subject: Support ignoring actions for internal redeploy --- .../yahoo/config/model/api/ConfigChangeAction.java | 3 ++ .../change/ContainerRestartValidator.java | 2 +- .../validation/change/VespaRefeedAction.java | 16 ++++---- .../validation/change/VespaRestartAction.java | 35 ++++++++++++++-- .../vespa/config/server/ApplicationRepository.java | 6 +-- .../config/server/configchange/RestartActions.java | 46 +++++++++++++++------- .../vespa/config/server/deploy/Deployment.java | 33 ++++++++++------ .../configchange/ConfigChangeActionsBuilder.java | 11 +++++- .../server/configchange/MockRefeedAction.java | 5 +++ .../server/configchange/RestartActionsTest.java | 16 ++++++++ 10 files changed, 126 insertions(+), 47 deletions(-) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java index e453aaf5bdb..ea4b6a2b02d 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java @@ -38,4 +38,7 @@ public interface ConfigChangeAction { /** Returns whether this change should be allowed */ boolean allowed(); + /** Returns whether this change should be ignored for internal redeploy */ + boolean ignoreForInternalRedeploy(); + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java index 5cee08ea9af..3176ad9f912 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java @@ -31,7 +31,7 @@ public class ContainerRestartValidator implements ChangeValidator { } private static ConfigChangeAction createConfigChangeAction(Container container) { - return new VespaRestartAction(createMessage(container), container.getServiceInfo()); + return new VespaRestartAction(createMessage(container), container.getServiceInfo(), true); } private static String createMessage(Container container) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java index 0fd38e5dbdd..be43c6eddfb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java @@ -6,7 +6,6 @@ import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.application.api.ValidationOverrides; import java.time.Instant; -import java.util.Collections; import java.util.List; /** @@ -27,31 +26,29 @@ public class VespaRefeedAction extends VespaConfigChangeAction implements Config private final String documentType; private final boolean allowed; - private final Instant now; - private VespaRefeedAction(String name, String message, List services, String documentType, boolean allowed, Instant now) { + private VespaRefeedAction(String name, String message, List services, String documentType, boolean allowed) { super(message, services); this.name = name; this.documentType = documentType; this.allowed = allowed; - this.now = now; } /** Creates a refeed action with some missing information */ // TODO: We should require document type or model its absence properly public static VespaRefeedAction of(String name, ValidationOverrides overrides, String message, Instant now) { - return new VespaRefeedAction(name, message, Collections.emptyList(), "", overrides.allows(name, now), now); + return new VespaRefeedAction(name, message, List.of(), "", overrides.allows(name, now)); } /** Creates a refeed action */ public static VespaRefeedAction of(String name, ValidationOverrides overrides, String message, List services, String documentType, Instant now) { - return new VespaRefeedAction(name, message, services, documentType, overrides.allows(name, now), now); + return new VespaRefeedAction(name, message, services, documentType, overrides.allows(name, now)); } @Override public VespaConfigChangeAction modifyAction(String newMessage, List newServices, String documentType) { - return new VespaRefeedAction(name, newMessage, newServices, documentType, allowed, now); + return new VespaRefeedAction(name, newMessage, newServices, documentType, allowed); } @Override @@ -63,6 +60,11 @@ public class VespaRefeedAction extends VespaConfigChangeAction implements Config @Override public boolean allowed() { return allowed; } + @Override + public boolean ignoreForInternalRedeploy() { + return false; + } + @Override public String toString() { return super.toString() + ", documentType='" + documentType + "'"; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java index d42db15d062..3ea18cac1d6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java @@ -4,8 +4,6 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeRestartAction; import com.yahoo.config.model.api.ServiceInfo; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; /** @@ -15,16 +13,24 @@ import java.util.List; */ public class VespaRestartAction extends VespaConfigChangeAction implements ConfigChangeRestartAction { + private final boolean ignoreForInternalRedeploy; + public VespaRestartAction(String message) { - super(message, new ArrayList<>()); + this(message, List.of()); } public VespaRestartAction(String message, ServiceInfo service) { - super(message, Collections.singletonList(service)); + this(message, List.of(service)); + } + + public VespaRestartAction(String message, ServiceInfo services, boolean ignoreForInternalRedeploy) { + super(message, List.of(services)); + this.ignoreForInternalRedeploy = ignoreForInternalRedeploy; } public VespaRestartAction(String message, List services) { super(message, services); + this.ignoreForInternalRedeploy = false; } @Override @@ -32,4 +38,25 @@ public class VespaRestartAction extends VespaConfigChangeAction implements Confi return new VespaRestartAction(newMessage, newServices); } + @Override + public boolean ignoreForInternalRedeploy() { + return ignoreForInternalRedeploy; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + + VespaRestartAction that = (VespaRestartAction) o; + return ignoreForInternalRedeploy == that.ignoreForInternalRedeploy; + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (ignoreForInternalRedeploy ? 1 : 0); + return result; + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 57502973873..4670184c303 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -402,15 +402,11 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye TimeoutBudget timeoutBudget, boolean force) { LocalSession localSession = getLocalSession(tenant, sessionId); - Deployment deployment = deployment(localSession, tenant, timeoutBudget.timeLeft(), force); + Deployment deployment = Deployment.prepared(localSession, this, hostProvisioner, tenant, logger, timeoutBudget.timeout(), clock, false, force); deployment.activate(); return localSession.getApplicationId(); } - private Deployment deployment(LocalSession session, Tenant tenant, Duration timeout, boolean force) { - return Deployment.prepared(session, this, hostProvisioner, tenant, logger, timeout, clock, false, force); - } - public Transaction deactivateCurrentActivateNew(Session active, LocalSession prepared, boolean force) { Tenant tenant = tenantRepository.getTenant(prepared.getTenantName()); Transaction transaction = tenant.getSessionRepository().createActivateTransaction(prepared); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java index 4db5a5e125f..326f146f64c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java @@ -5,6 +5,7 @@ import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; import java.util.*; +import java.util.stream.Collectors; /** * Represents all actions to restart services in order to handle a config change. @@ -18,6 +19,7 @@ public class RestartActions { private final String clusterName; private final String clusterType; private final String serviceType; + private final boolean ignoreForInternalRedeploy; private final Set services = new LinkedHashSet<>(); private final Set messages = new TreeSet<>(); @@ -31,10 +33,11 @@ public class RestartActions { return this; } - private Entry(String clusterName, String clusterType, String serviceType) { + private Entry(String clusterName, String clusterType, String serviceType, boolean ignoreForInternalRedeploy) { this.clusterName = clusterName; this.clusterType = clusterType; this.serviceType = serviceType; + this.ignoreForInternalRedeploy = ignoreForInternalRedeploy; } public String getClusterName() { @@ -49,6 +52,10 @@ public class RestartActions { return serviceType; } + public boolean ignoreForInternalRedeploy() { + return ignoreForInternalRedeploy; + } + public Set getServices() { return services; } @@ -59,28 +66,19 @@ public class RestartActions { } - private Entry addEntry(ServiceInfo service) { - String clusterName = service.getProperty("clustername").orElse(""); - String clusterType = service.getProperty("clustertype").orElse(""); - String entryId = clusterType + "." + clusterName + "." + service.getServiceType(); - Entry entry = actions.get(entryId); - if (entry == null) { - entry = new Entry(clusterName, clusterType, service.getServiceType()); - actions.put(entryId, entry); - } - return entry; - } - private final Map actions = new TreeMap<>(); - public RestartActions() { + public RestartActions() { } + + private RestartActions(Map actions) { + this.actions.putAll(actions); } public RestartActions(List actions) { for (ConfigChangeAction action : actions) { if (action.getType().equals(ConfigChangeAction.Type.RESTART)) { for (ServiceInfo service : action.getServices()) { - addEntry(service). + addEntry(service, action.ignoreForInternalRedeploy()). addService(service). addMessage(action.getMessage()); } @@ -88,6 +86,24 @@ public class RestartActions { } } + private Entry addEntry(ServiceInfo service, boolean ignoreForInternalRedeploy) { + String clusterName = service.getProperty("clustername").orElse(""); + String clusterType = service.getProperty("clustertype").orElse(""); + String entryId = clusterType + "." + clusterName + "." + service.getServiceType() + "." + ignoreForInternalRedeploy; + Entry entry = actions.get(entryId); + if (entry == null) { + entry = new Entry(clusterName, clusterType, service.getServiceType(), ignoreForInternalRedeploy); + actions.put(entryId, entry); + } + return entry; + } + + public RestartActions withoutIgnoreForInternalRestart(boolean withoutIgnoreForInternalRestart) { + return new RestartActions(actions.entrySet().stream() + .filter(entry -> !withoutIgnoreForInternalRestart || !entry.getValue().ignoreForInternalRedeploy()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); + } + public List getEntries() { return new ArrayList<>(actions.values()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 5f4f1298528..6d0ad2c1b98 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -52,12 +52,14 @@ public class Deployment implements com.yahoo.config.provision.Deployment { private final Tenant tenant; private final DeployLogger deployLogger; private final Clock clock; + private final boolean internalRedeploy; private boolean prepared; private ConfigChangeActions configChangeActions; private Deployment(LocalSession session, ApplicationRepository applicationRepository, Supplier params, - Optional provisioner, Tenant tenant, DeployLogger deployLogger, Clock clock, boolean prepared) { + Optional provisioner, Tenant tenant, DeployLogger deployLogger, Clock clock, + boolean internalRedeploy, boolean prepared) { this.session = session; this.applicationRepository = applicationRepository; this.params = params; @@ -65,26 +67,27 @@ public class Deployment implements com.yahoo.config.provision.Deployment { this.tenant = tenant; this.deployLogger = deployLogger; this.clock = clock; + this.internalRedeploy = internalRedeploy; this.prepared = prepared; } public static Deployment unprepared(LocalSession session, ApplicationRepository applicationRepository, Optional provisioner, Tenant tenant, PrepareParams params, DeployLogger logger, Clock clock) { - return new Deployment(session, applicationRepository, () -> params, provisioner, tenant, logger, clock, false); + return new Deployment(session, applicationRepository, () -> params, provisioner, tenant, logger, clock, false, false); } public static Deployment unprepared(LocalSession session, ApplicationRepository applicationRepository, Optional provisioner, Tenant tenant, DeployLogger logger, Duration timeout, Clock clock, boolean validate, boolean isBootstrap, boolean internalRestart) { Supplier params = createPrepareParams(clock, timeout, session, isBootstrap, !validate, false, internalRestart); - return new Deployment(session, applicationRepository, params, provisioner, tenant, logger, clock, false); + return new Deployment(session, applicationRepository, params, provisioner, tenant, logger, clock, true, false); } public static Deployment prepared(LocalSession session, ApplicationRepository applicationRepository, Optional provisioner, Tenant tenant, DeployLogger logger, Duration timeout, Clock clock, boolean isBootstrap, boolean force) { Supplier params = createPrepareParams(clock, timeout, session, isBootstrap, false, force, false); - return new Deployment(session, applicationRepository, params, provisioner, tenant, logger, clock, true); + return new Deployment(session, applicationRepository, params, provisioner, tenant, logger, clock, false, true); } /** Prepares this. This does nothing if this is already prepared */ @@ -136,17 +139,21 @@ public class Deployment implements com.yahoo.config.provision.Deployment { (previousActiveSession != null ? ". Based on session " + previousActiveSession.getSessionId() : "") + ". File references: " + applicationRepository.getFileReferences(applicationId)); - if (params.internalRestart() && !configChangeActions.getRestartActions().isEmpty()) { - Set hostnames = configChangeActions.getRestartActions().getEntries().stream() - .flatMap(entry -> entry.getServices().stream()) - .map(ServiceInfo::getHostName) - .collect(Collectors.toUnmodifiableSet()); + if (params.internalRestart()) { + RestartActions restartActions = configChangeActions.getRestartActions().withoutIgnoreForInternalRestart(internalRedeploy); - provisioner.get().restart(applicationId, HostFilter.from(hostnames, Set.of(), Set.of(), Set.of())); - deployLogger.log(Level.INFO, String.format("Scheduled service restart of %d nodes: %s", - hostnames.size(), hostnames.stream().sorted().collect(Collectors.joining(", ")))); + if (!restartActions.isEmpty()) { + Set hostnames = restartActions.getEntries().stream() + .flatMap(entry -> entry.getServices().stream()) + .map(ServiceInfo::getHostName) + .collect(Collectors.toUnmodifiableSet()); - this.configChangeActions = new ConfigChangeActions(new RestartActions(), configChangeActions.getRefeedActions()); + provisioner.get().restart(applicationId, HostFilter.from(hostnames, Set.of(), Set.of(), Set.of())); + deployLogger.log(Level.INFO, String.format("Scheduled service restart of %d nodes: %s", + hostnames.size(), hostnames.stream().sorted().collect(Collectors.joining(", ")))); + + this.configChangeActions = new ConfigChangeActions(new RestartActions(), configChangeActions.getRefeedActions()); + } } return session.getMetaData().getGeneration(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java index e2c3369d49e..ead1e79a416 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.configchange; import com.google.common.collect.ImmutableMap; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; import java.util.ArrayList; import java.util.List; @@ -22,11 +23,17 @@ public class ConfigChangeActionsBuilder { } public ConfigChangeActionsBuilder restart(String message, String clusterName, String clusterType, String serviceType, String serviceName) { - actions.add(new MockRestartAction(message, - List.of(createService(clusterName, clusterType, serviceType, serviceName)))); + return restart(message, clusterName, clusterType, serviceType, serviceName, false); + } + + public ConfigChangeActionsBuilder restart(String message, String clusterName, String clusterType, String serviceType, String serviceName, boolean ignoreForInternalRedeploy) { + actions.add(new VespaRestartAction(message, + createService(clusterName, clusterType, serviceType, serviceName), + ignoreForInternalRedeploy)); return this; } + ConfigChangeActionsBuilder refeed(String name, boolean allowed, String message, String documentType, String clusterName, String serviceName) { actions.add(new MockRefeedAction(name, allowed, diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRefeedAction.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRefeedAction.java index bdf63befd15..904ca10aa1c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRefeedAction.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRefeedAction.java @@ -29,6 +29,11 @@ public class MockRefeedAction extends MockConfigChangeAction implements ConfigCh @Override public boolean allowed() { return allowed; } + @Override + public boolean ignoreForInternalRedeploy() { + return false; + } + @Override public String getDocumentType() { return documentType; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RestartActionsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RestartActionsTest.java index ee0180802af..27940383644 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RestartActionsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RestartActionsTest.java @@ -5,8 +5,10 @@ import com.yahoo.config.model.api.ServiceInfo; import org.junit.Test; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; @@ -81,4 +83,18 @@ public class RestartActionsTest { assertThat(toString(entries.get(0)), equalTo("content.foo.searchnode:[baz][change]")); assertThat(toString(entries.get(1)), equalTo("search.foo.searchnode:[baz][change]")); } + + @Test + public void without_restart_i() { + ConfigChangeActions actions = new ConfigChangeActionsBuilder() + .restart(CHANGE_MSG, CLUSTER, CLUSTER_TYPE, SERVICE_TYPE, SERVICE_NAME) + .restart(CHANGE_MSG, CLUSTER, CLUSTER_TYPE_2, SERVICE_TYPE, SERVICE_NAME, true).build(); + + assertEquals(Set.of(CLUSTER_TYPE, CLUSTER_TYPE_2), + actions.getRestartActions().getEntries().stream().map(RestartActions.Entry::getClusterType).collect(Collectors.toSet())); + assertEquals(Set.of(CLUSTER_TYPE, CLUSTER_TYPE_2), + actions.getRestartActions().withoutIgnoreForInternalRestart(false).getEntries().stream().map(RestartActions.Entry::getClusterType).collect(Collectors.toSet())); + assertEquals(Set.of(CLUSTER_TYPE), + actions.getRestartActions().withoutIgnoreForInternalRestart(true).getEntries().stream().map(RestartActions.Entry::getClusterType).collect(Collectors.toSet())); + } } -- cgit v1.2.3 From 1502d36ebb5f610aa6ed13e42e53da149ff8bc47 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Mon, 28 Sep 2020 15:51:57 +0200 Subject: withoutIgnoreForInternalRestart -> useForInternalRestart --- .../com/yahoo/vespa/config/server/configchange/RestartActions.java | 4 ++-- .../main/java/com/yahoo/vespa/config/server/deploy/Deployment.java | 2 +- .../yahoo/vespa/config/server/configchange/RestartActionsTest.java | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java index 326f146f64c..fab36246a41 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java @@ -98,9 +98,9 @@ public class RestartActions { return entry; } - public RestartActions withoutIgnoreForInternalRestart(boolean withoutIgnoreForInternalRestart) { + public RestartActions useForInternalRestart(boolean useForInternalRestart) { return new RestartActions(actions.entrySet().stream() - .filter(entry -> !withoutIgnoreForInternalRestart || !entry.getValue().ignoreForInternalRedeploy()) + .filter(entry -> !useForInternalRestart || !entry.getValue().ignoreForInternalRedeploy()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 6d0ad2c1b98..3726ea97fcc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -140,7 +140,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { ". File references: " + applicationRepository.getFileReferences(applicationId)); if (params.internalRestart()) { - RestartActions restartActions = configChangeActions.getRestartActions().withoutIgnoreForInternalRestart(internalRedeploy); + RestartActions restartActions = configChangeActions.getRestartActions().useForInternalRestart(internalRedeploy); if (!restartActions.isEmpty()) { Set hostnames = restartActions.getEntries().stream() diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RestartActionsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RestartActionsTest.java index 27940383644..c19b81aa91b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RestartActionsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RestartActionsTest.java @@ -85,7 +85,7 @@ public class RestartActionsTest { } @Test - public void without_restart_i() { + public void use_for_internal_restart_test() { ConfigChangeActions actions = new ConfigChangeActionsBuilder() .restart(CHANGE_MSG, CLUSTER, CLUSTER_TYPE, SERVICE_TYPE, SERVICE_NAME) .restart(CHANGE_MSG, CLUSTER, CLUSTER_TYPE_2, SERVICE_TYPE, SERVICE_NAME, true).build(); @@ -93,8 +93,8 @@ public class RestartActionsTest { assertEquals(Set.of(CLUSTER_TYPE, CLUSTER_TYPE_2), actions.getRestartActions().getEntries().stream().map(RestartActions.Entry::getClusterType).collect(Collectors.toSet())); assertEquals(Set.of(CLUSTER_TYPE, CLUSTER_TYPE_2), - actions.getRestartActions().withoutIgnoreForInternalRestart(false).getEntries().stream().map(RestartActions.Entry::getClusterType).collect(Collectors.toSet())); + actions.getRestartActions().useForInternalRestart(false).getEntries().stream().map(RestartActions.Entry::getClusterType).collect(Collectors.toSet())); assertEquals(Set.of(CLUSTER_TYPE), - actions.getRestartActions().withoutIgnoreForInternalRestart(true).getEntries().stream().map(RestartActions.Entry::getClusterType).collect(Collectors.toSet())); + actions.getRestartActions().useForInternalRestart(true).getEntries().stream().map(RestartActions.Entry::getClusterType).collect(Collectors.toSet())); } } -- cgit v1.2.3