diff options
20 files changed, 108 insertions, 68 deletions
diff --git a/config/src/tests/payload_converter/payload_converter.cpp b/config/src/tests/payload_converter/payload_converter.cpp index 7500d798b24..dceb9751f99 100644 --- a/config/src/tests/payload_converter/payload_converter.cpp +++ b/config/src/tests/payload_converter/payload_converter.cpp @@ -1,8 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("payload_converter"); + #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/common/payload_converter.h> +#include <vespa/vespalib/data/slime/slime.h> + +#include <vespa/log/log.h> +LOG_SETUP("payload_converter"); using namespace config; using namespace vespalib; diff --git a/config/src/vespa/config/common/payload_converter.h b/config/src/vespa/config/common/payload_converter.h index 3e6cbbb44a6..27d1cf7c1c1 100644 --- a/config/src/vespa/config/common/payload_converter.h +++ b/config/src/vespa/config/common/payload_converter.h @@ -1,7 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/data/slime/object_traverser.h> +#include <vespa/vespalib/data/slime/array_traverser.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/asciistream.h> diff --git a/config/src/vespa/config/frt/slimeconfigrequest.h b/config/src/vespa/config/frt/slimeconfigrequest.h index 7ac55b7c70e..6f2f42c98d5 100644 --- a/config/src/vespa/config/frt/slimeconfigrequest.h +++ b/config/src/vespa/config/frt/slimeconfigrequest.h @@ -1,9 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/data/slime/slime.h> #include "frtconfigrequest.h" #include "protocol.h" +#include <vespa/vespalib/data/slime/slime.h> class FRT_Values; class FRT_RPCRequest; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 91cb1895a6d..862e75b088c 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -151,7 +151,7 @@ public class Flags { public static final UnboundBooleanFlag ENABLE_FEED_BLOCK_IN_DISTRIBUTOR = defineFeatureFlag( "enable-feed-block-in-distributor", true, - List.of("geirst"), "2021-01-27", "2021-07-01", + List.of("geirst"), "2021-01-27", "2021-09-01", "Enables blocking of feed in the distributor if resource usage is above limit on at least one content node", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -184,7 +184,7 @@ public class Flags { public static final UnboundIntFlag MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS = defineIntFlag( "max-activation-inhibited-out-of-sync-groups", 0, - List.of("vekterli"), "2021-02-19", "2021-07-01", + List.of("vekterli"), "2021-02-19", "2021-09-01", "Allows replicas in up to N content groups to not be activated " + "for query visibility if they are out of sync with a majority of other replicas", "Takes effect at redeployment", @@ -199,7 +199,7 @@ public class Flags { public static final UnboundIntFlag NUM_DISTRIBUTOR_STRIPES = defineIntFlag( "num-distributor-stripes", 0, - List.of("geirst", "vekterli"), "2021-04-20", "2021-07-01", + List.of("geirst", "vekterli"), "2021-04-20", "2021-09-01", "Specifies the number of stripes used by the distributor. When 0, legacy single stripe behavior is used.", "Takes effect after distributor restart", ZONE_ID, APPLICATION_ID); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java index 1d077449ed6..4e2a5052ea6 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; import java.time.Duration; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -18,12 +19,19 @@ import java.util.concurrent.CountDownLatch; import java.util.stream.Collectors; /** + * A mock implementation of {@link ContainerEngine}. + * + * Container operations are multi-thread safe. Note that this is a requirement since tests may use this through a real + * (multi-threaded) node-admin instance. + * * @author mpolden */ public class ContainerEngineMock implements ContainerEngine { - private final Map<ContainerName, Container> containers = new ConcurrentHashMap<>(); + private final Map<ContainerName, Container> containers = new HashMap<>(); private final Map<String, ImageDownload> images = new ConcurrentHashMap<>(); + private final Object monitor = new Object(); + private boolean asyncImageDownload = false; public ContainerEngineMock asyncImageDownload(boolean enabled) { @@ -50,56 +58,72 @@ public class ContainerEngineMock implements ContainerEngine { } public ContainerEngineMock addContainers(List<Container> containers) { - for (var container : containers) { - if (this.containers.containsKey(container.name())) { - throw new IllegalArgumentException("Container " + container.name() + " already exists"); + synchronized (monitor) { + for (var container : containers) { + if (this.containers.containsKey(container.name())) { + throw new IllegalArgumentException("Container " + container.name() + " already exists"); + } + this.containers.put(container.name(), container); } - this.containers.put(container.name(), container); + return this; } - return this; } public ContainerEngineMock addContainer(Container container) { - return addContainers(List.of(container)); + synchronized (monitor) { + return addContainers(List.of(container)); + } } @Override public void createContainer(NodeAgentContext context, ContainerData containerData, ContainerResources containerResources) { - addContainer(createContainer(context, PartialContainer.State.created, containerResources)); + synchronized (monitor) { + addContainer(createContainer(context, PartialContainer.State.created, containerResources)); + } } @Override public void startContainer(NodeAgentContext context) { - Container container = requireContainer(context.containerName(), PartialContainer.State.created); - Container newContainer = createContainer(context, PartialContainer.State.running, container.resources()); - containers.put(newContainer.name(), newContainer); + synchronized (monitor) { + Container container = requireContainer(context.containerName(), PartialContainer.State.created); + Container newContainer = createContainer(context, PartialContainer.State.running, container.resources()); + containers.put(newContainer.name(), newContainer); + } } @Override public void removeContainer(TaskContext context, PartialContainer container) { - requireContainer(container.name()); - containers.remove(container.name()); + synchronized (monitor) { + requireContainer(container.name()); + containers.remove(container.name()); + } } @Override public void updateContainer(NodeAgentContext context, ContainerId containerId, ContainerResources containerResources) { - Container container = requireContainer(context.containerName()); - containers.put(container.name(), new Container(containerId, container.name(), container.state(), - container.imageId(), container.image(), - container.labels(), container.pid(), - container.conmonPid(), container.hostname(), - containerResources, container.networks(), - container.managed())); + synchronized (monitor) { + Container container = requireContainer(context.containerName()); + containers.put(container.name(), new Container(containerId, container.name(), container.state(), + container.imageId(), container.image(), + container.labels(), container.pid(), + container.conmonPid(), container.hostname(), + containerResources, container.networks(), + container.managed())); + } } @Override public Optional<Container> getContainer(NodeAgentContext context) { - return Optional.ofNullable(containers.get(context.containerName())); + synchronized (monitor) { + return Optional.ofNullable(containers.get(context.containerName())); + } } @Override public List<PartialContainer> listContainers(TaskContext context) { - return List.copyOf(containers.values()); + synchronized (monitor) { + return List.copyOf(containers.values()); + } } @Override diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.h b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.h index 537b98a0618..5c251d14e17 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.h +++ b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.h @@ -11,7 +11,6 @@ #include "dimension.h" #include "dummy_metrics_manager.h" #include "gauge.h" -#include "json_formatter.h" #include "label.h" #include "metric_id.h" #include "metrics_manager.h" diff --git a/vespalib/src/tests/slime/slime_test.cpp b/vespalib/src/tests/slime/slime_test.cpp index e58b1599b8f..6e7b3689004 100644 --- a/vespalib/src/tests/slime/slime_test.cpp +++ b/vespalib/src/tests/slime/slime_test.cpp @@ -2,6 +2,8 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/data/slime/object_value.h> +#include <vespa/vespalib/data/slime/array_value.h> #include <vespa/vespalib/data/slime/strfmt.h> #include <vespa/vespalib/data/simple_buffer.h> #include <type_traits> diff --git a/vespalib/src/vespa/vespalib/data/slime/array_value.cpp b/vespalib/src/vespa/vespalib/data/slime/array_value.cpp index 1d04bfe9c7c..edcc5a8eb85 100644 --- a/vespalib/src/vespa/vespalib/data/slime/array_value.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/array_value.cpp @@ -3,6 +3,7 @@ #include "array_value.h" #include "array_traverser.h" #include "empty_value_factory.h" +#include "symbol_table.h" namespace vespalib::slime { diff --git a/vespalib/src/vespa/vespalib/data/slime/array_value.h b/vespalib/src/vespa/vespalib/data/slime/array_value.h index 5561b93aca2..19f93c3eeec 100644 --- a/vespalib/src/vespa/vespalib/data/slime/array_value.h +++ b/vespalib/src/vespa/vespalib/data/slime/array_value.h @@ -5,9 +5,7 @@ #include "value.h" #include "nix_value.h" #include "value_factory.h" -#include "symbol_table.h" #include <vector> -#include <vespa/vespalib/util/stash.h> namespace vespalib::slime { diff --git a/vespalib/src/vespa/vespalib/data/slime/basic_value.cpp b/vespalib/src/vespa/vespalib/data/slime/basic_value.cpp index d09ac003526..9e96cd7bcf6 100644 --- a/vespalib/src/vespa/vespalib/data/slime/basic_value.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/basic_value.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "basic_value.h" +#include <vespa/vespalib/util/stash.h> namespace vespalib::slime { diff --git a/vespalib/src/vespa/vespalib/data/slime/basic_value.h b/vespalib/src/vespa/vespalib/data/slime/basic_value.h index 0b962085b78..3768523ada0 100644 --- a/vespalib/src/vespa/vespalib/data/slime/basic_value.h +++ b/vespalib/src/vespa/vespalib/data/slime/basic_value.h @@ -5,7 +5,8 @@ #include "value.h" #include "memory.h" #include <vespa/vespalib/util/traits.h> -#include <vespa/vespalib/util/stash.h> + +namespace vespalib { class Stash; } namespace vespalib::slime { diff --git a/vespalib/src/vespa/vespalib/data/slime/empty_value_factory.cpp b/vespalib/src/vespa/vespalib/data/slime/empty_value_factory.cpp index 3be492b7bcc..95be18f57ae 100644 --- a/vespalib/src/vespa/vespalib/data/slime/empty_value_factory.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/empty_value_factory.cpp @@ -1,6 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "empty_value_factory.h" +#include "object_value.h" +#include "array_value.h" +#include <vespa/vespalib/util/stash.h> namespace vespalib::slime { diff --git a/vespalib/src/vespa/vespalib/data/slime/empty_value_factory.h b/vespalib/src/vespa/vespalib/data/slime/empty_value_factory.h index 5fc22b31df9..4046a838ef3 100644 --- a/vespalib/src/vespa/vespalib/data/slime/empty_value_factory.h +++ b/vespalib/src/vespa/vespalib/data/slime/empty_value_factory.h @@ -4,9 +4,6 @@ #include "value_factory.h" #include "nix_value.h" -#include "array_value.h" -#include "object_value.h" -#include <vespa/vespalib/util/stash.h> namespace vespalib::slime { diff --git a/vespalib/src/vespa/vespalib/data/slime/object_value.h b/vespalib/src/vespa/vespalib/data/slime/object_value.h index 651f3a156d2..7f8a809a482 100644 --- a/vespalib/src/vespa/vespalib/data/slime/object_value.h +++ b/vespalib/src/vespa/vespalib/data/slime/object_value.h @@ -6,12 +6,11 @@ #include "nix_value.h" #include "symbol.h" #include "symbol_lookup.h" -#include "symbol_table.h" #include "value_factory.h" #include "symbol_inserter.h" #include <vespa/vespalib/stllike/vector_map.h> -#include <vespa/vespalib/util/stash.h> +namespace vespalib { class Stash; } namespace vespalib::slime { /** diff --git a/vespalib/src/vespa/vespalib/data/slime/root_value.cpp b/vespalib/src/vespa/vespalib/data/slime/root_value.cpp index d99fc90fdee..f910c6ac4cf 100644 --- a/vespalib/src/vespa/vespalib/data/slime/root_value.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/root_value.cpp @@ -1,7 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "root_value.h" +#include "object_value.h" +#include <vespa/vespalib/util/stash.h> namespace vespalib::slime { +Value * +RootValue::wrap(SymbolTable &table, SymbolInserter &symbol) { + Value *value = & _stash->create<ObjectValue>(table, *_stash, symbol, _value); + _value = value; + return _value; +} + } // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/root_value.h b/vespalib/src/vespa/vespalib/data/slime/root_value.h index 314b386de6e..ed1c97b7d60 100644 --- a/vespalib/src/vespa/vespalib/data/slime/root_value.h +++ b/vespalib/src/vespa/vespalib/data/slime/root_value.h @@ -3,7 +3,6 @@ #pragma once #include "nix_value.h" -#include "object_value.h" #include "value_factory.h" namespace vespalib::slime { @@ -35,12 +34,8 @@ public: _value = value; return *value; } - Value *wrap(SymbolTable &table, SymbolInserter &symbol) { - Value *value = & _stash->create<ObjectValue>(table, *_stash, symbol, _value); - _value = value; - return _value; - } - ~RootValue() { } + Value *wrap(SymbolTable &table, SymbolInserter &symbol); + ~RootValue() = default; }; } // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/slime.h b/vespalib/src/vespa/vespalib/data/slime/slime.h index 3ee608799a6..9959d8e7fc8 100644 --- a/vespalib/src/vespa/vespalib/data/slime/slime.h +++ b/vespalib/src/vespa/vespalib/data/slime/slime.h @@ -3,7 +3,6 @@ #pragma once #include "array_traverser.h" -#include "array_value.h" #include "basic_value.h" #include "basic_value_factory.h" #include "binary_format.h" @@ -18,7 +17,6 @@ #include "named_symbol_lookup.h" #include "nix_value.h" #include "object_traverser.h" -#include "object_value.h" #include "resolved_symbol.h" #include "root_value.h" #include "symbol.h" diff --git a/vespalib/src/vespa/vespalib/data/slime/symbol.h b/vespalib/src/vespa/vespalib/data/slime/symbol.h index 77620f3ff92..020fc8bc827 100644 --- a/vespalib/src/vespa/vespalib/data/slime/symbol.h +++ b/vespalib/src/vespa/vespalib/data/slime/symbol.h @@ -12,13 +12,13 @@ namespace vespalib::slime { class Symbol { private: - static const uint32_t UNDEFINED = (uint32_t)-1; + static constexpr uint32_t UNDEFINED = (uint32_t)-1; uint32_t _value; public: - Symbol() : _value(UNDEFINED) {} - Symbol(uint32_t v) : _value(v) {} + Symbol() noexcept : _value(UNDEFINED) {} + Symbol(uint32_t v) noexcept : _value(v) {} bool undefined() const { return (_value == UNDEFINED); } uint32_t getValue() const { return _value; } bool operator<(const Symbol &rhs) const noexcept { return (_value < rhs._value); } diff --git a/vespalib/src/vespa/vespalib/data/slime/value_factory.h b/vespalib/src/vespa/vespalib/data/slime/value_factory.h index 484246cf9e7..b3072778f2a 100644 --- a/vespalib/src/vespa/vespalib/data/slime/value_factory.h +++ b/vespalib/src/vespa/vespalib/data/slime/value_factory.h @@ -14,7 +14,7 @@ class Value; **/ struct ValueFactory { virtual Value *create(Stash & stash) const = 0; - virtual ~ValueFactory() {} + virtual ~ValueFactory() = default; }; } // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/stllike/vector_map.h b/vespalib/src/vespa/vespalib/stllike/vector_map.h index f0cc97ecfae..d9c5fd44bea 100644 --- a/vespalib/src/vespa/vespalib/stllike/vector_map.h +++ b/vespalib/src/vespa/vespalib/stllike/vector_map.h @@ -28,18 +28,23 @@ public: typedef K key_type; typedef V mapped_type; private: - typedef std::vector< value_type> OrderedList; + using OrderedList = std::vector<value_type>; friend bool operator < (const std::pair<K, V> & a, const std::pair<K, V> & b) { LT lt; return lt(a.first, b.first); } - LT _lt; + struct KeyOrder { + bool operator () (const std::pair<K, V> & a, const K & b) { + LT lt; + return lt(a.first, b); + } + }; OrderedList _ht; public: - typedef typename OrderedList::iterator iterator; - typedef typename OrderedList::const_iterator const_iterator; + using iterator = typename OrderedList::iterator; + using const_iterator = typename OrderedList::const_iterator; public: - vector_map(size_t reserveSize=0) : _ht(reserveSize) { } + vector_map() : _ht() { } iterator begin() { return _ht.begin(); } iterator end() { return _ht.end(); } const_iterator begin() const { return _ht.begin(); } @@ -47,26 +52,17 @@ public: size_t capacity() const { return _ht.capacity(); } size_t size() const { return _ht.size(); } bool empty() const { return _ht.empty(); } - V & operator [] (const K & key) const { return _ht.find(key)->second; } - V & operator [] (const K & key) { - value_type v(key, V()); - LT lt; - iterator f = std::lower_bound(begin(), end(), v); - if ((f == end()) || lt(key, f->first)) { - f = _ht.insert(f, v); - } - return f->second; - } + V & operator [] (const K & key); void erase(const K & key) { return _ht.erase(find(key)); } void erase(iterator it) { return _ht.erase(it); } void erase(const_iterator it) { return _ht.erase(it); } iterator find(const K & key) { - iterator f = std::lower_bound(begin(), end(), value_type(key, V())); + iterator f = std::lower_bound(begin(), end(), key, KeyOrder()); LT lt; return ((f != end()) && !lt(key, f->first)) ? f : end(); } const_iterator find(const K & key) const { - const_iterator f = std::lower_bound(begin(), end(), value_type(key, V())); + const_iterator f = std::lower_bound(begin(), end(), key, KeyOrder()); LT lt; return ((f != end()) && !lt(key, f->first)) ? f : end(); } @@ -82,6 +78,19 @@ void swap(vector_map<K, V, LT> & a, vector_map<K, V, LT> & b) a.swap(b); } +template< typename K, typename V, typename LT > +V & +vector_map<K, V, LT>::operator[](const K &key) { + LT lt; + iterator f = std::lower_bound(begin(), end(), key, KeyOrder()); + if (f == end()) { + _ht.template emplace_back(key, V()); + return _ht.rbegin()->second; + } else if (lt(key, f->first)) { + f = _ht.template emplace(f, key, V()); + } + return f->second; +} } |