diff options
28 files changed, 177 insertions, 297 deletions
diff --git a/build_settings.cmake b/build_settings.cmake index 1549ac83c74..e046d7f71f3 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -65,6 +65,9 @@ endif() set(VESPA_ATOMIC_LIB "atomic") if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") set(CXX_SPECIFIC_WARN_OPTS "-Wnon-virtual-dtor -Wformat-security -Wno-overloaded-virtual") + if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 18.0) + set(CXX_SPECIFIC_WARN_OPTS "${CXX_SPECIFIC_WARN_OPTS} -Wno-error=vla-cxx-extension") + endif() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-delete-null-pointer-checks -fsized-deallocation") if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") set(VESPA_ATOMIC_LIB "") diff --git a/client/README.md b/client/README.md index f51db87d631..e730614230f 100644 --- a/client/README.md +++ b/client/README.md @@ -39,7 +39,7 @@ This is a [work-in-progress javascript app](js/app) for querying a Vespa applica <!-- ToDo: move this / demote this somehow --> -### vespa_query_dsl +### vespa\_query\_dsl This lib is used for composing Vespa [YQL queries](https://docs.vespa.ai/en/reference/query-language-reference.html). diff --git a/client/go/internal/vespa/document/throttler.go b/client/go/internal/vespa/document/throttler.go index 39900156563..3eb0ccd17f6 100644 --- a/client/go/internal/vespa/document/throttler.go +++ b/client/go/internal/vespa/document/throttler.go @@ -37,8 +37,8 @@ type dynamicThrottler struct { func newThrottler(connections int, nowFunc func() time.Time) *dynamicThrottler { var ( - minInflight = 16 * int64(connections) - maxInflight = 256 * minInflight // 4096 max streams per connection on the server side + minInflight = 2 * int64(connections) + maxInflight = 256 * minInflight // 512 max streams per connection on the server side ) t := &dynamicThrottler{ minInflight: minInflight, @@ -49,7 +49,7 @@ func newThrottler(connections int, nowFunc func() time.Time) *dynamicThrottler { start: nowFunc(), now: nowFunc, } - t.targetInflight.Store(8 * minInflight) + t.targetInflight.Store(minInflight) t.targetTimesTen.Store(10 * maxInflight) return t } @@ -57,7 +57,7 @@ func newThrottler(connections int, nowFunc func() time.Time) *dynamicThrottler { func NewThrottler(connections int) Throttler { return newThrottler(connections, time.Now) } func (t *dynamicThrottler) Sent() { - currentInflight := t.targetInflight.Load() + currentInflight := t.TargetInflight() t.sent++ if t.sent*t.sent*t.sent < 100*currentInflight*currentInflight { return @@ -73,8 +73,12 @@ func (t *dynamicThrottler) Sent() { t.throughputs[index] = currentThroughput // Loop over throughput measurements and pick the one which optimises throughput and latency. - choice := float64(currentInflight) + best := float64(currentInflight) maxObjective := float64(-1) + choice := 0 + j := -1 + k := -1 + s := 0.0 for i := len(t.throughputs) - 1; i >= 0; i-- { if t.throughputs[i] == 0 { continue // Skip unknown values @@ -83,10 +87,25 @@ func (t *dynamicThrottler) Sent() { objective := t.throughputs[i] * math.Pow(inflight, throttlerWeight-1) // Optimise throughput (weight), but also latency (1 - weight) if objective > maxObjective { maxObjective = objective - choice = inflight + best = inflight + choice = i } + // Additionally, smooth the throughput values, to reduce the impact of noise, and reduce jumpiness + if j != -1 { + u := t.throughputs[j] + if k != -1 { + t.throughputs[j] = (2*u + t.throughputs[i] + s) / 4 + } + s = u + } + k = j + j = i + } + target := int64((rand.Float64()*0.40+0.84)*best + rand.Float64()*4 - 1) // Random walk, skewed towards increase + // If the best inflight is at the high end of the known, we override the random walk to speed up upwards exploration + if choice == j && choice+1 < len(t.throughputs) { + target = int64(1 + float64(t.minInflight)*math.Pow(256, (float64(choice)+1.5)/float64(len(t.throughputs)))) } - target := int64((rand.Float64()*0.20 + 0.92) * choice) // Random walk, skewed towards increase t.targetInflight.Store(max(t.minInflight, min(t.maxInflight, target))) } diff --git a/client/go/internal/vespa/document/throttler_test.go b/client/go/internal/vespa/document/throttler_test.go index 03f0bc75bdc..b386e0d5105 100644 --- a/client/go/internal/vespa/document/throttler_test.go +++ b/client/go/internal/vespa/document/throttler_test.go @@ -9,14 +9,19 @@ import ( func TestThrottler(t *testing.T) { clock := &manualClock{tick: time.Second} tr := newThrottler(8, clock.now) - for i := 0; i < 100; i++ { + + if got, want := tr.TargetInflight(), int64(16); got != want { + t.Errorf("got TargetInflight() = %d, but want %d", got, want) + } + for i := 0; i < 30; i++ { tr.Sent() + tr.Success() } - if got, want := tr.TargetInflight(), int64(1024); got != want { + if got, want := tr.TargetInflight(), int64(18); got != want { t.Errorf("got TargetInflight() = %d, but want %d", got, want) } - tr.Throttled(5) - if got, want := tr.TargetInflight(), int64(128); got != want { + tr.Throttled(34) + if got, want := tr.TargetInflight(), int64(17); got != want { t.Errorf("got TargetInflight() = %d, but want %d", got, want) } } diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java index 7e0c6fe3f63..114b88f03a8 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java @@ -11,11 +11,12 @@ import com.yahoo.vespa.clustercontroller.core.RemoteClusterControllerTaskSchedul import com.yahoo.vespa.clustercontroller.core.restapiv2.ClusterControllerStateRestAPI; import com.yahoo.vespa.clustercontroller.core.status.StatusHandler; import com.yahoo.vespa.zookeeper.server.VespaZooKeeperServer; + +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Logger; /** @@ -27,9 +28,10 @@ public class ClusterController extends AbstractComponent private static final Logger log = Logger.getLogger(ClusterController.class.getName()); private final JDiscMetricWrapper metricWrapper; + private final Object monitor = new Object(); private final Map<String, FleetController> controllers = new TreeMap<>(); private final Map<String, StatusHandler.ContainerStatusPageServer> status = new TreeMap<>(); - private final AtomicInteger referents = new AtomicInteger(); + private final Map<String, Integer> referents = new HashMap<>(); private final AtomicBoolean shutdown = new AtomicBoolean(); /** @@ -44,9 +46,9 @@ public class ClusterController extends AbstractComponent } public void setOptions(FleetControllerOptions options, Metric metricImpl) throws Exception { - referents.incrementAndGet(); metricWrapper.updateMetricImplementation(metricImpl); - synchronized (controllers) { + synchronized (monitor) { + referents.merge(options.clusterName(), 1, Integer::sum); FleetController controller = controllers.get(options.clusterName()); if (controller == null) { controller = FleetController.create(options, metricWrapper); @@ -68,21 +70,34 @@ public class ClusterController extends AbstractComponent * we must also let the last configurer shut down this controller, to ensure this is shut down * before the ZK server it had injected from the configurers. */ - void countdown() { - if (referents.decrementAndGet() == 0) - shutdown(); + void countdown(String clusterName) { + synchronized (monitor) { + referents.compute(clusterName, (__, count) -> { + if (count == null) throw new IllegalStateException("trying to remove unknown cluster: " + clusterName); + if (count == 1) { + shutDownController(controllers.remove(clusterName)); + status.remove(clusterName); + return null; + } + return count - 1; + }); + } + } + + private void shutDownController(FleetController controller) { + if (controller == null) return; + try { + controller.shutdown(); + } catch (Exception e) { + log.warning("Failed to shut down fleet controller: " + e.getMessage()); + } } void shutdown() { if (shutdown.compareAndSet(false, true)) { - synchronized (controllers) { + synchronized (monitor) { for (FleetController controller : controllers.values()) { - try { - shutdownController(controller); - } - catch (Exception e) { - log.warning("Failed to shut down fleet controller: " + e.getMessage()); - } + shutDownController(controller); } } } @@ -90,7 +105,7 @@ public class ClusterController extends AbstractComponent @Override public Map<String, RemoteClusterControllerTaskScheduler> getFleetControllers() { - synchronized (controllers) { + synchronized (monitor) { return new LinkedHashMap<>(controllers); } } @@ -105,8 +120,4 @@ public class ClusterController extends AbstractComponent return status; } - void shutdownController(FleetController controller) throws Exception { - controller.shutdown(); - } - } diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java index 5a2034f0372..265a99e2f72 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java @@ -45,7 +45,7 @@ public class ClusterControllerClusterConfigurer extends AbstractComponent { @Override public void deconstruct() { - if (controller != null) controller.countdown(); + if (controller != null) controller.countdown(options.clusterName()); } FleetControllerOptions getOptions() { return options; } diff --git a/config-model/src/main/java/com/yahoo/schema/document/SDField.java b/config-model/src/main/java/com/yahoo/schema/document/SDField.java index f165141b16e..2483fa47667 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/SDField.java @@ -46,7 +46,7 @@ import java.util.TreeMap; * * @author bratseth */ -public class SDField extends Field implements TypedKey, ImmutableSDField { +public class SDField extends Field implements ImmutableSDField { /** Use this field for modifying index-structure, even if it doesn't have any indexing code */ private boolean indexStructureField = false; @@ -315,7 +315,7 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { supplyStructField.accept(field.getName(), field.getDataType()); } } - if ((subType == null) && (structFields.size() > 0)) { + if ((subType == null) && (!structFields.isEmpty())) { throw new IllegalArgumentException("Cannot find matching (repo=" + sdoc + ") for subfields in " + this + " [" + getDataType() + getDataType().getClass() + "] with " + structFields.size() + " struct fields"); @@ -627,7 +627,7 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { public Attribute addAttribute(Attribute attribute) { String name = attribute.getName(); - if (name == null || "".equals(name)) { + if (name == null || name.isEmpty()) { name = getName(); attribute.setName(name); } diff --git a/config-model/src/main/java/com/yahoo/schema/document/TypedKey.java b/config-model/src/main/java/com/yahoo/schema/document/TypedKey.java deleted file mode 100644 index 652d21d7f7d..00000000000 --- a/config-model/src/main/java/com/yahoo/schema/document/TypedKey.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.schema.document; - -import com.yahoo.document.DataType; - -/** - * Common interface for various typed key (or field definitions). - * Used by code which wants to use common algorithms for dealing with typed keys, like the logical mapping - * - * @author bratseth - */ -public interface TypedKey { - - String getName(); - - void setDataType(DataType type); - - DataType getDataType(); - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java index d50d5e36134..785b45d8def 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java @@ -3,10 +3,12 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.document.DataType; import com.yahoo.document.Field; -import com.yahoo.schema.document.TypedKey; import java.io.Serializable; -import java.util.*; +import java.util.Collection; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Set; import java.util.stream.Collectors; import static com.yahoo.text.Lowercase.toLowerCase; @@ -16,7 +18,7 @@ import static com.yahoo.text.Lowercase.toLowerCase; * * @author bratseth */ -public class SummaryField extends Field implements Cloneable, TypedKey { +public class SummaryField extends Field implements Cloneable { /** A source (field name). */ public static class Source implements Serializable { @@ -62,7 +64,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey { */ private Set<Source> sources = new java.util.LinkedHashSet<>(); - private Set<String> destinations =new java.util.LinkedHashSet<>(); + private Set<String> destinations = new java.util.LinkedHashSet<>(); /** True if this field was defined implicitly */ private boolean implicit = false; diff --git a/configdefinitions/src/vespa/CMakeLists.txt b/configdefinitions/src/vespa/CMakeLists.txt index 81e587fcace..0ab12932880 100644 --- a/configdefinitions/src/vespa/CMakeLists.txt +++ b/configdefinitions/src/vespa/CMakeLists.txt @@ -6,8 +6,6 @@ vespa_add_library(configdefinitions ) vespa_generate_config(configdefinitions application-id.def) install_config_definition(application-id.def cloud.config.application-id.def) -vespa_generate_config(configdefinitions athenz-provider-service.def) -install_config_definition(athenz-provider-service.def vespa.hosted.athenz.instanceproviderservice.config.athenz-provider-service.def) vespa_generate_config(configdefinitions attributes.def) install_config_definition(attributes.def vespa.config.search.attributes.def) vespa_generate_config(configdefinitions cluster-info.def) diff --git a/configdefinitions/src/vespa/athenz-provider-service.def b/configdefinitions/src/vespa/athenz-provider-service.def deleted file mode 100644 index 5ee9be323e8..00000000000 --- a/configdefinitions/src/vespa/athenz-provider-service.def +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.hosted.athenz.instanceproviderservice.config - -# Athenz domain -domain string - -# Athenz service name -serviceName string - -# Secret name of private Key -secretName string - -# Secret version -secretVersion int - -# Tempory resources -sisSecretName string default="" -sisSecretVersion int default=0 -sisUrl string default = "" - -# Secret name of CA certificate -caCertSecretName string - -# Certificate DNS suffix -certDnsSuffix string default="" - -# Athenz ZTS server url -ztsUrl string default="" - -# Path to Athenz CA JKS trust store -athenzCaTrustStore string default="" - -# Period between certificate updates -updatePeriodDays int default=1 - -# Tenant Service id -tenantService string default=vespa.vespa.tenant diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index fb6516ce6cf..d71840a95c2 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -45,13 +45,11 @@ </components> <preprocess:include file='config-models.xml' required='false' /> - <preprocess:include file='routing-status.xml' required='false' /> <preprocess:include file='model-integration.xml' required='true' /> <component id="com.yahoo.vespa.configserver.flags.ConfigServerFlagSource" bundle="configserver-flags"/> <component id="com.yahoo.vespa.configserver.flags.db.FlagsDbImpl" bundle="configserver-flags"/> - <preprocess:include file='metrics-packets.xml' required='false' /> <component id="com.yahoo.vespa.service.slobrok.SlobrokMonitorManagerImpl" bundle="service-monitor" /> <component id="com.yahoo.vespa.service.health.HealthMonitorManager" bundle="service-monitor" /> <component id="com.yahoo.vespa.service.manager.UnionMonitorManager" bundle="service-monitor" /> @@ -153,13 +151,9 @@ <preprocess:include file='http-server.xml' required='false' /> </http> - <preprocess:include file='athenz-identity-provider.xml' required='false' /> - <preprocess:include file='configserver-config.xml' required='false' /> <preprocess:include file='configserver-components.xml' required='false' /> - <preprocess:include file='zookeeper-server-config.xml' required='false' /> - </container> </services> diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java index ea0cd2312a6..d3e6241a6e5 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java @@ -134,10 +134,8 @@ abstract class SimpleParser extends StructuredParser { if (topLevelItem != null && topLevelItem != not) { // => neutral rank items becomes implicit positives not.addPositiveItem(getItemAsPositiveItem(topLevelItem, not)); - return not; - } else { - return not; } + return not; } if (topLevelItem != null) { return topLevelItem; diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java index ffa6c82e941..611df6ad284 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -1532,7 +1532,7 @@ public class JsonRendererTestCase { + "}"; assertEquals( "Unexpected character ('a' (code 97)): was expecting comma to separate Object entries\n" + - " at [Source: (String)\"{ \"root\": { \"invalidvalue\": 1adsf, }}\"; line: 1, column: 41]", + " at [Source: (String)\"{ \"root\": { \"invalidvalue\": 1adsf, }}\"; line: 1, column: 40]", validateJSON(json)); } diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index f6dea40f72e..02c3f663f98 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -37,7 +37,7 @@ <guava.vespa.version>33.2.0-jre</guava.vespa.version> <guice.vespa.version>6.0.0</guice.vespa.version> <j2objc-annotations.vespa.version>3.0.0</j2objc-annotations.vespa.version> - <jackson2.vespa.version>2.16.2</jackson2.vespa.version> + <jackson2.vespa.version>2.17.1</jackson2.vespa.version> <jackson-databind.vespa.version>${jackson2.vespa.version}</jackson-databind.vespa.version> <jakarta.inject.vespa.version>2.0.1</jakarta.inject.vespa.version> <javax.activation-api.vespa.version>1.2.0</javax.activation-api.vespa.version> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 7b1ac3f0f90..a577bbe74df 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -53,7 +53,8 @@ public class Flags { List.of("hakonhall"), "2024-05-06", "2024-07-06", "Whether to provision new GCP VM instances with a service account that are independent " + "of the zone, and aligned with the Athenz service names (configserver and tenant-host).", - "Takes effect when provisioning new VM instances"); + "Takes effect when provisioning new VM instances", + APPLICATION, INSTANCE_ID); public static final UnboundDoubleFlag DEFAULT_TERM_WISE_LIMIT = defineDoubleFlag( "default-term-wise-limit", 1.0, diff --git a/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java b/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java index 97f681404e9..1a42b688437 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java @@ -167,10 +167,10 @@ public class DynamicThrottlePolicy extends StaticThrottlePolicy { } /** - * Determines where on each latency level the attractor sits. 2 is at the very end, and makes this to *boom*. + * Determines where on each latency level the attractor sits. 2 is at the very end, and makes this go *boom*. * 0.2 is at the very start, and makes the algorithm more conservative. Probably fine to stay away from this. */ - // Original javadoc is non-sense, but kept for historical reasons. + // Original javadoc is nonsense, but kept for historical reasons. /* * Sets the lower efficiency threshold at which the algorithm should perform window size back off. Efficiency is * the correlation between throughput and window size. The algorithm will increase the window size until efficiency diff --git a/metrics/src/tests/CMakeLists.txt b/metrics/src/tests/CMakeLists.txt index 043dd7f736d..779b799cc75 100644 --- a/metrics/src/tests/CMakeLists.txt +++ b/metrics/src/tests/CMakeLists.txt @@ -9,7 +9,6 @@ vespa_add_executable(metrics_gtest_runner_app TEST metricsettest.cpp metrictest.cpp snapshottest.cpp - stresstest.cpp summetrictest.cpp valuemetrictest.cpp gtest_runner.cpp diff --git a/metrics/src/tests/stresstest.cpp b/metrics/src/tests/stresstest.cpp deleted file mode 100644 index a5213ba8b2d..00000000000 --- a/metrics/src/tests/stresstest.cpp +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/metrics/metricmanager.h> -#include <vespa/metrics/metrics.h> -#include <vespa/metrics/summetric.hpp> -#include <vespa/vespalib/util/time.h> -#include <vespa/vespalib/util/size_literals.h> -#include <thread> -#include <vespa/vespalib/gtest/gtest.h> - -#include <vespa/log/log.h> -LOG_SETUP(".metrics.test.stress"); - -namespace metrics { - -namespace { -struct InnerMetricSet : public MetricSet { - LongCountMetric _count; - LongAverageMetric _value1; - LongAverageMetric _value2; - SumMetric<LongAverageMetric> _valueSum; - - InnerMetricSet(const char* name, MetricSet* owner = 0); - ~InnerMetricSet(); - - MetricSet* clone(std::vector<Metric::UP> &ownerList, CopyType copyType, - MetricSet* owner, bool includeUnused) const override; -}; - -InnerMetricSet::InnerMetricSet(const char* name, MetricSet* owner) - : MetricSet(name, {}, "", owner), - _count("count", {}, "", this), - _value1("value1", {}, "", this), - _value2("value2", {}, "", this), - _valueSum("valuesum", {}, "", this) -{ - _valueSum.addMetricToSum(_value1); - _valueSum.addMetricToSum(_value2); -} -InnerMetricSet::~InnerMetricSet() = default; - -MetricSet* -InnerMetricSet::clone(std::vector<Metric::UP> &ownerList, CopyType copyType, - MetricSet* owner, bool includeUnused) const -{ - if (copyType != CLONE) { - return MetricSet::clone(ownerList, copyType, owner, includeUnused); -} - InnerMetricSet * myset = new InnerMetricSet(getName().c_str(), owner); - myset->assignValues(*this); - return myset; -} - -struct OuterMetricSet : public MetricSet { - InnerMetricSet _inner1; - InnerMetricSet _inner2; - SumMetric<InnerMetricSet> _innerSum; - InnerMetricSet _tmp; - - OuterMetricSet(MetricSet* owner = 0); - ~OuterMetricSet(); -}; - -OuterMetricSet::OuterMetricSet(MetricSet* owner) - : MetricSet("outer", {}, "", owner), - _inner1("inner1", this), - _inner2("inner2", this), - _innerSum("innersum", {}, "", this), - _tmp("innertmp", 0) -{ - _innerSum.addMetricToSum(_inner1); - _innerSum.addMetricToSum(_inner2); -} - -OuterMetricSet::~OuterMetricSet() = default; - -struct Hammer { - using UP = std::unique_ptr<Hammer>; - - OuterMetricSet& _metrics; - std::atomic<bool> _stop_requested; - std::thread _thread; - - Hammer(OuterMetricSet& metrics) - : _metrics(metrics), - _stop_requested(false), - _thread() - { - _thread = std::thread([this](){run();}); - } - ~Hammer() { - _stop_requested = true; - _thread.join(); - //std::cerr << "Loadgiver thread joined\n"; - } - - void run() { - uint64_t i = 0; - while (!_stop_requested.load(std::memory_order_relaxed)) { - ++i; - setMetrics(i, _metrics._inner1); - setMetrics(i + 3, _metrics._inner2); - } - } - - void setMetrics(uint64_t val, InnerMetricSet& set) { - set._count.inc(val); - set._value1.addValue(val); - set._value2.addValue(val + 10); - } -}; - -} - - -TEST(StressTest, test_stress) -{ - OuterMetricSet metrics; - - LOG(info, "Starting load givers"); - std::vector<Hammer::UP> hammers; - for (uint32_t i=0; i<10; ++i) { - hammers.push_back(std::make_unique<Hammer>(metrics)); - } - LOG(info, "Waiting to let loadgivers hammer a while"); - std::this_thread::sleep_for(5s); - - LOG(info, "Removing loadgivers"); - hammers.clear(); - - LOG(info, "Printing end state"); - std::ostringstream ost; - metrics.print(ost, true, "", 5); - // std::cerr << ost.str() << "\n"; -} - -} diff --git a/searchlib/src/vespa/searchlib/fef/test/matchdatabuilder.cpp b/searchlib/src/vespa/searchlib/fef/test/matchdatabuilder.cpp index 3645496e4fb..41551ac1062 100644 --- a/searchlib/src/vespa/searchlib/fef/test/matchdatabuilder.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/matchdatabuilder.cpp @@ -10,11 +10,11 @@ LOG_SETUP(".fef.matchdatabuilder"); namespace search::fef::test { -MatchDataBuilder::MatchDataBuilder(QueryEnvironment &queryEnv, MatchData &data) : - _queryEnv(queryEnv), - _data(data), - _index(), - _match() +MatchDataBuilder::MatchDataBuilder(QueryEnvironment &queryEnv, MatchData &data) + : _queryEnv(queryEnv), + _data(data), + _index(), + _match() { // reset all match data objects. for (TermFieldHandle handle = 0; handle < _data.getNumTermFields(); ++handle) { @@ -22,7 +22,7 @@ MatchDataBuilder::MatchDataBuilder(QueryEnvironment &queryEnv, MatchData &data) } } -MatchDataBuilder::~MatchDataBuilder() {} +MatchDataBuilder::~MatchDataBuilder() = default; TermFieldMatchData * MatchDataBuilder::getTermFieldMatchData(uint32_t termId, uint32_t fieldId) @@ -59,7 +59,7 @@ MatchDataBuilder::addElement(const vespalib::string &fieldName, int32_t weight, LOG(error, "Field '%s' does not exist.", fieldName.c_str()); return false; } - _index[info->id()].elements.push_back(MyElement(weight, length)); + _index[info->id()].elements.emplace_back(weight, length); return true; } @@ -77,8 +77,7 @@ MatchDataBuilder::addOccurence(const vespalib::string &fieldName, uint32_t termI } const ITermFieldData *tfd = _queryEnv.getTerm(termId)->lookupField(info->id()); if (tfd == nullptr) { - LOG(error, "Field '%s' is not searched by the given term.", - fieldName.c_str()); + LOG(error, "Field '%s' is not searched by the given term.", fieldName.c_str()); return false; } _match[termId][info->id()].insert(Position(pos, element)); @@ -99,14 +98,13 @@ MatchDataBuilder::setWeight(const vespalib::string &fieldName, uint32_t termId, } const ITermFieldData *tfd = _queryEnv.getTerm(termId)->lookupField(info->id()); if (tfd == nullptr) { - LOG(error, "Field '%s' is not searched by the given term.", - fieldName.c_str()); + LOG(error, "Field '%s' is not searched by the given term.", fieldName.c_str()); return false; } uint32_t eid = _index[info->id()].elements.size(); _match[termId][info->id()].clear(); _match[termId][info->id()].insert(Position(0, eid)); - _index[info->id()].elements.push_back(MyElement(weight, 1)); + _index[info->id()].elements.emplace_back(weight, 1); return true; } @@ -142,19 +140,13 @@ MatchDataBuilder::apply(uint32_t docId) // For each occurence of that term, in that field, do for (const auto& occ : field_elem.second) { // Append a term match position to the term match data. - match->appendPosition(TermFieldMatchDataPosition( - occ.eid, - occ.pos, - field.getWeight(occ.eid), - field.getLength(occ.eid))); - LOG(debug, - "Added occurence of term '%u' in field '%s'" - " at position '%u'.", + match->appendPosition(TermFieldMatchDataPosition(occ.eid, occ.pos, + field.getWeight(occ.eid), + field.getLength(occ.eid))); + LOG(debug, "Added occurence of term '%u' in field '%s' at position '%u'.", termId, name.c_str(), occ.pos); if (occ.pos >= field.getLength(occ.eid)) { - LOG(warning, - "Added occurence of term '%u' in field '%s'" - " at position '%u' >= fieldLen '%u'.", + LOG(warning, "Added occurence of term '%u' in field '%s' at position '%u' >= fieldLen '%u'.", termId, name.c_str(), occ.pos, field.getLength(occ.eid)); } } diff --git a/searchlib/src/vespa/searchlib/fef/test/matchdatabuilder.h b/searchlib/src/vespa/searchlib/fef/test/matchdatabuilder.h index 0e5025efd37..753e1596520 100644 --- a/searchlib/src/vespa/searchlib/fef/test/matchdatabuilder.h +++ b/searchlib/src/vespa/searchlib/fef/test/matchdatabuilder.h @@ -13,7 +13,7 @@ public: struct MyElement { int32_t weight; uint32_t length; - MyElement(int32_t w, uint32_t l) : weight(w), length(l) {} + MyElement(int32_t w, uint32_t l) noexcept : weight(w), length(l) {} }; struct MyField { uint32_t fieldLength; @@ -21,7 +21,7 @@ public: MyField() : fieldLength(0), elements() {} MyElement &getElement(uint32_t eid) { while (elements.size() <= eid) { - elements.push_back(MyElement(0, 0)); + elements.emplace_back(0, 0); } return elements[eid]; } @@ -68,6 +68,8 @@ public: * @param data The match data to build in. */ MatchDataBuilder(QueryEnvironment &queryEnv, MatchData &data); + MatchDataBuilder(const MatchDataBuilder &) = delete; + MatchDataBuilder & operator=(const MatchDataBuilder &) = delete; ~MatchDataBuilder(); /** @@ -133,10 +135,6 @@ public: bool apply(uint32_t docId); private: - MatchDataBuilder(const MatchDataBuilder &); // hide - MatchDataBuilder & operator=(const MatchDataBuilder &); // hide - -private: QueryEnvironment &_queryEnv; MatchData &_data; IndexData _index; diff --git a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp index 3ab3a1123eb..441ade27d1f 100644 --- a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp +++ b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp @@ -45,10 +45,9 @@ public: return score; } double calc_with_limit(TypedCells rhs, double limit) const noexcept override { - vespalib::ConstArrayRef<AttributeCellType> rhs_vector = rhs.typify<AttributeCellType>(); + vespalib::ConstArrayRef<AttributeCellType> rhs_vector = rhs.unsafe_typify<AttributeCellType>(); double sum = 0.0; size_t sz = _lhs_vector.size(); - assert(sz == rhs_vector.size()); for (size_t i = 0; i < sz && sum <= limit; ++i) { double diff = _lhs_vector[i] - rhs_vector[i]; sum += diff*diff; diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/DynamicThrottler.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/DynamicThrottler.java index 951a1776b6f..567788b8501 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/DynamicThrottler.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/DynamicThrottler.java @@ -28,12 +28,12 @@ public class DynamicThrottler extends StaticThrottler { public DynamicThrottler(FeedClientBuilderImpl builder) { super(builder); - targetInflight = new AtomicLong(8 * minInflight); + targetInflight = new AtomicLong(minInflight); } @Override public void sent(long __, CompletableFuture<HttpResponse> ___) { - double currentInflight = targetInflight.get(); + double currentInflight = targetInflight(); if (++sent * sent * sent < 1e2 * currentInflight * currentInflight) return; @@ -43,22 +43,36 @@ public class DynamicThrottler extends StaticThrottler { // Use buckets for throughput over inflight, along the log-scale, in [minInflight, maxInflight). int index = (int) (throughputs.length * log(max(1, min(255, currentInflight / minInflight))) - / log(256)); // 4096 (server max streams per connection) / 16 (our min per connection) + / log(256)); // 512 (server max streams per connection) / 2 (our min per connection) throughputs[index] = currentThroughput; // Loop over throughput measurements and pick the one which optimises throughput and latency. - double choice = currentInflight; + double best = currentInflight; double max = -1; - for (int i = throughputs.length; i-- > 0; ) { + int j = -1, k = -1, choice = 0; + double s = 0; + for (int i = 0; i < throughputs.length; i++) { if (throughputs[i] == 0) continue; // Skip unknown values. double inflight = minInflight * pow(256, (i + 0.5) / throughputs.length); double objective = throughputs[i] * pow(inflight, (weight - 1)); // Optimise throughput (weight), but also latency (1 - weight). if (objective > max) { max = objective; - choice = inflight; + best = inflight; + choice = i; } + // Additionally, smooth the throughput values, to reduce the impact of noise, and reduce jumpiness. + if (j != -1) { + double t = throughputs[j]; + if (k != -1) throughputs[j] = (2 * t + throughputs[i] + s) / 4; + s = t; + } + k = j; + j = i; } - long target = (long) ((random() * 0.20 + 0.92) * choice); // Random walk, skewed towards increase. + long target = (long) ((random() * 0.40 + 0.84) * best + random() * 4 - 1); // Random step, skewed towards increase. + // If the best inflight is at the high end of the known, we override the random walk to speed up upwards exploration. + if (choice == j && choice + 1 < throughputs.length) + target = (long) (1 + minInflight * pow(256, (choice + 1.5) / throughputs.length)); targetInflight.set(max(minInflight, min(maxInflight, target))); } diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/StaticThrottler.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/StaticThrottler.java index 9010b0a7ad8..f0ee434e87c 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/StaticThrottler.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/StaticThrottler.java @@ -22,7 +22,7 @@ public class StaticThrottler implements Throttler { public StaticThrottler(FeedClientBuilderImpl builder) { minInflight = 2L * builder.connectionsPerEndpoint * builder.endpoints.size(); - maxInflight = 256 * minInflight; // 4096 max streams per connection on the server side. + maxInflight = 256 * minInflight; // 512 max streams per connection on the server side. targetX10 = new AtomicLong(10 * maxInflight); // 10x the actual value to allow for smaller updates. } diff --git a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/DynamicThrottlerTest.java b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/DynamicThrottlerTest.java new file mode 100644 index 00000000000..7e07fc6e116 --- /dev/null +++ b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/DynamicThrottlerTest.java @@ -0,0 +1,30 @@ +package ai.vespa.feed.client.impl; + +import org.junit.jupiter.api.Test; + +import java.net.URI; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author jonmv + */ +class DynamicThrottlerTest { + + @Test + void testThrottler() { + DynamicThrottler throttler = new DynamicThrottler(new FeedClientBuilderImpl(List.of(URI.create("http://localhost:8080")))); + assertEquals(16, throttler.targetInflight()); + + for (int i = 0; i < 30; i++) { + throttler.sent(1, null); + throttler.success(); + } + assertEquals(18, throttler.targetInflight()); + + throttler.throttled(34); + assertEquals(17, throttler.targetInflight()); + } + +} diff --git a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpRequestStrategyTest.java b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpRequestStrategyTest.java index 54fab9b859b..b1a04ac9ed4 100644 --- a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpRequestStrategyTest.java +++ b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpRequestStrategyTest.java @@ -33,6 +33,7 @@ import static ai.vespa.feed.client.FeedClient.CircuitBreaker.State.OPEN; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -105,7 +106,7 @@ class HttpRequestStrategyTest { cluster.expect((__, vessel) -> vessel.completeExceptionally(new RuntimeException("boom"))); ExecutionException expected = assertThrows(ExecutionException.class, () -> strategy.enqueue(id1, request).get()); - assertTrue(expected.getCause() instanceof FeedException); + assertInstanceOf(FeedException.class, expected.getCause()); assertEquals("java.lang.RuntimeException: boom", expected.getCause().getMessage()); assertEquals(1, strategy.stats().requests()); @@ -200,7 +201,7 @@ class HttpRequestStrategyTest { @Override public int retries() { return 1; } }) .setCircuitBreaker(breaker) - .setConnectionsPerEndpoint(1), + .setConnectionsPerEndpoint(3), // Must be >= 0.5x text ops. cluster); DocumentId id1 = DocumentId.of("ns", "type", "1"); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java index b483d6977d6..6e07661235e 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java @@ -1397,6 +1397,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { Phaser phaser = new Phaser(2); // Synchronize this thread (dispatch) with the visitor callback thread. AtomicReference<String> error = new AtomicReference<>(); // Set if error occurs during processing of visited documents. callback.onStart(response, fullyApplied); + final AtomicLong locallyReceivedDocCount = new AtomicLong(0); VisitorControlHandler controller = new VisitorControlHandler() { final ScheduledFuture<?> abort = streaming ? visitDispatcher.schedule(this::abort, visitTimeout(request), MILLISECONDS) : null; final AtomicReference<VisitorSession> session = new AtomicReference<>(); @@ -1410,7 +1411,10 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { try (response) { callback.onEnd(response); - response.writeDocumentCount(getVisitorStatistics() == null ? 0 : getVisitorStatistics().getDocumentsVisited()); + // Locally tracked document count is only correct if we have a local data handler. + // Otherwise, we have to report the statistics received transitively from the content nodes. + long statsDocCount = (getVisitorStatistics() != null ? getVisitorStatistics().getDocumentsVisited() : 0); + response.writeDocumentCount(parameters.getLocalDataHandler() != null ? locallyReceivedDocCount.get() : statsDocCount); if (session.get() != null) response.writeTrace(session.get().getTrace()); @@ -1456,6 +1460,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { if (m instanceof PutDocumentMessage put) document = put.getDocumentPut().getDocument(); else if (parameters.visitRemoves() && m instanceof RemoveDocumentMessage remove) removeId = remove.getDocumentId(); else throw new UnsupportedOperationException("Got unsupported message type: " + m.getClass().getName()); + locallyReceivedDocCount.getAndAdd(1); callback.onDocument(response, document, removeId, diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java index 58cf34712aa..b2c0b1b2ce8 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java @@ -290,7 +290,7 @@ public class DocumentV1ApiTest { parameters.getLocalDataHandler().onMessage(new RemoveDocumentMessage(new DocumentId("id:space:music::t-square-truth")), tokens.get(3)); VisitorStatistics statistics = new VisitorStatistics(); statistics.setBucketsVisited(1); - statistics.setDocumentsVisited(3); + statistics.setDocumentsVisited(123); // Ignored in favor of tracking actually emitted entries parameters.getControlHandler().onVisitorStatistics(statistics); parameters.getControlHandler().onDone(VisitorControlHandler.CompletionCode.TIMEOUT, "timeout is OK"); }); @@ -323,7 +323,7 @@ public class DocumentV1ApiTest { "remove": "id:space:music::t-square-truth" } ], - "documentCount": 3, + "documentCount": 4, "trace": [ { "message": "Tracy Chapman" }, { @@ -441,13 +441,18 @@ public class DocumentV1ApiTest { assertEquals("[Content:cluster=content]", parameters.getRemoteDataHandler()); assertEquals("[document]", parameters.fieldSet()); assertEquals(60_000L, parameters.getSessionTimeoutMs()); + VisitorStatistics statistics = new VisitorStatistics(); + statistics.setBucketsVisited(1); + statistics.setDocumentsVisited(2); + // Visiting with remote data handlers should report the remotely aggregated statistics + parameters.getControlHandler().onVisitorStatistics(statistics); parameters.getControlHandler().onDone(VisitorControlHandler.CompletionCode.SUCCESS, "We made it!"); }); response = driver.sendRequest("http://localhost/document/v1/space/music/docid?destinationCluster=content&selection=true&cluster=content&timeout=60", POST); assertSameJson(""" { "pathId": "/document/v1/space/music/docid", - "documentCount": 0 + "documentCount": 2 }""", response.readAll()); assertEquals(200, response.getStatus()); @@ -488,7 +493,7 @@ public class DocumentV1ApiTest { assertSameJson(""" { "pathId": "/document/v1/space/music/docid", - "documentCount": 0 + "documentCount": 1 }""", response.readAll()); assertEquals(200, response.getStatus()); @@ -542,7 +547,7 @@ public class DocumentV1ApiTest { assertSameJson(""" { "pathId": "/document/v1/space/music/docid", - "documentCount": 0, + "documentCount": 1, "message": "boom" }""", response.readAll()); |