diff options
263 files changed, 3565 insertions, 1384 deletions
diff --git a/client/src/main/java/ai/vespa/client/dsl/EndQuery.java b/client/src/main/java/ai/vespa/client/dsl/EndQuery.java index 4facf6ce0fb..2af1e0bb49d 100644 --- a/client/src/main/java/ai/vespa/client/dsl/EndQuery.java +++ b/client/src/main/java/ai/vespa/client/dsl/EndQuery.java @@ -177,7 +177,7 @@ public class EndQuery { } else if (others.isEmpty()) { sb.append("order by ").append(orderStr); } else { - sb.append("order by ").append(orderStr).append(", ").append(others); + sb.append("order by ").append(orderStr).append(" ").append(others); } if (groupQueryStr != null) { diff --git a/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy b/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy index e363a9bcc13..19c87d6aecd 100644 --- a/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy +++ b/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy @@ -59,7 +59,7 @@ class QTest extends Specification { .build() expect: - q == """yql=select * from sd1 where f1 contains "v1" and f2 contains "v2" or f3 contains "v3" and !(f4 contains "v4") order by f1 desc, f2 asc, limit 2 offset 1 timeout 3;¶mk1=paramv1""" + q == """yql=select * from sd1 where f1 contains "v1" and f2 contains "v2" or f3 contains "v3" and !(f4 contains "v4") order by f1 desc, f2 asc limit 2 offset 1 timeout 3;¶mk1=paramv1""" } def "matches"() { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index 4fe6c3a96f2..30ce142d503 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -91,7 +91,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, private NormalizeLevel normalizing = new NormalizeLevel(); /** Extra query commands of this field */ - private List<String> queryCommands=new java.util.ArrayList<>(0); + private List<String> queryCommands = new java.util.ArrayList<>(0); /** Summary fields defined in this field */ private Map<String, SummaryField> summaryFields = new java.util.LinkedHashMap<>(0); @@ -749,20 +749,11 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, return queryCommands.contains(name); } - /** - * A list of query commands - * - * @return a list of strings with query commands. - */ + /** Returns a list of query commands */ @Override - public List<String> getQueryCommands() { - return queryCommands; - } + public List<String> getQueryCommands() { return queryCommands; } - /** - * The document that this field was declared in, or null - * - */ + /** Returns the document that this field was declared in, or null */ private SDDocumentType getOwnerDocType() { return ownerDocType; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java index a0d47d7fa81..0a29fae04bf 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java @@ -4,9 +4,10 @@ package com.yahoo.searchdefinition.fieldoperation; import com.yahoo.searchdefinition.document.SDField; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class IndexingRewriteOperation implements FieldOperation { - public void apply(SDField field) { - } + + public void apply(SDField field) { } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index f81757ac568..b30cf9248a1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -20,7 +20,9 @@ import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; import ai.vespa.metricsproxy.rpc.RpcServer; import ai.vespa.metricsproxy.service.ConfigSentinelClient; import ai.vespa.metricsproxy.service.SystemPollerProvider; +import ai.vespa.metricsproxy.telegraf.Telegraf; import ai.vespa.metricsproxy.telegraf.TelegrafConfig; +import ai.vespa.metricsproxy.telegraf.TelegrafRegistry; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; @@ -118,6 +120,8 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC addHttpHandler(ApplicationMetricsHandler.class, ApplicationMetricsHandler.V1_PATH); addMetricsProxyComponent(ApplicationMetricsRetriever.class); + + addTelegrafComponents(); } private void addHttpHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) { @@ -133,6 +137,15 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC return metricsHandler; } + private void addTelegrafComponents() { + getAdmin().ifPresent(admin -> { + if (admin.getUserMetrics().usesExternalMetricSystems()) { + addMetricsProxyComponent(Telegraf.class); + addMetricsProxyComponent(TelegrafRegistry.class); + } + }); + } + @Override protected void doPrepare(DeployState deployState) { } @@ -224,7 +237,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC Optional.of(monitoring.getInterval()) : Optional.empty(); } - private void addMetricsProxyComponent(Class<?> componentClass) { + private void addMetricsProxyComponent(Class<?> componentClass) { addSimpleComponent(componentClass.getName(), null, METRICS_PROXY_BUNDLE_NAME); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java index 9c752f3aa0d..a8fbcf50b02 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java @@ -17,14 +17,15 @@ import static java.util.Collections.unmodifiableList; */ @Immutable public class MetricsConsumer { + private final String id; private final MetricSet metricSet; private final List<CloudWatch> cloudWatches = new ArrayList<>(); /** - * @param id The consumer - * @param metricSet The metrics for this consumer + * @param id the consumer + * @param metricSet the metrics for this consumer */ public MetricsConsumer(String id, MetricSet metricSet) { this.id = Objects.requireNonNull(id, "A consumer must have a non-null id.");; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java index 58b77ee1297..c05cad89852 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java @@ -9,6 +9,7 @@ import java.util.Set; * @author gjoranv */ public class SystemMetrics { + public static final String CPU_UTIL = "cpu.util"; public static final String CPU_SYS_UTIL = "cpu.sys.util"; public static final String CPU_THROTTLED_TIME = "cpu.throttled_time.rate"; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java index 9b0d9dbfadc..1f81f16a80b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java @@ -28,4 +28,13 @@ public class Metrics { return consumers.keySet().stream() .anyMatch(existing -> existing.equalsIgnoreCase(id)); } + + /** + * Returns true if any of the consumers have specified external metric systems. + */ + public boolean usesExternalMetricSystems() { + return consumers.values().stream() + .anyMatch(consumer -> ! consumer.cloudWatches().isEmpty()); + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java index 6b1a94e16ae..67c7b67ad9e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.http; -import com.google.common.collect.ImmutableList; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.config.application.api.DeployLogger; @@ -30,7 +29,7 @@ public final class AccessControl { public static final ComponentId ACCESS_CONTROL_CHAIN_ID = ComponentId.fromString("access-control-chain"); - private static final List<String> UNPROTECTED_HANDLERS = ImmutableList.of( + public static final List<String> UNPROTECTED_HANDLERS = List.of( FileStatusHandlerComponent.CLASS, ContainerCluster.APPLICATION_STATUS_HANDLER_CLASS, ContainerCluster.BINDINGS_OVERVIEW_HANDLER_CLASS, diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc index 055f57dd7c0..e3ba7dc500d 100644 --- a/config-model/src/main/resources/schema/admin.rnc +++ b/config-model/src/main/resources/schema/admin.rnc @@ -89,7 +89,7 @@ Metrics = element metrics { Cloudwatch = element cloudwatch { attribute region { xsd:Name } & - attribute namespace { xsd:Name } & + attribute namespace { xsd:string { pattern = "[\w_\-/#:\.]+" } } & ( ( element access-key-name { xsd:Name } & diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java index 144c45a7dd2..dbcbad7bf49 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java @@ -1,13 +1,19 @@ package com.yahoo.vespa.model.admin.metricsproxy; +import ai.vespa.metricsproxy.telegraf.Telegraf; import ai.vespa.metricsproxy.telegraf.TelegrafConfig; +import ai.vespa.metricsproxy.telegraf.TelegrafRegistry; +import com.yahoo.component.ComponentId; import com.yahoo.vespa.model.VespaModel; import org.junit.Test; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CLUSTER_CONFIG_ID; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; +import static org.hamcrest.CoreMatchers.hasItem; +import static org.hamcrest.CoreMatchers.not; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; /** * @author gjoranv @@ -15,25 +21,39 @@ import static org.junit.Assert.assertEquals; public class TelegrafTest { @Test - public void telegraf_config_is_generated_for_cloudwatch_in_services() { + public void telegraf_components_are_set_up_when_cloudwatch_is_configured() { + String services = servicesWithCloudwatch(); + VespaModel hostedModel = getModel(services, hosted); + + var clusterComponents = hostedModel.getAdmin().getMetricsProxyCluster().getComponentsMap(); + assertThat(clusterComponents.keySet(), hasItem(ComponentId.fromString(Telegraf.class.getName()))); + assertThat(clusterComponents.keySet(), hasItem(ComponentId.fromString(TelegrafRegistry.class.getName()))); + } + + @Test + public void telegraf_components_are_not_set_up_when_no_external_systems_are_added_in_services() { String services = String.join("\n", "<services>", " <admin version='2.0'>", " <adminserver hostalias='node1'/>", " <metrics>", - " <consumer id='cloudwatch-consumer'>", - " <metric id='my-metric'/>", - " <cloudwatch region='us-east-1' namespace='my-namespace' >", - " <access-key-name>my-access-key</access-key-name>", - " <secret-key-name>my-secret-key</secret-key-name>", - " </cloudwatch>", - " </consumer>", + " <consumer id='foo' />", " </metrics>", " </admin>", - "</services>" - ); + "</services>"); + VespaModel hostedModel = getModel(services, hosted); + + var clusterComponents = hostedModel.getAdmin().getMetricsProxyCluster().getComponentsMap(); + assertThat(clusterComponents.keySet(), not(hasItem(ComponentId.fromString(Telegraf.class.getName())))); + assertThat(clusterComponents.keySet(), not(hasItem(ComponentId.fromString(TelegrafRegistry.class.getName())))); + } + + @Test + public void telegraf_config_is_generated_for_cloudwatch_in_services() { + String services = servicesWithCloudwatch(); VespaModel hostedModel = getModel(services, hosted); TelegrafConfig config = hostedModel.getConfig(TelegrafConfig.class, CLUSTER_CONFIG_ID); + var cloudWatch0 = config.cloudWatch(0); assertEquals("cloudwatch-consumer", cloudWatch0.consumer()); assertEquals("us-east-1", cloudWatch0.region()); @@ -43,6 +63,25 @@ public class TelegrafTest { assertEquals("", cloudWatch0.profile()); } + private String servicesWithCloudwatch() { + return String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='cloudwatch-consumer'>", + " <metric id='my-metric'/>", + " <cloudwatch region='us-east-1' namespace='my-namespace' >", + " <access-key-name>my-access-key</access-key-name>", + " <secret-key-name>my-secret-key</secret-key-name>", + " </cloudwatch>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + } + @Test public void multiple_cloudwatches_are_allowed_for_the_same_consumer() { String services = String.join("\n", diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index b06c93d6406..604e2abbbae 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -27,7 +27,7 @@ </consumer> <consumer id="cloudwatch-self-hosted-with-default-auth"> <metric-set id="public" /> - <cloudwatch region="us-east1" namespace="my-namespace" /> + <cloudwatch region="us-east1" namespace="namespace_legal.chars:/#1" /> </consumer> <consumer id="cloudwatch-self-hosted-with-profile"> <metric id="my-custom-metric" /> diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index f2ae997a164..9a091f1161c 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -391,6 +391,7 @@ "methods": [ "public void <init>(com.yahoo.config.provisioning.FlavorsConfig$Flavor)", "public void <init>(com.yahoo.config.provision.NodeResources)", + "public void <init>(java.lang.String, com.yahoo.config.provision.NodeResources)", "public com.yahoo.config.provision.Flavor with(com.yahoo.config.provision.host.FlavorOverrides)", "public com.yahoo.config.provision.Flavor with(com.yahoo.config.provision.NodeResources)", "public java.lang.String name()", diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java index 16369d82f9f..5aed5d8e2e7 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java @@ -158,7 +158,6 @@ public final class ClusterSpec { } /** Identifier of a group within a cluster */ - @SuppressWarnings("deprecation") public static final class Group { private final int index; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java index 2711406c216..d11d7137226 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java @@ -49,6 +49,11 @@ public class Flavor { this(resources.toString(), resources, Optional.empty(), Type.DOCKER_CONTAINER, false, 0, resources.vcpu()); } + /** Creates a *host* flavor for testing */ + public Flavor(String name, NodeResources resources) { + this(name, resources, Optional.empty(), Flavor.Type.VIRTUAL_MACHINE, true, 0, resources.vcpu()); + } + private Flavor(String name, NodeResources resources, Optional<FlavorOverrides> flavorOverrides, diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/HostName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/HostName.java index 510122c2342..25c42884295 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/HostName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/HostName.java @@ -4,7 +4,7 @@ package com.yahoo.config.provision; import java.util.Objects; /** - * Represents a host name + * A host name * * @author mortent */ @@ -18,12 +18,7 @@ public class HostName implements Comparable<HostName> { public String value() { return name; } - /** - * Create a {@link HostName} with a given name. - * - * @param name Name - * @return instance of {@link HostName}. - */ + /** Create a {@link HostName} with a given name */ public static HostName from(String name) { return new HostName(name); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java index a9f031cae70..eb462c86f4f 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java @@ -14,7 +14,7 @@ import java.util.Optional; import java.util.stream.Collectors; /** - * All the flavors *configured* in this zone (i.e this should be called HostFlavors). + * All the flavors configured in this zone (i.e this should be called HostFlavors). * * @author bratseth */ diff --git a/config/src/tests/trace/trace.cpp b/config/src/tests/trace/trace.cpp index 41c874eb1d4..9a355f39ecc 100644 --- a/config/src/tests/trace/trace.cpp +++ b/config/src/tests/trace/trace.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/common/trace.h> +#include <vespa/vespalib/data/slime/slime.h> using namespace config; diff --git a/config/src/vespa/config/common/configdefinition.cpp b/config/src/vespa/config/common/configdefinition.cpp index 861fc224867..92af068cff5 100644 --- a/config/src/vespa/config/common/configdefinition.cpp +++ b/config/src/vespa/config/common/configdefinition.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "configdefinition.h" #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/data/slime/slime.h> using namespace vespalib; using namespace vespalib::slime; diff --git a/config/src/vespa/config/common/configdefinition.h b/config/src/vespa/config/common/configdefinition.h index 3d86e5d2567..9154f3ad88a 100644 --- a/config/src/vespa/config/common/configdefinition.h +++ b/config/src/vespa/config/common/configdefinition.h @@ -3,8 +3,11 @@ #include <vespa/vespalib/stllike/string.h> #include <vector> -#include <vespa/vespalib/data/slime/slime.h> +namespace vespalib::slime { + class Cursor; + class Inspector; +} namespace config { /** diff --git a/config/src/vespa/config/common/trace.cpp b/config/src/vespa/config/common/trace.cpp index d1bb154eda9..76310d08c7d 100644 --- a/config/src/vespa/config/common/trace.cpp +++ b/config/src/vespa/config/common/trace.cpp @@ -2,6 +2,7 @@ #include "trace.h" #include <vespa/vespalib/trace/slime_trace_serializer.h> #include <vespa/vespalib/trace/slime_trace_deserializer.h> +#include <vespa/vespalib/data/slime/slime.h> using namespace vespalib; using namespace vespalib::slime; diff --git a/config/src/vespa/config/common/trace.h b/config/src/vespa/config/common/trace.h index 772cdb6f31e..c120fe30d12 100644 --- a/config/src/vespa/config/common/trace.h +++ b/config/src/vespa/config/common/trace.h @@ -1,11 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/trace/tracenode.h> #include <vespa/vespalib/stllike/string.h> -#include <memory> +#include <vespa/vespalib/data/memory.h> +namespace vespalib::slime { + class Cursor; + class Inspector; +} namespace config { /** diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java index 76eef33d6c0..aa3d6a2c0f8 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java @@ -48,7 +48,6 @@ public class IndexFacts { static final String unionName = "unionOfAllKnown"; /** A search definition which contains the union of all settings. */ - @SuppressWarnings("deprecation") private SearchDefinition unionSearchDefinition = new SearchDefinition(unionName); private boolean frozen; diff --git a/container-search/src/main/java/com/yahoo/search/Result.java b/container-search/src/main/java/com/yahoo/search/Result.java index 4080b09f40b..ab48d5797b2 100644 --- a/container-search/src/main/java/com/yahoo/search/Result.java +++ b/container-search/src/main/java/com/yahoo/search/Result.java @@ -89,7 +89,6 @@ public final class Result extends com.yahoo.processing.Response implements Clone * with a result. It should <b>always</b> be called when adding * hits from a result, but there is no constraints on the order of the calls. */ - @SuppressWarnings("deprecation") public void mergeWith(Result result) { totalHitCount += result.getTotalHitCount(); deepHitCount += result.getDeepHitCount(); diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java b/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java index 5f1cfccf549..6243dc694c2 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java @@ -39,8 +39,8 @@ class FederationResult { } /** - * Wait on each target for that targets timeout - * On the worst case this is the same as waiting for the max target timeout, + * Wait on each target for that targets timeout. + * In the worst case this is the same as waiting for the max target timeout, * in the average case it may be much better because lower timeout sources do not get to * drive the timeout above their own timeout value. * When this completes, results can be accessed from the TargetResults with no blocking diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java index 421544b5b49..60c5d42c531 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java @@ -354,7 +354,7 @@ public class FederationSearcher extends ForkingSearcher { } private void warnIfUnresolvedSearchChains(List<UnresolvedSearchChainException> missingTargets, - HitGroup errorHitGroup) { + HitGroup errorHitGroup) { if (!missingTargets.isEmpty()) { errorHitGroup.addError(missingSearchChainsErrorMessage(missingTargets)); } @@ -492,9 +492,9 @@ public class FederationSearcher extends ForkingSearcher { * TODO This is probably a dirty hack for bug 4711376. There are probably better ways. * But I will leave that to trd-processing@ * - * @param group The merging hitgroup to be updated if necessary - * @param orderer The per provider hit orderer. - * @return The hitorderer chosen + * @param group the merging hitgroup to be updated if necessary + * @param orderer the per provider hit orderer + * @return he hitorderer chosen */ private HitOrderer dirtyCopyIfModifiedOrderer(HitGroup group, HitOrderer orderer) { if (orderer != null) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index e3d41873a8e..854f72b27fb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -130,13 +130,18 @@ public class ApplicationController { int count = 0; for (TenantAndApplicationId id: curator.readApplicationIds()) { lockApplicationIfPresent(id, application -> { - if (id.tenant().value().startsWith("by-")) - application = application.with(DeploymentSpec.empty); - else + if (id.tenant().value().startsWith("by-")) { // TODO jonmv: Remove after run once. + for (Instance instance : application.get().instances().values()) + for (ZoneId zone : instance.deployments().keySet()) + deactivate(instance.id(), zone); + curator.removeApplication(id); + } + else { for (InstanceName instance : application.get().deploymentSpec().instanceNames()) - if ( ! application.get().instances().containsKey(instance)) + if (!application.get().instances().containsKey(instance)) application = withNewInstance(application, id.instance(instance)); - store(application); + store(application); + } }); count++; } @@ -229,7 +234,7 @@ public class ApplicationController { * * @throws IllegalArgumentException if the application already exists */ - public Application createApplication(TenantAndApplicationId id, Optional<Credentials> credentials) { + public Application createApplication(TenantAndApplicationId id, Credentials credentials) { try (Lock lock = lock(id)) { if (getApplication(id).isPresent()) throw new IllegalArgumentException("Could not create '" + id + "': Application already exists"); @@ -238,14 +243,9 @@ public class ApplicationController { com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId.validate(id.application().value()); - Optional<Tenant> tenant = controller.tenants().get(id.tenant()); - if (tenant.isEmpty()) + if (controller.tenants().get(id.tenant()).isEmpty()) throw new IllegalArgumentException("Could not create '" + id + "': This tenant does not exist"); - if (tenant.get().type() != Tenant.Type.user) { - if (credentials.isEmpty()) - throw new IllegalArgumentException("Could not create '" + id + "': No credentials provided"); - accessControl.createApplication(id, credentials.get()); - } + accessControl.createApplication(id, credentials); LockedApplication locked = new LockedApplication(new Application(id, clock.instant()), lock); store(locked); @@ -296,10 +296,6 @@ public class ApplicationController { throw new IllegalArgumentException("'" + instanceId + "' is a tester application!"); TenantAndApplicationId applicationId = TenantAndApplicationId.from(instanceId); - if ( getApplication(applicationId).isEmpty() - && controller.tenants().require(instanceId.tenant()).type() == Tenant.Type.user) - createApplication(applicationId, Optional.empty()); - if (getInstance(instanceId).isEmpty()) createInstance(instanceId); @@ -508,11 +504,7 @@ public class ApplicationController { * * @throws IllegalArgumentException if the application has deployments or the caller is not authorized */ - public void deleteApplication(TenantAndApplicationId id, Optional<Credentials> credentials) { - Tenant tenant = controller.tenants().require(id.tenant()); - if (tenant.type() != Tenant.Type.user && credentials.isEmpty()) - throw new IllegalArgumentException("Could not delete application '" + id + "': No credentials provided"); - + public void deleteApplication(TenantAndApplicationId id, Credentials credentials) { lockApplicationOrThrow(id, application -> { var deployments = application.get().instances().values().stream() .filter(instance -> ! instance.deployments().isEmpty()) @@ -531,8 +523,7 @@ public class ApplicationController { applicationStore.removeAll(id.tenant(), id.application()); applicationStore.removeAllTesters(id.tenant(), id.application()); - if (tenant.type() != Tenant.Type.user) - accessControl.deleteApplication(id, credentials.get()); + accessControl.deleteApplication(id, credentials); curator.removeApplication(id); controller.jobController().collectGarbage(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java index 6caf716aed4..12b985d1812 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import java.security.Principal; import java.security.PublicKey; @@ -40,7 +39,6 @@ public abstract class LockedTenant { static LockedTenant of(Tenant tenant, Lock lock) { switch (tenant.type()) { case athenz: return new Athenz((AthenzTenant) tenant); - case user: return new User((UserTenant) tenant); case cloud: return new Cloud((CloudTenant) tenant); default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.getClass().getName() + "'."); } @@ -99,32 +97,6 @@ public abstract class LockedTenant { } - /** A locked UserTenant. */ - public static class User extends LockedTenant { - - private final Optional<Contact> contact; - - private User(TenantName name, Optional<Contact> contact) { - super(name); - this.contact = contact; - } - - private User(UserTenant tenant) { - this(tenant.name(), tenant.contact()); - } - - @Override - public UserTenant get() { - return new UserTenant(name, contact); - } - - public User with(Contact contact) { - return new User(name, Optional.of(contact)); - } - - } - - /** A locked CloudTenant. */ public static class Cloud extends LockedTenant { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index e794334232f..ae905d2b209 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.security.TenantSpec; import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import java.time.Duration; import java.time.Instant; @@ -47,8 +46,13 @@ public class TenantController { Instant start = controller.clock().instant(); int count = 0; for (TenantName name : curator.readTenantNames()) { - lockIfPresent(name, LockedTenant.class, this::store); - count++; + if (name.value().startsWith(Tenant.userPrefix)) // TODO jonmv: Remove after run once. + + curator.removeTenant(name); + else { + lockIfPresent(name, LockedTenant.class, this::store); + count++; + } } log.log(Level.INFO, String.format("Wrote %d tenants in %s", count, Duration.between(start, controller.clock().instant()))); @@ -94,14 +98,6 @@ public class TenantController { curator.writeTenant(tenant.get()); } - /** Create an user tenant with given username */ - public void createUser(UserTenant tenant) { - try (Lock lock = lock(tenant.name())) { - requireNonExistent(tenant.name()); - curator.writeTenant(tenant); - } - } - /** Create a tenant, provided the given credentials are valid. */ public void create(TenantSpec tenantSpec, Credentials credentials) { try (Lock lock = lock(tenantSpec.tenant())) { @@ -141,13 +137,6 @@ public class TenantController { } } - /** Deletes the given user tenant. */ - public void deleteUser(UserTenant tenant) { - try (Lock lock = lock(tenant.name())) { - curator.removeTenant(tenant.name()); - } - } - private void requireNonExistent(TenantName name) { if ( "hosted-vespa".equals(name.value()) || get(name).isPresent() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index 628d7f48c85..301dad94b7e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -29,7 +29,6 @@ import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.security.TenantSpec; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import javax.ws.rs.ForbiddenException; import java.util.Arrays; @@ -186,8 +185,8 @@ public class AthenzFacade implements AccessControl { AthenzIdentity identity = ((AthenzPrincipal) credentials.user()).getIdentity(); List<AthenzDomain> userDomains = ztsClient.getTenantDomains(service, identity, "admin"); return tenants.stream() - .filter(tenant -> tenant.type() == Tenant.Type.user && ((UserTenant) tenant).is(identity.getName()) - || tenant.type() == Tenant.Type.athenz && userDomains.contains(((AthenzTenant) tenant).domain())) + .filter(tenant -> tenant.type() == Tenant.Type.athenz + && userDomains.contains(((AthenzTenant) tenant).domain())) .collect(Collectors.toUnmodifiableList()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index be189004f6d..6904bff8548 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -435,10 +435,6 @@ public class JobController { if ( ! type.environment().isManuallyDeployed()) throw new IllegalArgumentException("Direct deployments are only allowed to manually deployed environments."); - if ( controller.tenants().require(id.tenant()).type() == Tenant.Type.user - && controller.applications().getApplication(TenantAndApplicationId.from(id)).isEmpty()) - controller.applications().createApplication(TenantAndApplicationId.from(id), Optional.empty()); - controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { if ( ! application.get().instances().containsKey(id.instance())) application = controller.applications().withNewInstance(application, id); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java index c854c5b45bf..002ec1f4a05 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java @@ -14,7 +14,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import com.yahoo.yolean.Exceptions; import java.time.Duration; @@ -57,11 +56,10 @@ public class ApplicationOwnershipConfirmer extends Maintainer { .filter(application -> application.createdAt().isBefore(controller().clock().instant().minus(Duration.ofDays(90)))) .forEach(application -> { try { - Tenant tenant = tenantOf(application.id()); - tenant.contact().ifPresent(contact -> { // TODO jvenstad: Makes sense to require, and run this only in main? + tenantOf(application.id()).contact().ifPresent(contact -> { // TODO jvenstad: Makes sense to require, and run this only in main? ownershipIssues.confirmOwnership(application.ownershipIssueId(), summaryOf(application.id()), - determineAssignee(tenant, application), + determineAssignee(application), contact) .ifPresent(newIssueId -> store(newIssueId, application.id())); }); @@ -94,7 +92,7 @@ public class ApplicationOwnershipConfirmer extends Maintainer { application.ownershipIssueId().ifPresent(issueId -> { try { Tenant tenant = tenantOf(application.id()); - ownershipIssues.ensureResponse(issueId, tenant.type() == Tenant.Type.athenz ? tenant.contact() : Optional.empty()); + ownershipIssues.ensureResponse(issueId, tenant.contact()); } catch (RuntimeException e) { log.log(Level.INFO, "Exception caught when attempting to escalate issue with id '" + issueId + "': " + Exceptions.toMessageString(e)); @@ -118,8 +116,8 @@ public class ApplicationOwnershipConfirmer extends Maintainer { }); } - private User determineAssignee(Tenant tenant, Application application) { - return application.owner().orElse(tenant instanceof UserTenant ? userFor(tenant) : null); + private User determineAssignee(Application application) { + return application.owner().orElse(null); } private Tenant tenantOf(TenantAndApplicationId applicationId) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java index cbe9d8c70c1..5825285f9b0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java @@ -40,9 +40,6 @@ public class ContactInformationMaintainer extends Maintainer { case athenz: tenants.lockIfPresent(tenant.name(), LockedTenant.Athenz.class, lockedTenant -> tenants.store(lockedTenant.with(contactRetriever.getContact(lockedTenant.get().propertyId())))); return; - case user: tenants.lockIfPresent(tenant.name(), LockedTenant.User.class, lockedTenant -> - tenants.store(lockedTenant.with(contactRetriever.getContact(Optional.empty())))); - return; case cloud: return; default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'."); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java index 3be15b67252..e7c0094b7ab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java @@ -114,7 +114,7 @@ public class DeploymentIssueReporter extends Maintainer { try { Tenant tenant = ownerOf(application.id()); tenant.contact().ifPresent(contact -> { - User assignee = tenant.type() == Tenant.Type.user ? userFor(tenant) : application.owner().orElse(null); + User assignee = application.owner().orElse(null); Optional<IssueId> ourIssueId = application.deploymentIssueId(); IssueId issueId = deploymentIssues.fileUnlessOpen(ourIssueId, application.id().defaultInstance(), assignee, contact); store(application.id(), issueId); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index 7f938885cac..9df87ab4c12 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -19,7 +19,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import java.net.URI; import java.security.Principal; @@ -68,7 +67,6 @@ public class TenantSerializer { switch (tenant.type()) { case athenz: toSlime((AthenzTenant) tenant, tenantObject); break; - case user: toSlime((UserTenant) tenant, tenantObject); break; case cloud: toSlime((CloudTenant) tenant, tenantObject); break; default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'."); } @@ -85,13 +83,6 @@ public class TenantSerializer { }); } - private void toSlime(UserTenant tenant, Cursor tenantObject) { - tenant.contact().ifPresent(contact -> { - Cursor contactCursor = tenantObject.setObject(contactField); - writeContact(contact, contactCursor); - }); - } - private void toSlime(CloudTenant tenant, Cursor root) { developerKeysToSlime(tenant.developerKeys(), root.setArray(pemDeveloperKeysField)); toSlime(tenant.billingInfo(), root.setObject(billingInfoField)); @@ -117,7 +108,7 @@ public class TenantSerializer { switch (type) { case athenz: return athenzTenantFrom(tenantObject); - case user: return userTenantFrom(tenantObject); + case user: return null; // TODO jonmv: Remove when run once. case cloud: return cloudTenantFrom(tenantObject); default: throw new IllegalArgumentException("Unexpected tenant type '" + type + "'."); } @@ -132,12 +123,6 @@ public class TenantSerializer { return new AthenzTenant(name, domain, property, propertyId, contact); } - private UserTenant userTenantFrom(Inspector tenantObject) { - TenantName name = TenantName.from(tenantObject.field(nameField).asString()); - Optional<Contact> contact = contactFrom(tenantObject.field(contactField)); - return new UserTenant(name, contact); - } - private CloudTenant cloudTenantFrom(Inspector tenantObject) { TenantName name = TenantName.from(tenantObject.field(nameField).asString()); BillingInfo billingInfo = billingInfoFrom(tenantObject.field(billingInfoField)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 94d7b120406..5e234d31322 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -90,7 +90,6 @@ import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.yolean.Exceptions; @@ -252,7 +251,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private HttpResponse handlePUT(Path path, HttpRequest request) { - if (path.matches("/application/v4/user")) return createUser(request); + if (path.matches("/application/v4/user")) return new EmptyResponse(); if (path.matches("/application/v4/tenant/{tenant}")) return updateTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); @@ -335,11 +334,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // TODO jonmv: Move to Athenz API. private HttpResponse authenticatedUser(HttpRequest request) { Principal user = requireUserPrincipal(request); - if (user == null) - throw new NotAuthorizedException("You must be authenticated."); String userName = user instanceof AthenzPrincipal ? ((AthenzPrincipal) user).getIdentity().getName() : user.getName(); - TenantName tenantName = TenantName.from(UserTenant.normalizeUser(userName)); List<Tenant> tenants = controller.tenants().asList(new Credentials(user)); Slime slime = new Slime(); @@ -348,7 +344,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Cursor tenantsArray = response.setArray("tenants"); for (Tenant tenant : tenants) tenantInTenantsListToSlime(tenant, request.getUri(), tenantsArray.addObject()); - response.setBool("tenantExists", tenants.stream().anyMatch(tenant -> tenant.name().equals(tenantName))); + response.setBool("tenantExists", true); return new SlimeJsonResponse(slime); } @@ -1423,26 +1419,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return response; } - private HttpResponse createUser(HttpRequest request) { - String user = Optional.of(requireUserPrincipal(request)) - .filter(AthenzPrincipal.class::isInstance) - .map(AthenzPrincipal.class::cast) - .map(AthenzPrincipal::getIdentity) - .filter(AthenzUser.class::isInstance) - .map(AthenzIdentity::getName) - .map(UserTenant::normalizeUser) - .orElseThrow(() -> new ForbiddenException("Not authenticated or not a user.")); - - UserTenant tenant = UserTenant.create(user); - try { - controller.tenants().createUser(tenant); - return new MessageResponse("Created user '" + user + "'"); - } catch (AlreadyExistsException e) { - // Ok - return new MessageResponse("User '" + user + "' already exists"); - } - } - private HttpResponse updateTenant(String tenantName, HttpRequest request) { getTenantOrThrow(tenantName); TenantName tenant = TenantName.from(tenantName); @@ -1463,11 +1439,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse createApplication(String tenantName, String applicationName, HttpRequest request) { Inspector requestObject = toSlime(request.getData()).get(); TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName); - Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user - ? Optional.empty() - : Optional.of(accessControlRequests.credentials(id.tenant(), requestObject, request.getJDiscRequest())); + Credentials credentials = accessControlRequests.credentials(id.tenant(), requestObject, request.getJDiscRequest()); Application application = controller.applications().createApplication(id, credentials); - Slime slime = new Slime(); toSlime(id, slime.setObject(), request); return new SlimeJsonResponse(slime); @@ -1693,16 +1666,13 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse deleteTenant(String tenantName, HttpRequest request) { Optional<Tenant> tenant = controller.tenants().get(tenantName); - if ( ! tenant.isPresent()) + if (tenant.isEmpty()) return ErrorResponse.notFoundError("Could not delete tenant '" + tenantName + "': Tenant not found"); - if (tenant.get().type() == Tenant.Type.user) - controller.tenants().deleteUser((UserTenant) tenant.get()); - else - controller.tenants().delete(tenant.get().name(), - accessControlRequests.credentials(tenant.get().name(), - toSlime(request.getData()).get(), - request.getJDiscRequest())); + controller.tenants().delete(tenant.get().name(), + accessControlRequests.credentials(tenant.get().name(), + toSlime(request.getData()).get(), + request.getJDiscRequest())); // TODO: Change to a message response saying the tenant was deleted return tenant(tenant.get(), request); @@ -1710,9 +1680,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse deleteApplication(String tenantName, String applicationName, HttpRequest request) { TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName); - Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user - ? Optional.empty() - : Optional.of(accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest())); + Credentials credentials = accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest()); controller.applications().deleteApplication(id, credentials); return new MessageResponse("Deleted application " + id); } @@ -1721,9 +1689,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName); controller.applications().deleteInstance(id.instance(instanceName)); if (controller.applications().requireApplication(id).instances().isEmpty()) { - Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user - ? Optional.empty() - : Optional.of(accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest())); + Credentials credentials = accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest()); controller.applications().deleteApplication(id, credentials); } return new MessageResponse("Deleted instance " + id.instance(instanceName).toFullString()); @@ -1797,7 +1763,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { }); }); break; - case user: break; case cloud: { CloudTenant cloudTenant = (CloudTenant) tenant; @@ -1836,7 +1801,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { metaData.setString("athensDomain", athenzTenant.domain().getName()); metaData.setString("property", athenzTenant.property().id()); break; - case user: break; case cloud: break; default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'."); } @@ -2055,7 +2019,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private static String tenantType(Tenant tenant) { switch (tenant.type()) { - case user: return "USER"; case athenz: return "ATHENS"; case cloud: return "CLOUD"; default: throw new IllegalArgumentException("Unknown tenant type: " + tenant.getClass().getSimpleName()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java index e0c750dec80..55133aa06c8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java @@ -65,7 +65,7 @@ public abstract class Tenant { athenz, /** Tenant authenticated through Okta, as a user. */ - user, + user, // TODO jonmv: Remove. /** Tenant authenticated through some cloud identity provider. */ cloud; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/UserTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/UserTenant.java deleted file mode 100644 index a46d847f6f3..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/UserTenant.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.tenant; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; - -import java.util.Optional; - -/** - * Represents an user tenant in hosted Vespa. - * - * @author mpolden - */ -public class UserTenant extends Tenant { - - /** - * This should only be used by serialization. - * Use {@link #create(String)}. - * */ - public UserTenant(TenantName name, Optional<Contact> contact) { - super(name, contact); - } - - @Override - public Type type() { - return Type.user; - } - - public UserTenant(TenantName name) { - super(name, Optional.empty()); - } - - /** Returns true if this is the tenant for the given user name */ - public boolean is(String username) { - return name().value().equals(normalizeUser(username)); - } - - @Override - public String toString() { - return "user tenant '" + name() + "'"; - } - - /** Create a new user tenant */ - public static UserTenant create(String username) { - TenantName name = TenantName.from(username); - return new UserTenant(requireName(requireUser(name))); - } - - public static UserTenant create(String username, Optional<Contact> contact) { - TenantName name = TenantName.from(username); - return new UserTenant(requireName(requireUser(name)), contact); - } - - /** Normalize given username. E.g. foo_bar becomes by-foo-bar */ - public static String normalizeUser(String username) { - int offset = 0; - if (username.startsWith(Tenant.userPrefix)) { - offset = Tenant.userPrefix.length(); - } - return Tenant.userPrefix + username.substring(offset).replace('_', '-'); - } - - private static TenantName requireUser(TenantName name) { - if (!name.value().startsWith(Tenant.userPrefix)) { - throw new IllegalArgumentException("User tenant must have prefix '" + Tenant.userPrefix + "'"); - } - if (name.value().substring(Tenant.userPrefix.length()).contains("_")) { - throw new IllegalArgumentException("User tenant cannot contain '_'"); - } - return name; - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index a816f82c044..b5796bb4ecc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -327,20 +327,20 @@ public final class ControllerTester { return tenant; } - public Optional<Credentials> credentialsFor(TenantName tenantName) { + public Credentials credentialsFor(TenantName tenantName) { Tenant tenant = controller().tenants().require(tenantName); switch (tenant.type()) { case athenz: - return Optional.of(new AthenzCredentials(new AthenzPrincipal(new AthenzUser("user")), + return new AthenzCredentials(new AthenzPrincipal(new AthenzUser("user")), ((AthenzTenant) tenant).domain(), new OktaIdentityToken("okta-identity-token"), - new OktaAccessToken("okta-access-token"))); + new OktaAccessToken("okta-access-token")); case cloud: - return Optional.of(new Credentials(new SimplePrincipal("dev"))); + return new Credentials(new SimplePrincipal("dev")); default: - return Optional.empty(); + throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'"); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java index 8997f34fb98..e7e508f1789 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipI import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; -import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import org.junit.Before; import org.junit.Test; @@ -42,70 +41,70 @@ public class ApplicationOwnershipConfirmerTest { @Test public void testConfirmation() { Optional<Contact> contact = Optional.of(tester.controllerTester().serviceRegistry().contactRetrieverMock().contact()); - var propertyApp = tester.newDeploymentContext(); + var app = tester.newDeploymentContext(); tester.controller().tenants().lockOrThrow(appId.tenant(), LockedTenant.Athenz.class, tenant -> tester.controller().tenants().store(tenant.with(contact.get()))); - propertyApp.submit().deploy(); + app.submit().deploy(); - UserTenant user = UserTenant.create("by-user", contact); - tester.controller().tenants().createUser(user); - var userApp = tester.newDeploymentContext("by-user", "application", "default"); - userApp.submit().deploy(); + var appWithoutContact = tester.newDeploymentContext("other", "application", "default"); + appWithoutContact.submit().deploy(); - assertFalse("No issue is initially stored for a new application.", propertyApp.application().ownershipIssueId().isPresent()); - assertFalse("No issue is initially stored for a new application.", userApp.application().ownershipIssueId().isPresent()); - assertFalse("No escalation has been attempted for a new application", issues.escalatedToContact || issues.escalatedToTerminator); + assertFalse("No issue is initially stored for a new application.", app.application().ownershipIssueId().isPresent()); + assertFalse("No issue is initially stored for a new application.", appWithoutContact.application().ownershipIssueId().isPresent()); + assertFalse("No escalation has been attempted for a new application", issues.escalated); // Set response from the issue mock, which will be obtained by the maintainer on issue filing. Optional<IssueId> issueId = Optional.of(IssueId.from("1")); issues.response = issueId; confirmer.maintain(); - assertFalse("No issue is stored for an application newer than 3 months.", propertyApp.application().ownershipIssueId().isPresent()); - assertFalse("No issue is stored for an application newer than 3 months.", userApp.application().ownershipIssueId().isPresent()); + assertFalse("No issue is stored for an application newer than 3 months.", app.application().ownershipIssueId().isPresent()); + assertFalse("No issue is stored for an application newer than 3 months.", appWithoutContact.application().ownershipIssueId().isPresent()); tester.clock().advance(Duration.ofDays(91)); confirmer.maintain(); - assertEquals("Confirmation issue has been filed for property owned application.", issueId, propertyApp.application().ownershipIssueId()); - assertEquals("Confirmation issue has been filed for user owned application.", issueId, userApp.application().ownershipIssueId()); - assertTrue(issues.escalatedToTerminator); - assertTrue("Both applications have had their responses ensured.", issues.escalatedToContact); + assertEquals("Confirmation issue has been filed for application with contact.", issueId, app.application().ownershipIssueId()); + assertTrue("The confirmation issue response has been ensured.", issues.escalated); + assertEquals("No confirmation issue has been filed for application without contact.", Optional.empty(), appWithoutContact.application().ownershipIssueId()); // No new issue is created, so return empty now. issues.response = Optional.empty(); confirmer.maintain(); - assertEquals("Confirmation issue reference is not updated when no issue id is returned.", issueId, propertyApp.application().ownershipIssueId()); - assertEquals("Confirmation issue reference is not updated when no issue id is returned.", issueId, userApp.application().ownershipIssueId()); - - // The user deletes all production deployments — see that the issue is forgotten. - assertEquals("Confirmation issue for user is still open.", issueId, userApp.application().ownershipIssueId()); - userApp.application().productionDeployments().values().stream().flatMap(List::stream) - .forEach(deployment -> tester.controller().applications().deactivate(userApp.instanceId(), deployment.zone())); - assertTrue("No production deployments are listed for user.", userApp.application().require(InstanceName.defaultName()).productionDeployments().isEmpty()); - confirmer.maintain(); + assertEquals("Confirmation issue reference is not updated when no issue id is returned.", issueId, app.application().ownershipIssueId()); // Time has passed, and a new confirmation issue is in order for the property which is still in production. Optional<IssueId> issueId2 = Optional.of(IssueId.from("2")); issues.response = issueId2; confirmer.maintain(); - assertEquals("A new confirmation issue id is stored when something is returned to the maintainer.", issueId2, propertyApp.application().ownershipIssueId()); - assertEquals("Confirmation issue for application without production deployments has not been filed.", issueId, userApp.application().ownershipIssueId()); + assertEquals("A new confirmation issue id is stored when something is returned to the maintainer.", issueId2, app.application().ownershipIssueId()); - assertFalse("No owner is stored for application", propertyApp.application().owner().isPresent()); + assertFalse("No owner is stored for application", app.application().owner().isPresent()); issues.owner = Optional.of(User.from("username")); confirmer.maintain(); - assertEquals("Owner has been added to application", propertyApp.application().owner().get().username(), "username"); + assertEquals("Owner has been added to application", app.application().owner().get().username(), "username"); + + // The app deletes all production deployments — see that the issue is forgotten. + assertEquals("Confirmation issue for application is still open.", issueId2, app.application().ownershipIssueId()); + app.application().productionDeployments().values().stream().flatMap(List::stream) + .forEach(deployment -> tester.controller().applications().deactivate(app.instanceId(), deployment.zone())); + assertTrue("No production deployments are listed for user.", app.application().require(InstanceName.defaultName()).productionDeployments().isEmpty()); + confirmer.maintain(); + + // Time has passed, and a new confirmation issue is in order for the property which is still in production. + Optional<IssueId> issueId3 = Optional.of(IssueId.from("3")); + issues.response = issueId3; + confirmer.maintain(); + assertEquals("Confirmation issue for application without production deployments has not been filed.", issueId2, app.application().ownershipIssueId()); } private class MockOwnershipIssues implements OwnershipIssues { private Optional<IssueId> response; - private boolean escalatedToContact = false; - private boolean escalatedToTerminator = false; + private boolean escalated = false; private Optional<User> owner = Optional.empty(); @Override @@ -115,8 +114,7 @@ public class ApplicationOwnershipConfirmerTest { @Override public void ensureResponse(IssueId issueId, Optional<Contact> contact) { - if (contact.isPresent()) escalatedToContact = true; - else escalatedToTerminator = true; + escalated = true; } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java index ff1c952c2a5..9c085bb72f7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; -import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import org.junit.Test; import java.net.URI; @@ -77,14 +76,6 @@ public class TenantSerializerTest { } @Test - public void user_tenant() { - UserTenant tenant = UserTenant.create("by-foo", Optional.of(contact())); - UserTenant serialized = (UserTenant) serializer.tenantFrom(serializer.toSlime(tenant)); - assertEquals(tenant.name(), serialized.name()); - assertEquals(contact(), serialized.contact().get()); - } - - @Test public void cloud_tenant() { CloudTenant tenant = new CloudTenant(TenantName.from("elderly-lady"), new BillingInfo("old cat lady", "vespa"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 442770ba23e..cfde999973f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -182,15 +182,16 @@ public class ApplicationApiTest extends ControllerContainerTest { // GET the authenticated user (with associated tenants) tester.assertResponse(request("/application/v4/user", GET).userIdentity(USER_ID), new File("user.json")); - // PUT a user tenant + // TODO jonmv: Remove when dashboard is gone. + // PUT a user tenant — does nothing tester.assertResponse(request("/application/v4/user", PUT).userIdentity(USER_ID), - "{\"message\":\"Created user 'by-myuser'\"}"); + ""); // GET the authenticated user which now exists (with associated tenants) tester.assertResponse(request("/application/v4/user", GET).userIdentity(USER_ID), - new File("user-which-exists.json")); - // DELETE the user + new File("user.json")); + // DELETE the user — it doesn't exist, so access control fails tester.assertResponse(request("/application/v4/tenant/by-myuser", DELETE).userIdentity(USER_ID), - "{\"tenant\":\"by-myuser\",\"type\":\"USER\",\"applications\":[]}"); + "{\n \"code\" : 403,\n \"message\" : \"Access denied\"\n}", 403); // GET all tenants tester.assertResponse(request("/application/v4/tenant/", GET).userIdentity(USER_ID), new File("tenant-list.json")); @@ -753,17 +754,10 @@ public class ApplicationApiTest extends ControllerContainerTest { .userIdentity(USER_ID), "{\"message\":\"Aborting run 2 of staging-test for tenant1.application1.instance1\"}"); - // PUT (create) the authenticated user - byte[] data = new byte[0]; - tester.assertResponse(request("/application/v4/user?user=new_user&domain=by", PUT) - .data(data) - .userIdentity(new UserId("new_user")), // Normalized to by-new-user by API - new File("create-user-response.json")); - // GET user lists only tenants for the authenticated user tester.assertResponse(request("/application/v4/user", GET) .userIdentity(new UserId("other_user")), - "{\"user\":\"other_user\",\"tenants\":[],\"tenantExists\":false}"); + "{\"user\":\"other_user\",\"tenants\":[],\"tenantExists\":true}"); // OPTIONS return 200 OK tester.assertResponse(request("/application/v4/", Request.Method.OPTIONS) @@ -1374,22 +1368,22 @@ public class ApplicationApiTest extends ControllerContainerTest { // PUT (create) the authenticated user tester.assertResponse(request("/application/v4/user?user=new_user&domain=by", PUT) .userIdentity(userId), // Normalized to by-new-user by API - new File("create-user-response.json")); + ""); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from("domain1"), com.yahoo.config.provision.AthenzService.from("service")) .build(); - // POST (deploy) an application to a dev zone + // POST (deploy) an application to a dev zone fails because user tenant is used — these do not exist. MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); tester.assertResponse(request("/application/v4/tenant/by-new-user/application/application1/environment/dev/region/us-west-1/instance/default", POST) .data(entity) .userIdentity(userId), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"User user.new-user is not allowed to launch service domain1.service. Please reach out to the domain admin.\"}", - 400); + "{\n \"code\" : 403,\n \"message\" : \"Access denied\"\n}", + 403); createTenantAndApplication(); - // POST (deploy) an application to dev through a deployment job + // POST (deploy) an application to dev through a deployment job, with user instance and a proper tenant tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/new-user/deploy/dev-us-east-1", POST) .data(entity) .userIdentity(userId), @@ -1401,11 +1395,12 @@ public class ApplicationApiTest extends ControllerContainerTest { .domains.get(ATHENZ_TENANT_DOMAIN) .admin(HostedAthenzIdentities.from(userId)); - // POST (deploy) an application to a dev zone - tester.assertResponse(request("/application/v4/tenant/by-new-user/application/application1/environment/dev/region/us-east-1/instance/default", POST) + // POST (deploy) an application to a dev zone fails because user tenant is used — these do not exist. + tester.assertResponse(request("/application/v4/tenant/by-new-user/application/application1/environment/dev/region/us-west-1/instance/default", POST) .data(entity) .userIdentity(userId), - new File("deploy-result.json")); + "{\n \"code\" : 403,\n \"message\" : \"Access denied\"\n}", + 403); // POST (deploy) an application to dev through a deployment job tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/new-user/deploy/dev-us-east-1", POST) @@ -1434,7 +1429,7 @@ public class ApplicationApiTest extends ControllerContainerTest { AthenzCredentials credentials = new AthenzCredentials( new AthenzPrincipal(new AthenzUser(developer.id())), sandboxDomain, OKTA_IT, OKTA_AT); tester.controller().tenants().create(tenantSpec, credentials); - tester.controller().applications().createApplication(TenantAndApplicationId.from("sandbox", "myapp"), Optional.of(credentials)); + tester.controller().applications().createApplication(TenantAndApplicationId.from("sandbox", "myapp"), credentials); // Create an application package referencing the service from the other domain ApplicationPackage applicationPackage = new ApplicationPackageBuilder() diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json index 79b9a785801..9902267dbb5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json @@ -1,5 +1,5 @@ { "user": "myuser", "tenants": @include(tenant-list.json), - "tenantExists": false + "tenantExists": true }
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index d70a09414bb..93d88ff8abd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -71,10 +71,10 @@ public class UserApiTest extends ControllerContainerCloudTest { .data("{\"token\":\"hello\"}"), new File("tenant-without-applications.json")); - // PUT a tenant is not available to anyone. + // PUT a tenant is ignored. tester.assertResponse(request("/application/v4/user/", PUT) .roles(operator), - "{\"error-code\":\"FORBIDDEN\",\"message\":\"Not authenticated or not a user.\"}", 403); + "", 200); // GET at user/v1 root fails as no access control is defined there. tester.assertResponse(request("/user/v1/"), diff --git a/defaults/src/main/java/com/yahoo/vespa/defaults/Defaults.java b/defaults/src/main/java/com/yahoo/vespa/defaults/Defaults.java index 6fb6e4f0860..0565b1cff09 100644 --- a/defaults/src/main/java/com/yahoo/vespa/defaults/Defaults.java +++ b/defaults/src/main/java/com/yahoo/vespa/defaults/Defaults.java @@ -39,6 +39,7 @@ public class Defaults { vespaPortConfigServerHttp = vespaPortConfigServerRpc + 1; vespaPortConfigProxyRpc = findConfigProxyPort(vespaPortBase + 90); } + static private String findVespaHome(String defHome) { Optional<String> vespaHomeEnv = Optional.ofNullable(System.getenv("VESPA_HOME")); if ( ! vespaHomeEnv.isPresent() || vespaHomeEnv.get().trim().isEmpty()) { diff --git a/dist/vespa.spec b/dist/vespa.spec index 8ce494fd7cb..1c1f5071684 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -6,6 +6,12 @@ # Force special prefix for Vespa %define _prefix /opt/vespa %define _vespa_deps_prefix /opt/vespa-deps +%define _vespa_user vespa +%define _vespa_group vespa +%define _create_vespa_group 1 +%define _create_vespa_user 1 +%define _create_vespa_service 1 +%define _defattr_is_vespa_vespa 0 Name: vespa Version: _VESPA_VERSION_ @@ -211,7 +217,7 @@ Vespa - The open big data serving engine %prep %if 0%{?installdir:1} -%setup -D -T +%setup -c -D -T %else %setup -q %endif @@ -238,6 +244,7 @@ cmake3 -DCMAKE_INSTALL_PREFIX=%{_prefix} \ -DEXTRA_INCLUDE_DIRECTORY="%{_extra_include_directory}" \ -DCMAKE_INSTALL_RPATH="%{_prefix}/lib64%{?_extra_link_directory:;%{_extra_link_directory}};/usr/lib/jvm/jre-11-openjdk/lib" \ %{?_vespa_llvm_version:-DVESPA_LLVM_VERSION="%{_vespa_llvm_version}"} \ + -DVESPA_USER=%{_vespa_user} \ -DVESPA_UNPRIVILEGED=no \ . @@ -253,49 +260,97 @@ cp -r %{installdir} %{buildroot} make install DESTDIR=%{buildroot} %endif +%if %{_create_vespa_service} mkdir -p %{buildroot}/usr/lib/systemd/system cp %{buildroot}/%{_prefix}/etc/systemd/system/vespa.service %{buildroot}/usr/lib/systemd/system cp %{buildroot}/%{_prefix}/etc/systemd/system/vespa-configserver.service %{buildroot}/usr/lib/systemd/system +%endif %clean rm -rf $RPM_BUILD_ROOT %pre -getent group vespa >/dev/null || groupadd -r vespa -getent passwd vespa >/dev/null || \ - useradd -r -g vespa --home-dir %{_prefix} --create-home -s /sbin/nologin \ - -c "Create owner of all Vespa data files" vespa -# Home dir created with rwx on user only. -chmod a+rx %{_prefix} +%if %{_create_vespa_group} +getent group %{_vespa_group} >/dev/null || groupadd -r %{_vespa_group} +%endif +%if %{_create_vespa_user} +getent passwd %{_vespa_user} >/dev/null || \ + useradd -r -g %{_vespa_group} --home-dir %{_prefix} -s /sbin/nologin \ + -c "Create owner of all Vespa data files" %{_vespa_user} +%endif echo "pathmunge %{_prefix}/bin" > /etc/profile.d/vespa.sh echo "export VESPA_HOME=%{_prefix}" >> /etc/profile.d/vespa.sh exit 0 +%if %{_create_vespa_service} %post %systemd_post vespa-configserver.service %systemd_post vespa.service +%endif +%if %{_create_vespa_service} %preun %systemd_preun vespa.service %systemd_preun vespa-configserver.service +%endif %postun +%if %{_create_vespa_service} %systemd_postun_with_restart vespa.service %systemd_postun_with_restart vespa-configserver.service +%endif if [ $1 -eq 0 ]; then # this is an uninstallation rm -f /etc/profile.d/vespa.sh - ! getent passwd vespa >/dev/null || userdel vespa - ! getent group vespa >/dev/null || groupdel vespa +%if %{_create_vespa_user} + ! getent passwd %{_vespa_user} >/dev/null || userdel %{_vespa_user} +%endif +%if %{_create_vespa_group} + ! getent group %{_vespa_group} >/dev/null || groupdel %{_vespa_group} +%endif fi %files -%defattr(-,vespa,vespa,-) +%if %{_defattr_is_vespa_vespa} +%defattr(-,%{_vespa_user},%{_vespa_group},-) +%endif %doc -%{_prefix}/* +%dir %{_prefix} +%{_prefix}/bin +%dir %{_prefix}/conf +%{_prefix}/conf/configserver +%{_prefix}/conf/configserver-app +%dir %{_prefix}/conf/logd +%{_prefix}/conf/node-admin-app +%dir %{_prefix}/conf/vespa +%dir %attr(-,%{_vespa_user},-) %{_prefix}/conf/zookeeper +%dir %{_prefix}/etc +%{_prefix}/etc/systemd +%{_prefix}/etc/vespa +%{_prefix}/include +%{_prefix}/lib +%{_prefix}/lib64 +%{_prefix}/libexec +%dir %attr(1777,-,-) %{_prefix}/logs +%dir %attr(1777,%{_vespa_user},-) %{_prefix}/logs/vespa +%dir %attr(-,%{_vespa_user},-) %{_prefix}/logs/vespa/configserver +%dir %attr(-,%{_vespa_user},-) %{_prefix}/logs/vespa/node-admin +%dir %attr(-,%{_vespa_user},-) %{_prefix}/logs/vespa/search +%{_prefix}/man +%{_prefix}/sbin +%{_prefix}/share +%dir %attr(1777,-,-) %{_prefix}/tmp +%dir %attr(1777,%{_vespa_user},-) %{_prefix}/tmp/vespa +%dir %{_prefix}/var +%dir %{_prefix}/var/db +%dir %attr(-,%{_vespa_user},-) %{_prefix}/var/db/vespa +%dir %attr(-,%{_vespa_user},-) %{_prefix}/var/db/vespa/logcontrol +%dir %attr(-,%{_vespa_user},-) %{_prefix}/var/zookeeper %config(noreplace) %{_prefix}/conf/logd/logd.cfg %config(noreplace) %{_prefix}/conf/vespa/default-env.txt %config(noreplace) %{_prefix}/etc/vespamalloc.conf +%if %{_create_vespa_service} %attr(644,root,root) /usr/lib/systemd/system/vespa.service %attr(644,root,root) /usr/lib/systemd/system/vespa-configserver.service +%endif %changelog diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java index 32302a98757..ecfe9b2468a 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java @@ -34,6 +34,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { private final Map<String, String> labels = new HashMap<>(); private final List<String> environmentAssignments = new ArrayList<>(); private final List<String> volumeBindSpecs = new ArrayList<>(); + private final List<String> dnsOptions = new ArrayList<>(); private final List<Ulimit> ulimits = new ArrayList<>(); private final Set<Capability> addCapabilities = new HashSet<>(); private final Set<Capability> dropCapabilities = new HashSet<>(); @@ -96,6 +97,12 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { } @Override + public Docker.CreateContainerCommand withDnsOption(String dnsOption) { + dnsOptions.add(dnsOption); + return this; + } + + @Override public Docker.CreateContainerCommand withPrivileged(boolean privileged) { this.privileged = privileged; return this; @@ -171,6 +178,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { .withPidsLimit(-1L) .withCapAdd(addCapabilities.toArray(new Capability[0])) .withCapDrop(dropCapabilities.toArray(new Capability[0])) + .withDnsOptions(dnsOptions) .withPrivileged(privileged); containerResources.ifPresent(cr -> hostConfig @@ -241,6 +249,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { toRepeatedOption("--cap-add", addCapabilitiesList), toRepeatedOption("--cap-drop", dropCapabilitiesList), toRepeatedOption("--security-opt", securityOpts), + toRepeatedOption("--dns-option", dnsOptions), toOptionalOption("--net", networkMode), toOptionalOption("--ip", ipv4Address), toOptionalOption("--ip6", ipv6Address), diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java index 4e7ef5a1ff6..648c94d71ab 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java @@ -53,6 +53,7 @@ public interface Docker { CreateContainerCommand withAddCapability(String capabilityName); CreateContainerCommand withDropCapability(String capabilityName); CreateContainerCommand withSecurityOpts(String securityOpt); + CreateContainerCommand withDnsOption(String dnsOption); CreateContainerCommand withPrivileged(boolean privileged); void create(); diff --git a/eval/src/vespa/eval/eval/interpreted_function.cpp b/eval/src/vespa/eval/eval/interpreted_function.cpp index 0a630a3e20a..ec28604fd87 100644 --- a/eval/src/vespa/eval/eval/interpreted_function.cpp +++ b/eval/src/vespa/eval/eval/interpreted_function.cpp @@ -3,21 +3,16 @@ #include "interpreted_function.h" #include "node_visitor.h" #include "node_traverser.h" -#include "check_type.h" -#include "tensor_spec.h" -#include "operation.h" #include "tensor_nodes.h" #include "tensor_engine.h" +#include "make_tensor_function.h" +#include "compile_tensor_function.h" #include <vespa/vespalib/util/classname.h> #include <vespa/eval/eval/llvm/compile_cache.h> #include <vespa/vespalib/util/benchmark_timer.h> #include <set> -#include "make_tensor_function.h" -#include "compile_tensor_function.h" - -namespace vespalib { -namespace eval { +namespace vespalib::eval { namespace { @@ -42,11 +37,12 @@ InterpretedFunction::State::State(const TensorEngine &engine_in) params(nullptr), stash(), stack(), - program_offset(0) + program_offset(0), + if_cnt(0) { } -InterpretedFunction::State::~State() {} +InterpretedFunction::State::~State() = default; void InterpretedFunction::State::init(const LazyParams ¶ms_in) { @@ -82,7 +78,7 @@ InterpretedFunction::InterpretedFunction(const TensorEngine &engine, const nodes _program = compile_tensor_function(optimized, _stash); } -InterpretedFunction::~InterpretedFunction() {} +InterpretedFunction::~InterpretedFunction() = default; const Value & InterpretedFunction::eval(Context &ctx, const LazyParams ¶ms) const @@ -123,5 +119,4 @@ InterpretedFunction::detect_issues(const Function &function) return Function::Issues(std::move(checker.issues)); } -} // namespace vespalib::eval -} // namespace vespalib +} diff --git a/eval/src/vespa/eval/eval/interpreted_function.h b/eval/src/vespa/eval/eval/interpreted_function.h index e3e8d18b44f..e638ccffcea 100644 --- a/eval/src/vespa/eval/eval/interpreted_function.h +++ b/eval/src/vespa/eval/eval/interpreted_function.h @@ -7,8 +7,7 @@ #include "lazy_params.h" #include <vespa/vespalib/util/stash.h> -namespace vespalib { -namespace eval { +namespace vespalib::eval { namespace nodes { struct Node; } struct TensorEngine; @@ -107,5 +106,4 @@ public: static Function::Issues detect_issues(const Function &function); }; -} // namespace vespalib::eval -} // namespace vespalib +} diff --git a/eval/src/vespa/eval/eval/lazy_params.cpp b/eval/src/vespa/eval/eval/lazy_params.cpp index aec8cf78059..2c00c4c312b 100644 --- a/eval/src/vespa/eval/eval/lazy_params.cpp +++ b/eval/src/vespa/eval/eval/lazy_params.cpp @@ -1,19 +1,16 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "lazy_params.h" -#include <assert.h> +#include <vespa/vespalib/util/stash.h> +#include <cassert> namespace vespalib::eval { -LazyParams::~LazyParams() -{ -} +LazyParams::~LazyParams() = default; //----------------------------------------------------------------------------- -SimpleObjectParams::~SimpleObjectParams() -{ -} +SimpleObjectParams::~SimpleObjectParams() = default; const Value & SimpleObjectParams::resolve(size_t idx, Stash &) const @@ -24,9 +21,7 @@ SimpleObjectParams::resolve(size_t idx, Stash &) const //----------------------------------------------------------------------------- -SimpleParams::~SimpleParams() -{ -} +SimpleParams::~SimpleParams() = default; const Value & SimpleParams::resolve(size_t idx, Stash &stash) const diff --git a/eval/src/vespa/eval/eval/tensor_function.h b/eval/src/vespa/eval/eval/tensor_function.h index 4b862e9ec6a..55d27fb74ea 100644 --- a/eval/src/vespa/eval/eval/tensor_function.h +++ b/eval/src/vespa/eval/eval/tensor_function.h @@ -2,20 +2,17 @@ #pragma once -#include <memory> -#include <vector> -#include <variant> -#include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/util/arrayref.h> -#include <vespa/vespalib/util/overload.h> #include "tensor_spec.h" #include "lazy_params.h" #include "value_type.h" #include "value.h" #include "aggr.h" - #include "interpreted_function.h" +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/arrayref.h> +#include <vespa/vespalib/util/overload.h> +#include <variant> namespace vespalib { diff --git a/eval/src/vespa/eval/eval/value.h b/eval/src/vespa/eval/eval/value.h index 6701173bcd3..cad76c93c5c 100644 --- a/eval/src/vespa/eval/eval/value.h +++ b/eval/src/vespa/eval/eval/value.h @@ -2,13 +2,11 @@ #pragma once -#include <vespa/vespalib/stllike/string.h> -#include <memory> -#include <vespa/vespalib/util/stash.h> #include "value_type.h" +#include <vespa/vespalib/util/traits.h> +#include <memory> -namespace vespalib { -namespace eval { +namespace vespalib::eval { class Tensor; @@ -40,7 +38,6 @@ public: static const ValueType &double_type() { return _type; } }; -} // namespace vespalib::eval -} // namespace vespalib +} VESPA_CAN_SKIP_DESTRUCTION(::vespalib::eval::DoubleValue); diff --git a/fbench/src/fbench/fbench.cpp b/fbench/src/fbench/fbench.cpp index 593ae30a0e5..c2996eebb5c 100644 --- a/fbench/src/fbench/fbench.cpp +++ b/fbench/src/fbench/fbench.cpp @@ -3,10 +3,10 @@ #include <httpclient/httpclient.h> #include <util/filereader.h> #include <util/clientstatus.h> +#include <vespa/vespalib/crypto/crypto_exception.h> #include <vespa/vespalib/net/crypto_engine.h> #include <vespa/vespalib/net/tls/transport_security_options.h> #include <vespa/vespalib/net/tls/tls_crypto_engine.h> -#include <vespa/vespalib/net/tls/crypto_exception.h> #include <vespa/vespalib/io/mapped_file_input.h> #include "client.h" #include "fbench.h" @@ -99,7 +99,7 @@ FBench::init_crypto_engine(const std::string &ca_certs_file_name, } try { _crypto_engine = std::make_shared<vespalib::TlsCryptoEngine>(tls_opts); - } catch (vespalib::net::tls::CryptoException &e) { + } catch (vespalib::crypto::CryptoException &e) { fprintf(stderr, "%s\n", e.what()); return false; } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 00741cb69f9..815fcda6ee7 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -67,6 +67,12 @@ public class Flags { "Takes effect on next host admin tick.", HOSTNAME); + public static final UnboundBooleanFlag SERVICE_MODEL_CACHE = defineFeatureFlag( + "service-model-cache", true, + "Whether the service model is cached.", + "Takes effect on restart of config server.", + HOSTNAME); + public static final UnboundListFlag<String> DISABLED_HOST_ADMIN_TASKS = defineListFlag( "disabled-host-admin-tasks", List.of(), String.class, "List of host-admin task names (as they appear in the log, e.g. root>main>UpgradeTask) that should be skipped", diff --git a/fnet/src/tests/frt/values/values_test.cpp b/fnet/src/tests/frt/values/values_test.cpp index 5bc6b0e2dce..3b36e8989c1 100644 --- a/fnet/src/tests/frt/values/values_test.cpp +++ b/fnet/src/tests/frt/values/values_test.cpp @@ -3,6 +3,7 @@ #include <vespa/fnet/frt/values.h> #include <vespa/fnet/databuffer.h> #include <vespa/fnet/info.h> +#include <vespa/vespalib/util/stash.h> using vespalib::Stash; diff --git a/fnet/src/vespa/fnet/frt/rpcrequest.h b/fnet/src/vespa/fnet/frt/rpcrequest.h index cc871e7ac0c..eaa34a46b7a 100644 --- a/fnet/src/vespa/fnet/frt/rpcrequest.h +++ b/fnet/src/vespa/fnet/frt/rpcrequest.h @@ -5,6 +5,7 @@ #include "values.h" #include "error.h" #include <vespa/fnet/context.h> +#include <vespa/vespalib/util/stash.h> #include <atomic> class FNETConnection; diff --git a/fnet/src/vespa/fnet/frt/values.cpp b/fnet/src/vespa/fnet/frt/values.cpp index a5f59df19b2..3b37aa9a1bc 100644 --- a/fnet/src/vespa/fnet/frt/values.cpp +++ b/fnet/src/vespa/fnet/frt/values.cpp @@ -3,6 +3,7 @@ #include "values.h" #include <vespa/fnet/databuffer.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> #include <cassert> static_assert(sizeof(uint8_t) == 1, "uint8_t must be 1 byte."); @@ -81,7 +82,7 @@ FRT_Values::FRT_Values(Stash &stash) _stash(stash) { } -FRT_Values::~FRT_Values() { } +FRT_Values::~FRT_Values() = default; LocalBlob::LocalBlob(const char *data, uint32_t len) : _data(Alloc::alloc(len)), @@ -294,7 +295,7 @@ FRT_Values::AddSharedData(FRT_ISharedBlob *blob) { } void -FRT_Values::AddData(vespalib::alloc::Alloc buf, uint32_t len) { +FRT_Values::AddData(vespalib::alloc::Alloc && buf, uint32_t len) { AddSharedData(&_stash.create<LocalBlob>(std::move(buf), len)); } diff --git a/fnet/src/vespa/fnet/frt/values.h b/fnet/src/vespa/fnet/frt/values.h index e00aec8423c..2aa7551c423 100644 --- a/fnet/src/vespa/fnet/frt/values.h +++ b/fnet/src/vespa/fnet/frt/values.h @@ -3,9 +3,10 @@ #pragma once #include "isharedblob.h" -#include <vespa/vespalib/util/stash.h> #include <cstring> +namespace vespalib { class Stash; } +namespace vespalib::alloc { class Alloc; } namespace fnet { char * copyString(char *dst, const char *src, size_t len); char * copyData(char *dst, const void *src, size_t len); @@ -216,7 +217,7 @@ public: char *AddString(uint32_t len); FRT_StringValue *AddStringArray(uint32_t len); void AddSharedData(FRT_ISharedBlob *blob); - void AddData(Alloc buf, uint32_t len); + void AddData(Alloc && buf, uint32_t len); void AddData(const char *buf, uint32_t len); char *AddData(uint32_t len); FRT_DataValue *AddDataArray(uint32_t len); diff --git a/jrt/src/com/yahoo/jrt/Connector.java b/jrt/src/com/yahoo/jrt/Connector.java index 57fad5a163d..10f2b3742f2 100644 --- a/jrt/src/com/yahoo/jrt/Connector.java +++ b/jrt/src/com/yahoo/jrt/Connector.java @@ -1,44 +1,67 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jrt; -import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.concurrent.CachedThreadPoolWithFallback; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.Executor; import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; -class Connector { +class Connector implements AutoCloseable { - private final ExecutorService executor = new ThreadPoolExecutor(1, 64, 1L, TimeUnit.SECONDS, - new LinkedBlockingQueue<>(), - ThreadFactoryFactory.getDaemonThreadFactory("jrt.connector")); + private static final Object globalLock = new Object(); + private static CachedThreadPoolWithFallback globalExecutor = null; + private static long usages = 0; - private void connect(Connection conn) { - conn.transportThread().addConnection(conn.connect()); + private static CachedThreadPoolWithFallback acquire() { + synchronized (globalLock) { + if (globalExecutor == null) { + globalExecutor = new CachedThreadPoolWithFallback("jrt.connector", 1, 64, 1L, TimeUnit.SECONDS); + } + usages++; + return globalExecutor; + } } - public void connectLater(Connection conn) { - try { - executor.execute(() -> connect(conn)); - } catch (RejectedExecutionException e) { - conn.transportThread().addConnection(conn); + private static void release(CachedThreadPoolWithFallback executor) { + synchronized (globalLock) { + assert executor == globalExecutor; + usages--; + if (usages == 0) { + globalExecutor.close(); + globalExecutor = null; + } } } - public Connector shutdown() { - executor.shutdown(); - return this; + private final AtomicReference<CachedThreadPoolWithFallback> executor; + + Connector() { + executor = new AtomicReference<>(acquire()); } - public void join() { - while (true) { + private void connect(Connection conn) { + conn.transportThread().addConnection(conn.connect()); + } + + public void connectLater(Connection conn) { + Executor executor = this.executor.get(); + if (executor != null) { try { - if (executor.awaitTermination(60, TimeUnit.SECONDS)) { - return; - } - } catch (InterruptedException e) {} + executor.execute(() -> connect(conn)); + return; + } catch (RejectedExecutionException ignored) { + } + } + conn.transportThread().addConnection(conn); + } + + @Override + public void close() { + CachedThreadPoolWithFallback toShutdown = executor.getAndSet(null); + if (toShutdown != null) { + release(toShutdown); } } } diff --git a/jrt/src/com/yahoo/jrt/Supervisor.java b/jrt/src/com/yahoo/jrt/Supervisor.java index 09360c2da7b..d4168e97743 100644 --- a/jrt/src/com/yahoo/jrt/Supervisor.java +++ b/jrt/src/com/yahoo/jrt/Supervisor.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jrt; - import java.util.HashMap; import java.util.concurrent.atomic.AtomicReference; @@ -103,19 +102,6 @@ public class Supervisor { } /** - * Remove a method from the set of methods held by this Supervisor - * - * @param methodName name of the method to remove - **/ - public void removeMethod(String methodName) { - synchronized (methodMapLock) { - HashMap<String, Method> newMap = new HashMap<>(methodMap()); - newMap.remove(methodName); - methodMap.setRelease(newMap); - } - } - - /** * Remove a method from the set of methods held by this * Supervisor. Use this if you know exactly which method to remove * and not only the name. diff --git a/jrt/src/com/yahoo/jrt/Transport.java b/jrt/src/com/yahoo/jrt/Transport.java index 6f5a381fd6b..8abd3942a39 100644 --- a/jrt/src/com/yahoo/jrt/Transport.java +++ b/jrt/src/com/yahoo/jrt/Transport.java @@ -4,7 +4,6 @@ package com.yahoo.jrt; import java.nio.channels.SocketChannel; import java.util.ArrayList; -import java.util.Iterator; import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; @@ -19,7 +18,7 @@ import java.util.logging.Logger; **/ public class Transport { - private static Logger log = Logger.getLogger(Transport.class.getName()); + private static final Logger log = Logger.getLogger(Transport.class.getName()); private final FatalErrorHandler fatalHandler; // NB: this must be set first private final CryptoEngine cryptoEngine; @@ -28,7 +27,7 @@ public class Transport { private final AtomicInteger runCnt; private final TransportMetrics metrics = TransportMetrics.getInstance(); - private final ArrayList<TransportThread> threads = new ArrayList<TransportThread>(); + private final ArrayList<TransportThread> threads = new ArrayList<>(); private final Random rnd = new Random(); /** @@ -174,7 +173,7 @@ public class Transport { * @return this object, to enable chaining with join **/ public Transport shutdown() { - connector.shutdown().join(); + connector.close(); for (TransportThread thread: threads) { thread.shutdown(); } diff --git a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp index bf98bbd75ef..e37a6fb2dcb 100644 --- a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp +++ b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp @@ -80,7 +80,9 @@ CfHandler::doConfigure() if (getenv("VESPA_HOSTNAME") != NULL && getenv("VESPA_TENANT") != NULL && getenv("VESPA_APPLICATION")!= NULL && - getenv("VESPA_INSTANCE") != NULL ) + getenv("VESPA_INSTANCE") != NULL && + getenv("VESPA_ENVIRONMENT") != NULL && + getenv("VESPA_REGION") != NULL) { path = cfFilePath(config.splunkHome, "inputs.conf"); tmpPath = path + ".new"; @@ -88,7 +90,7 @@ CfHandler::doConfigure() if (fp != NULL) { fprintf(fp, "[default]\n"); fprintf(fp, "host = %s\n", getenv("VESPA_HOSTNAME")); - fprintf(fp, "_meta = vespa_tenant::%s vespa_app::%s.%s\n", getenv("VESPA_TENANT"), getenv("VESPA_APPLICATION"), getenv("VESPA_INSTANCE")); + fprintf(fp, "_meta = vespa_tenant::%s vespa_app::%s.%s vespa_zone::%s.%s\n", getenv("VESPA_TENANT"), getenv("VESPA_APPLICATION"), getenv("VESPA_INSTANCE"), getenv("VESPA_ENVIRONMENT"), getenv("VESPA_REGION")); fclose(fp); rename(tmpPath.c_str(), path.c_str()); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java index c9d7618b9d7..c04dca465a1 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java @@ -35,6 +35,7 @@ import static com.google.common.base.Strings.isNullOrEmpty; * @author gjoranv */ public class VespaMetrics { + private static final Logger log = Logger.getLogger(VespaMetrics.class.getPackage().getName()); public static final ConsumerId VESPA_CONSUMER_ID = toConsumerId("Vespa"); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java index ae0ef2fa57a..51bdae1aab3 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java @@ -21,6 +21,7 @@ import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; * @author gjoranv */ public class ValuesFetcher { + private static final Logger log = Logger.getLogger(ValuesFetcher.class.getName()); public static final ConsumerId DEFAULT_PUBLIC_CONSUMER_ID = toConsumerId("default"); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java index c51970ce3ae..9ddd7885fcb 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java @@ -33,6 +33,7 @@ import static java.util.stream.Collectors.toMap; * @author gjoranv */ public class ApplicationMetricsRetriever extends AbstractComponent { + private static final Logger log = Logger.getLogger(ApplicationMetricsRetriever.class.getName()); private static final int PARALLELISM = 20; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java index f2ee326029a..01cf6b19836 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java @@ -32,6 +32,7 @@ import static java.util.Collections.emptyList; * @author gjoranv */ public class NodeMetricsClient { + private static final Logger log = Logger.getLogger(NodeMetricsClient.class.getName()); static final Duration METRICS_TTL = Duration.ofSeconds(30); @@ -80,7 +81,6 @@ public class NodeMetricsClient { return snapshotsRetrieved; } - /** * Convenience class for storing a metrics snapshot with its timestamp. */ diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ConsumerId.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ConsumerId.java index 62de9649bb0..795d1005b10 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ConsumerId.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ConsumerId.java @@ -7,6 +7,7 @@ import java.util.Objects; * @author gjoranv */ public class ConsumerId { + public final String id; private ConsumerId(String id) { this.id = id; } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/CpuJiffies.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/CpuJiffies.java index 481068f0df2..e07a67770bc 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/CpuJiffies.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/CpuJiffies.java @@ -2,6 +2,7 @@ package ai.vespa.metricsproxy.service; class CpuJiffies { + private int cpuId; private long jiffies; @@ -37,4 +38,5 @@ class CpuJiffies { public long getTotalJiffies() { return jiffies; } + } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java index 922a2a15ffd..9068be81b65 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java @@ -20,6 +20,7 @@ import java.util.logging.Logger; * @author bjorncs */ public abstract class HttpMetricFetcher { + private final static Logger log = Logger.getLogger(HttpMetricFetcher.class.getPackage().getName()); public final static String STATE_PATH = "/state/v1/"; // The call to apache will do 3 retries. As long as we check the services in series, we can't have this too high. @@ -31,8 +32,8 @@ public abstract class HttpMetricFetcher { /** - * @param service The service to fetch metrics from - * @param port The port to use + * @param service the service to fetch metrics from + * @param port the port to use */ HttpMetricFetcher(VespaService service, int port, String path) { this.service = service; @@ -86,4 +87,5 @@ public abstract class HttpMetricFetcher { .build()) .build(); } + } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java index 379e5296bb8..c8fbc83eb59 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java @@ -24,6 +24,7 @@ import java.util.logging.Logger; * @author Eirik Nygaard */ public class SystemPoller { + final private static Logger log = Logger.getLogger(SystemPoller.class.getName()); private final int pollingIntervalSecs; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java index 42c8a13e626..2afc0267434 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java @@ -6,6 +6,7 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.log.LogLevel; import com.yahoo.system.execution.ProcessExecutor; import com.yahoo.system.execution.ProcessResult; +import com.yahoo.vespa.defaults.Defaults; import org.apache.velocity.VelocityContext; import org.apache.velocity.app.VelocityEngine; @@ -24,6 +25,7 @@ public class Telegraf extends AbstractComponent { private static final String TELEGRAF_CONFIG_PATH = "/etc/telegraf/telegraf.conf"; private static final String TELEGRAF_CONFIG_TEMPLATE_PATH = "templates/telegraf.conf.vm"; + private static final String TELEGRAF_LOG_FILE_PATH = Defaults.getDefaults().underVespaHome("logs/telegraf/telegraf.log"); private final TelegrafRegistry telegrafRegistry; private static final Logger logger = Logger.getLogger(Telegraf.class.getName()); @@ -38,6 +40,7 @@ public class Telegraf extends AbstractComponent { protected static void writeConfig(TelegrafConfig telegrafConfig, Writer writer) { VelocityContext context = new VelocityContext(); + context.put("logFilePath", TELEGRAF_LOG_FILE_PATH); context.put("intervalSeconds", telegrafConfig.intervalSeconds()); context.put("cloudwatchPlugins", telegrafConfig.cloudWatch()); // TODO: Add node cert if hosted diff --git a/metrics-proxy/src/main/resources/templates/telegraf.conf.vm b/metrics-proxy/src/main/resources/templates/telegraf.conf.vm index c427ee1ce4b..e99bab8b02d 100644 --- a/metrics-proxy/src/main/resources/templates/telegraf.conf.vm +++ b/metrics-proxy/src/main/resources/templates/telegraf.conf.vm @@ -9,7 +9,7 @@ flush_jitter = "0s" precision = "" logtarget = "file" - logfile = "/var/log/telegraf/telegraf.log" + logfile = "$logFilePath" logfile_rotation_interval = "1d" logfile_rotation_max_size = "20MB" logfile_rotation_max_archives = 5 diff --git a/metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt b/metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt index 85656465901..accd2cc87eb 100644 --- a/metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt +++ b/metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt @@ -9,7 +9,7 @@ flush_jitter = "0s" precision = "" logtarget = "file" - logfile = "/var/log/telegraf/telegraf.log" + logfile = "/opt/vespa/logs/telegraf/telegraf.log" logfile_rotation_interval = "1d" logfile_rotation_max_size = "20MB" logfile_rotation_max_archives = 5 diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index c790e73037e..782f8592350 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -66,6 +66,9 @@ public class DockerOperationsImpl implements DockerOperations { .withHostName(context.node().hostname()) .withResources(containerResources) .withManagedBy(MANAGER_NAME) + // The inet6 option is needed to prefer AAAA records with gethostbyname(3), used by (at least) a yca package + // TODO: Try to remove this + .withDnsOption("inet6") .withUlimit("nofile", 262_144, 262_144) // The nproc aka RLIMIT_NPROC resource limit works as follows: // - A process has a (soft) nproc limit, either inherited by the parent or changed with setrlimit(2). diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java index e2ad9e3de97..69bc9f5e092 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java @@ -181,6 +181,11 @@ public class DockerMock implements Docker { } @Override + public CreateContainerCommand withDnsOption(String dnsOption) { + return this; + } + + @Override public CreateContainerCommand withPrivileged(boolean privileged) { return this; } diff --git a/node-repository/src/main/config/node-repository.xml b/node-repository/src/main/config/node-repository.xml index 274be6d572a..186f052a274 100644 --- a/node-repository/src/main/config/node-repository.xml +++ b/node-repository/src/main/config/node-repository.xml @@ -1,6 +1,8 @@ <!-- services.xml snippet for the node repository. Included in config server services.xml if the package is installed--> <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <component id="com.yahoo.vespa.hosted.provision.provisioning.InfraDeployerImpl" bundle="node-repository"/> +<component id="com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsFetcher" bundle="node-repository"/> +<component id="com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb" bundle="node-repository"/> <component id="com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner" bundle="node-repository" /> <component id="NodeRepository" class="com.yahoo.vespa.hosted.provision.NodeRepository" bundle="node-repository"/> <component id="com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance" bundle="node-repository"/> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 7c0e0e7868b..f881f888752 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -83,7 +83,7 @@ public final class Node { this.reservedTo = Objects.requireNonNull(reservedTo, "reservedTo cannot be null"); if (state == State.active) - requireNonEmpty(ipConfig.primary(), "An active node must have at least one valid IP address"); + requireNonEmpty(ipConfig.primary(), "Active node " + hostname + " must have at least one valid IP address"); if (parentHostname.isPresent()) { if (!ipConfig.pool().asSet().isEmpty()) throw new IllegalArgumentException("A child node cannot have an IP address pool"); @@ -375,8 +375,6 @@ public final class Node { .deviation(); } - - @Override public boolean equals(Object o) { if (this == o) return true; @@ -430,8 +428,13 @@ public final class Node { /** Returns whether this is a state where the node is assigned to an application */ public boolean isAllocated() { - return this == reserved || this == active || this == inactive || this == failed || this == parked; + return allocatedStates().contains(this); } + + public static Set<State> allocatedStates() { + return Set.of(reserved, active, inactive, failed, parked); + } + } /** The mean and mean deviation (squared difference) of a bunch of numbers */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java new file mode 100644 index 00000000000..71f7dc3701e --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -0,0 +1,185 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.autoscale; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; +import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceLimits; + +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * The autoscaler makes decisions about the flavor and node count that should be allocated to a cluster + * based on observed behavior. + * + * @author bratseth + */ +public class Autoscaler { + + /* + TODO: + - Scale group size + - Have a better idea about whether we have sufficient information to make decisions + - Consider taking spikes/variance into account + - Measure observed regulation lag (startup+redistribution) into account when deciding regulation observation window + - Test AutoscalingMaintainer + - Scale by performance not just load+cost + */ + + private static final int minimumMeasurements = 500; // TODO: Per node instead? Also say something about interval? + + /** What cost difference factor warrants reallocation? */ + private static final double costDifferenceRatioWorthReallocation = 0.1; + /** What difference factor from ideal (for any resource) warrants a change? */ + private static final double idealDivergenceWorthReallocation = 0.1; + + // We only depend on the ratios between these values + private static final double cpuUnitCost = 12.0; + private static final double memoryUnitCost = 1.2; + private static final double diskUnitCost = 0.045; + + private final HostResourcesCalculator hostResourcesCalculator; + private final NodeMetricsDb metricsDb; + private final NodeRepository nodeRepository; + private final NodeResourceLimits nodeResourceLimits; + + public Autoscaler(HostResourcesCalculator hostResourcesCalculator, + NodeMetricsDb metricsDb, + NodeRepository nodeRepository) { + this.hostResourcesCalculator = hostResourcesCalculator; + this.metricsDb = metricsDb; + this.nodeRepository = nodeRepository; + this.nodeResourceLimits = new NodeResourceLimits(nodeRepository.zone()); + } + + public Optional<ClusterResources> autoscale(ApplicationId applicationId, ClusterSpec cluster, List<Node> clusterNodes) { + if (clusterNodes.stream().anyMatch(node -> node.status().wantToRetire() || + node.allocation().get().membership().retired() || + node.allocation().get().isRemovable())) + return Optional.empty(); // Don't autoscale clusters that are in flux + ClusterResources currentAllocation = new ClusterResources(clusterNodes); + Optional<Double> cpuLoad = averageLoad(Resource.cpu, cluster, clusterNodes); + Optional<Double> memoryLoad = averageLoad(Resource.memory, cluster, clusterNodes); + Optional<Double> diskLoad = averageLoad(Resource.disk, cluster, clusterNodes); + if (cpuLoad.isEmpty() || memoryLoad.isEmpty() || diskLoad.isEmpty()) return Optional.empty(); + + Optional<ClusterResourcesWithCost> bestAllocation = findBestAllocation(cpuLoad.get(), + memoryLoad.get(), + diskLoad.get(), + currentAllocation, + cluster); + if (bestAllocation.isEmpty()) return Optional.empty(); + + if (closeToIdeal(Resource.cpu, cpuLoad.get()) && + closeToIdeal(Resource.memory, memoryLoad.get()) && + closeToIdeal(Resource.disk, diskLoad.get()) && + similarCost(bestAllocation.get().cost(), currentAllocation.nodes() * costOf(currentAllocation.nodeResources()))) + return Optional.empty(); // Avoid small, unnecessary changes + return bestAllocation.map(a -> a.clusterResources()); + } + + private Optional<ClusterResourcesWithCost> findBestAllocation(double cpuLoad, double memoryLoad, double diskLoad, + ClusterResources currentAllocation, ClusterSpec cluster) { + Optional<ClusterResourcesWithCost> bestAllocation = Optional.empty(); + for (ResourceIterator i = new ResourceIterator(cpuLoad, memoryLoad, diskLoad, currentAllocation); i.hasNext(); ) { + ClusterResources allocation = i.next(); + Optional<ClusterResourcesWithCost> allocatableResources = toAllocatableResources(allocation, cluster); + if (allocatableResources.isEmpty()) continue; + if (bestAllocation.isEmpty() || allocatableResources.get().cost() < bestAllocation.get().cost()) + bestAllocation = allocatableResources; + } + return bestAllocation; + } + + private boolean similarCost(double cost1, double cost2) { + return similar(cost1, cost2, costDifferenceRatioWorthReallocation); + } + + private boolean closeToIdeal(Resource resource, double value) { + return similar(resource.idealAverageLoad(), value, idealDivergenceWorthReallocation); + } + + private boolean similar(double r1, double r2, double threshold) { + return Math.abs(r1 - r2) / r1 < threshold; + } + + /** + * Returns the smallest allocatable node resources larger than the given node resources, + * or empty if none available. + */ + private Optional<ClusterResourcesWithCost> toAllocatableResources(ClusterResources resources, ClusterSpec cluster) { + if (allowsHostSharing(nodeRepository.zone().cloud())) { + // Return the requested resources, adjusted to be legal or empty if they cannot fit on existing hosts + NodeResources nodeResources = nodeResourceLimits.enlargeToLegal(resources.nodeResources(), cluster.type()); + for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors()) + if (flavor.resources().satisfies(nodeResources)) + return Optional.of(new ClusterResourcesWithCost(resources.with(nodeResources), + costOf(nodeResources) * resources.nodes())); + return Optional.empty(); + } + else { + // return the cheapest flavor satisfying the target resources, if any + double bestCost = Double.MAX_VALUE; + Optional<Flavor> bestFlavor = Optional.empty(); + for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors()) { + if ( ! flavor.resources().satisfies(resources.nodeResources())) continue; + if (bestFlavor.isEmpty() || bestCost > costOf(flavor.resources())) { + bestFlavor = Optional.of(flavor); + bestCost = costOf(flavor); + } + } + if (bestFlavor.isEmpty()) + return Optional.empty(); + else + return Optional.of(new ClusterResourcesWithCost(resources.with(bestFlavor.get().resources()), + bestCost * resources.nodes())); + } + } + + /** + * Returns the average load of this resource in the measurement window, + * or empty if we are not in a position to make decisions from these measurements at this time. + */ + private Optional<Double> averageLoad(Resource resource, ClusterSpec cluster, List<Node> clusterNodes) { + NodeMetricsDb.Window window = metricsDb.getWindow(nodeRepository.clock().instant().minus(scalingWindow(cluster.type())), + resource, + clusterNodes.stream().map(Node::hostname).collect(Collectors.toList())); + + if (window.measurementCount() < minimumMeasurements) return Optional.empty(); + if (window.hostnames() != clusterNodes.size()) return Optional.empty(); // Regulate only when all nodes are measured + + return Optional.of(window.average()); + } + + /** The duration of the window we need to consider to make a scaling decision */ + private Duration scalingWindow(ClusterSpec.Type clusterType) { + if (clusterType.isContent()) return Duration.ofHours(12); // Ideally we should use observed redistribution time + return Duration.ofHours(12); // TODO: Measure much more often to get this down to minutes. And, ideally we should take node startup time into account + } + + // TODO: Put this in zone config instead? + private boolean allowsHostSharing(CloudName cloudName) { + if (cloudName.value().equals("aws")) return false; + return true; + } + + private double costOf(Flavor flavor) { + NodeResources chargedResources = hostResourcesCalculator.availableCapacityOf(flavor.name(), flavor.resources()); + return costOf(chargedResources); + } + + private double costOf(NodeResources resources) { + return resources.vcpu() * cpuUnitCost + + resources.memoryGb() * memoryUnitCost + + resources.diskGb() * diskUnitCost; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResources.java new file mode 100644 index 00000000000..e068b4404d8 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResources.java @@ -0,0 +1,65 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.autoscale; + +import com.yahoo.config.provision.NodeResources; +import com.yahoo.vespa.hosted.provision.Node; + +import java.util.List; +import java.util.Objects; + +/** A description of the resources of a cluster */ +public class ClusterResources { + + /** The node count in the cluster */ + private final int nodes; + + /** The number of node groups in the cluster */ + private final int groups; + + /** The resources of each node in the cluster */ + private final NodeResources nodeResources; + + public ClusterResources(List<Node> nodes) { + this(nodes.size(), + (int)nodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(), + nodes.get(0).flavor().resources()); + } + + public ClusterResources(int nodes, int groups, NodeResources nodeResources) { + this.nodes = nodes; + this.groups = groups; + this.nodeResources = nodeResources; + } + + /** Returns the total number of allocated nodes (over all groups) */ + public int nodes() { return nodes; } + public int groups() { return groups; } + public NodeResources nodeResources() { return nodeResources; } + + public ClusterResources with(NodeResources resources) { + return new ClusterResources(nodes, groups, resources); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof ClusterResources)) return false; + + ClusterResources other = (ClusterResources)o; + if (other.nodes != this.nodes) return false; + if (other.groups != this.groups) return false; + if (other.nodeResources != this.nodeResources) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(nodes, groups, nodeResources); + } + + @Override + public String toString() { + return "cluster resources: " + nodes + " * " + nodeResources + (groups > 1 ? " in " + groups + " groups" : ""); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java new file mode 100644 index 00000000000..55b28ef3ce1 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterResourcesWithCost.java @@ -0,0 +1,26 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.autoscale; + +/** + * @author bratseth + */ +public class ClusterResourcesWithCost { + + private final ClusterResources resources; + private final double cost; + + public ClusterResourcesWithCost(ClusterResources resources, double cost) { + this.resources = resources; + this.cost = cost; + } + + public ClusterResources clusterResources() { return resources;} + + public double cost() { return cost; } + + @Override + public String toString() { + return "$" + cost + ": " + clusterResources(); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java new file mode 100644 index 00000000000..a599606c314 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java @@ -0,0 +1,74 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.autoscale; + +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.ObjectTraverser; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Consumes a response from the metrics/v2 API and populates the fields of this with the resulting values + * + * @author bratseth + */ +public class MetricsResponse { + + private final List<NodeMetrics.MetricValue> metricValues = new ArrayList<>(); + + public MetricsResponse(byte[] response) { + this(SlimeUtils.jsonToSlime(response)); + } + + public MetricsResponse(String response) { + this(SlimeUtils.jsonToSlime(response)); + } + + public List<NodeMetrics.MetricValue> metrics() { return metricValues; } + + private MetricsResponse(Slime response) { + Inspector root = response.get(); + Inspector nodes = root.field("nodes"); + nodes.traverse((ArrayTraverser)(__, node) -> consumeNode(node)); + } + + private void consumeNode(Inspector node) { + String hostname = node.field("hostname").asString(); + consumeNodeMetrics(hostname, node.field("node")); + consumeServiceMetrics(hostname, node.field("services")); + } + + private void consumeNodeMetrics(String hostname, Inspector node) { + long timestamp = node.field("timestamp").asLong(); + Map<String, Double> values = consumeMetrics(node.field("metrics")); + for (Resource resource : Resource.values()) + addMetricIfPresent(hostname, resource.metricName(), timestamp, values); + } + + private void addMetricIfPresent(String hostname, String metricName, long timestamp, Map<String, Double> values) { + if (values.containsKey(metricName)) + metricValues.add(new NodeMetrics.MetricValue(hostname, metricName, timestamp, values.get(metricName).floatValue())); + } + + private void consumeServiceMetrics(String hostname, Inspector node) { + String name = node.field("name").asString(); + long timestamp = node.field("timestamp").asLong(); + Map<String, Double> values = consumeMetrics(node.field("metrics")); + } + + private Map<String, Double> consumeMetrics(Inspector metrics) { + Map<String, Double> values = new HashMap<>(); + metrics.traverse((ArrayTraverser) (__, item) -> consumeMetricsItem(item, values)); + return values; + } + + private void consumeMetricsItem(Inspector item, Map<String, Double> values) { + item.field("values").traverse((ObjectTraverser)(name, value) -> values.put(name, value.asDouble())); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java new file mode 100644 index 00000000000..97ac1e72be9 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java @@ -0,0 +1,48 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.autoscale; + +import com.yahoo.config.provision.ApplicationId; + +import java.util.Collection; + +/** + * Interface to retrieve metrics on (tenant) nodes. + * + * @author bratseth + */ +public interface NodeMetrics { + + /** + * Fetches metrics for an application. This call may be expensive. + * + * @param application the application to fetch metrics from + */ + Collection<MetricValue> fetchMetrics(ApplicationId application); + + final class MetricValue { + + private final String hostname; + private final String name; + private long timestamp; + private final float value; + + public MetricValue(String hostname, String name, long timestamp, float value) { + this.hostname = hostname; + this.name = name; + this.timestamp = timestamp; + this.value = value; + } + + public String hostname() { return hostname; } + public String name() { return name; } + public long timestamp() { return timestamp; } + public float value() { return value; } + + @Override + public String toString() { + return "metric value " + name + ": " + value + " at " + timestamp + " for " + hostname; + } + + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java new file mode 100644 index 00000000000..14a35e3efbc --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java @@ -0,0 +1,169 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.autoscale; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * An in-memory time-series "database" of node metrics. + * Thread model: One writer, many readers. + * + * @author bratseth + */ +public class NodeMetricsDb { + + private static final Duration dbWindow = Duration.ofHours(24); + + /** Measurements by key. Each list of measurements is sorted by increasing timestamp */ + private Map<MeasurementKey, List<Measurement>> db = new HashMap<>(); + + /** Lock all access for now since we modify lists inside a map */ + private final Object lock = new Object(); + + /** Add a measurement to this */ + public void add(Collection<NodeMetrics.MetricValue> metricValues) { + synchronized (lock) { + for (var value : metricValues) { + List<Measurement> measurements = db.computeIfAbsent(new MeasurementKey(value.hostname(), + Resource.fromMetric(value.name())), + (__) -> new ArrayList<>()); + measurements.add(new Measurement(value.timestamp(), value.value())); + } + } + } + + /** Must be called intermittently (as long as add is called) to gc old measurements */ + public void gc(Clock clock) { + synchronized (lock) { + // TODO: We may need to do something more complicated to avoid spending too much memory to + // lower the measurement interval (see NodeRepositoryMaintenance) + // Each measurement is Object + long + float = 16 + 8 + 4 = 28 bytes + // 24 hours with 1k nodes and 3 resources and 1 measurement/sec is about 10Gb + + long oldestTimestamp = clock.instant().minus(dbWindow).toEpochMilli(); + for (Iterator<List<Measurement>> i = db.values().iterator(); i.hasNext(); ) { + List<Measurement> measurements = i.next(); + + while (!measurements.isEmpty() && measurements.get(0).timestamp < oldestTimestamp) + measurements.remove(0); + + if (measurements.isEmpty()) + i.remove(); + } + } + } + + /** Returns a window within which we can ask for specific information from this db */ + public Window getWindow(Instant startTime, Resource resource, List<String> hostnames) { + return new Window(startTime, resource, hostnames); + } + + public class Window { + + private final long startTime; + private List<MeasurementKey> keys; + + private Window(Instant startTime, Resource resource, List<String> hostnames) { + this.startTime = startTime.toEpochMilli(); + keys = hostnames.stream().map(hostname -> new MeasurementKey(hostname, resource)).collect(Collectors.toList()); + } + + public int measurementCount() { + synchronized (lock) { + return (int) keys.stream() + .flatMap(key -> db.getOrDefault(key, List.of()).stream()) + .filter(measurement -> measurement.timestamp >= startTime) + .count(); + } + } + + /** Returns the count of hostnames which have measurements in this window */ + public int hostnames() { + synchronized (lock) { + int count = 0; + for (MeasurementKey key : keys) { + List<Measurement> measurements = db.get(key); + if (measurements == null || measurements.isEmpty()) continue; + + if (measurements.get(measurements.size() - 1).timestamp >= startTime) + count++; + } + return count; + } + } + + public double average() { + synchronized (lock) { + double sum = 0; + int count = 0; + for (MeasurementKey key : keys) { + List<Measurement> measurements = db.get(key); + if (measurements == null) continue; + + int index = measurements.size() - 1; + while (index >= 0 && measurements.get(index).timestamp >= startTime) { + sum += measurements.get(index).value; + count++; + + index--; + } + } + return sum / count; + } + } + + } + + private static class MeasurementKey { + + private final String hostname; + private final Resource resource; + + public MeasurementKey(String hostname, Resource resource) { + this.hostname = hostname; + this.resource = resource; + } + + @Override + public int hashCode() { + return Objects.hash(hostname, resource); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if ( ! (o instanceof MeasurementKey)) return false; + MeasurementKey other = (MeasurementKey)o; + if ( ! this.hostname.equals(other.hostname)) return false; + if ( ! this.resource.equals(other.resource)) return false; + return true; + } + + } + + private static class Measurement { + + /** The time of this measurement in epoch millis */ + private final long timestamp; + + /** The measured value */ + private final float value; + + public Measurement(long timestamp, float value) { + this.timestamp = timestamp; + this.value = value; + } + + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java new file mode 100644 index 00000000000..54d8eac238f --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java @@ -0,0 +1,112 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.autoscale; + +import ai.vespa.util.http.VespaHttpClientBuilder; +import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.orchestrator.HostNameNotFoundException; +import com.yahoo.vespa.orchestrator.Orchestrator; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.BasicResponseHandler; +import org.apache.http.impl.client.CloseableHttpClient; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Collection; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Fetches node metrics over the metrics/v2 API + * + * @author bratseth + */ +public class NodeMetricsFetcher extends AbstractComponent implements NodeMetrics { + + private static final Logger log = Logger.getLogger(NodeMetricsFetcher.class.getName()); + + private static final String apiPath = "/metrics/v2/values"; + + private final NodeRepository nodeRepository; + private final Orchestrator orchestrator; + private final HttpClient httpClient; + + @Inject + public NodeMetricsFetcher(NodeRepository nodeRepository, Orchestrator orchestrator) { + this(nodeRepository, orchestrator, new ApacheHttpClient()); + } + + NodeMetricsFetcher(NodeRepository nodeRepository, Orchestrator orchestrator, HttpClient httpClient) { + this.nodeRepository = nodeRepository; + this.orchestrator = orchestrator; + this.httpClient = httpClient; + } + + @Override + public Collection<MetricValue> fetchMetrics(ApplicationId application) { + Node metricsV2Container = nodeRepository.list() + .owner(application) + .state(Node.State.active) + .container() + .filter(node -> expectedUp(node)) + .asList().get(0); + String url = "http://" + metricsV2Container.hostname() + ":" + 4080 + apiPath + "?consumer=vespa-consumer-metrics"; + String response = httpClient.get(url); + return new MetricsResponse(response).metrics(); + } + + @Override + public void deconstruct() { + httpClient.close(); + } + + private boolean expectedUp(Node node) { + try { + return ! orchestrator.getNodeStatus(new HostName(node.hostname())).isSuspended(); + } + catch (HostNameNotFoundException e) { + return false; + } + } + + /** The simplest possible http client interface */ + public interface HttpClient { + + String get(String url); + void close(); + + } + + /** Implements the HttpClient interface by delegating to an Apache HTTP client */ + public static class ApacheHttpClient implements HttpClient { + + private final CloseableHttpClient httpClient = VespaHttpClientBuilder.createWithBasicConnectionManager().build(); + + @Override + public String get(String url) { + try { + return httpClient.execute(new HttpGet(url), new BasicResponseHandler()); + } + catch (IOException e) { + throw new UncheckedIOException("Could not get " + url, e); + } + } + + @Override + public void close() { + try { + httpClient.close(); + } + catch (IOException e) { + log.log(Level.WARNING, "Exception deconstructing", e); + } + } + + + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java new file mode 100644 index 00000000000..9c85ca870d5 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java @@ -0,0 +1,44 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.autoscale; + +import com.yahoo.config.provision.NodeResources; + +/** + * A resource subject to autoscaling + * + * @author bratseth + */ +public enum Resource { + + cpu { + String metricName() { return "cpu.util"; } + double idealAverageLoad() { return 0.2; } + double valueFrom(NodeResources resources) { return resources.vcpu(); } + }, + + memory { + String metricName() { return "memory.util"; } + double idealAverageLoad() { return 0.7; } + double valueFrom(NodeResources resources) { return resources.memoryGb(); } + }, + + disk { + String metricName() { return "disk.util"; } + double idealAverageLoad() { return 0.7; } + double valueFrom(NodeResources resources) { return resources.diskGb(); } + }; + + abstract String metricName(); + + /** The load we should have of this resource on average, when one node in the cluster is down */ + abstract double idealAverageLoad(); + + abstract double valueFrom(NodeResources resources); + + public static Resource fromMetric(String metricName) { + for (Resource resource : values()) + if (resource.metricName().equals(metricName)) return resource; + throw new IllegalArgumentException("Metric '" + metricName + "' does not map to a resource"); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java new file mode 100644 index 00000000000..464fe570b95 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java @@ -0,0 +1,104 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.autoscale; + +import com.yahoo.config.provision.NodeResources; + +/** + * Provides iteration over possible cluster resource allocations given a target total load + * and current groups/nodes allocation. + */ +public class ResourceIterator { + + // Configured min and max nodes TODO: These should come from the application package + private static final int minimumNodesPerCluster = 3; // Since this is with redundancy it cannot be lower than 2 + private static final int maximumNodesPerCluster = 150; + + // When a query is issued on a node the cost is the sum of a fixed cost component and a cost component + // proportional to document count. We must account for this when comparing configurations with more or fewer nodes. + // TODO: Measure this, and only take it into account with queries + private static final double fixedCpuCostFraction = 0.1; + + // Describes the observed state + private final ClusterResources allocation; + private final double cpuLoad; + private final double memoryLoad; + private final double diskLoad; + private final int groupSize; + + // Derived from the observed state + private final int nodeIncrement; + private final boolean singleGroupMode; + + // Iterator state + private int currentNodes; + + public ResourceIterator(double cpuLoad, double memoryLoad, double diskLoad, ClusterResources currentAllocation) { + this.cpuLoad = cpuLoad; + this.memoryLoad = memoryLoad; + this.diskLoad = diskLoad; + + // ceil: If the division does not produce a whole number we assume some node is missing + groupSize = (int)Math.ceil((double)currentAllocation.nodes() / currentAllocation.groups()); + allocation = currentAllocation; + + // What number of nodes is it effective to add or remove at the time from this cluster? + // This is the group size, since we (for now) assume the group size is decided by someone wiser than us + // and we decide the number of groups. + // The exception is when we only have one group, where we can add and remove single nodes in it. + singleGroupMode = currentAllocation.groups() == 1; + nodeIncrement = singleGroupMode ? 1 : groupSize; + + currentNodes = currentAllocation.nodes(); + while (currentNodes - nodeIncrement >= minimumNodesPerCluster + && (singleGroupMode || currentNodes - nodeIncrement > groupSize)) // group level redundancy + currentNodes -= nodeIncrement; + } + + public ClusterResources next() { + int nodesWithRedundancy = currentNodes - (singleGroupMode ? 1 : groupSize); + ClusterResources next = new ClusterResources(currentNodes, + singleGroupMode ? 1 : currentNodes / groupSize, + resourcesFor(nodesWithRedundancy)); + currentNodes += nodeIncrement; + return next; + } + + public boolean hasNext() { + return currentNodes <= maximumNodesPerCluster; + } + + /** + * For the observed load this instance is initialized with, returns the resources needed per node to be at + * ideal load given a target node count + */ + private NodeResources resourcesFor(int nodeCount) { + // Cpu: Scales with cluster size (TODO: Only reads, writes scales with group size) + // Memory and disk: Scales with group size + + double cpu, memory, disk; + if (singleGroupMode) { + // The fixed cost portion of cpu does not scale with changes to the node count + // TODO: Only for the portion of cpu consumed by queries + double totalCpu = totalUsage(Resource.cpu, cpuLoad); + cpu = fixedCpuCostFraction * totalCpu / groupSize / Resource.cpu.idealAverageLoad() + + (1 - fixedCpuCostFraction) * totalCpu / nodeCount / Resource.cpu.idealAverageLoad(); + memory = totalGroupUsage(Resource.memory, memoryLoad) / nodeCount / Resource.memory.idealAverageLoad(); + disk = totalGroupUsage(Resource.disk, diskLoad) / nodeCount / Resource.disk.idealAverageLoad(); + } + else { + cpu = totalUsage(Resource.cpu, cpuLoad) / nodeCount / Resource.cpu.idealAverageLoad(); + memory = totalGroupUsage(Resource.memory, memoryLoad) / groupSize / Resource.memory.idealAverageLoad(); + disk = totalGroupUsage(Resource.disk, diskLoad) / groupSize / Resource.disk.idealAverageLoad(); + } + return allocation.nodeResources().withVcpu(cpu).withMemoryGb(memory).withDiskGb(disk); + } + + private double totalUsage(Resource resource, double load) { + return load * resource.valueFrom(allocation.nodeResources()) * allocation.nodes(); + } + + private double totalGroupUsage(Resource resource, double load) { + return load * resource.valueFrom(allocation.nodeResources()) * groupSize; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java new file mode 100644 index 00000000000..f3c1d8603b3 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -0,0 +1,62 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Deployer; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; +import com.yahoo.vespa.hosted.provision.autoscale.ClusterResources; +import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Maintainer making automatic scaling decisions + * + * @author bratseth + */ +public class AutoscalingMaintainer extends Maintainer { + + private final Autoscaler autoscaler; + private final Deployer deployer; + + public AutoscalingMaintainer(NodeRepository nodeRepository, + HostResourcesCalculator hostResourcesCalculator, + NodeMetricsDb metricsDb, + Deployer deployer, + Duration interval) { + super(nodeRepository, interval); + this.autoscaler = new Autoscaler(hostResourcesCalculator, metricsDb, nodeRepository); + this.deployer = deployer; + } + + @Override + protected void maintain() { + if ( ! nodeRepository().zone().environment().isProduction()) return; + + activeNodesByApplication().forEach((applicationId, nodes) -> autoscale(applicationId, nodes)); + } + + private void autoscale(ApplicationId application, List<Node> applicationNodes) { + MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, nodeRepository()); + if ( ! deployment.isValid()) return; // Another config server will consider this application + nodesByCluster(applicationNodes).forEach((clusterSpec, clusterNodes) -> { + Optional<ClusterResources> target = autoscaler.autoscale(application, clusterSpec, clusterNodes); + target.ifPresent(t -> log.info("Autoscale: Application " + application + " cluster " + clusterSpec + + " from " + applicationNodes.size() + " * " + applicationNodes.get(0).flavor().resources() + + " to " + t.nodes() + " * " + t.nodeResources())); + }); + } + + private Map<ClusterSpec, List<Node>> nodesByCluster(List<Node> applicationNodes) { + return applicationNodes.stream().collect(Collectors.groupingBy(n -> n.allocation().get().membership().cluster())); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java index 0d5a8587902..27fba9e8f8e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java @@ -2,17 +2,22 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.component.AbstractComponent; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import java.time.Duration; import java.time.Instant; import java.util.List; +import java.util.Map; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; /** * A maintainer is some job which runs at a fixed rate to perform some maintenance task on the node repo. @@ -75,6 +80,12 @@ public abstract class Maintainer extends AbstractComponent implements Runnable { private String name() { return this.getClass().getSimpleName(); } + /** A utility to group active tenant applications by application */ + protected Map<ApplicationId, List<Node>> activeNodesByApplication() { + return nodeRepository().list().nodeType(NodeType.tenant).state(Node.State.active).asList() + .stream().collect(Collectors.groupingBy(n -> n.allocation().get().owner())); + } + static long staggeredDelay(List<HostName> cluster, HostName host, Instant now, Duration interval) { if ( ! cluster.contains(host)) return interval.toMillis(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java new file mode 100644 index 00000000000..178e8385008 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java @@ -0,0 +1,49 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; + +import java.time.Duration; +import java.util.logging.Level; + +/** + * Maintainer which keeps the node metric db up to date by periodically fetching metrics from all + * active nodes. + * + * @author bratseth + */ +public class NodeMetricsDbMaintainer extends Maintainer { + + private static final int maxWarningsPerInvocation = 2; + + private final NodeMetrics nodeMetrics; + private final NodeMetricsDb nodeMetricsDb; + + public NodeMetricsDbMaintainer(NodeRepository nodeRepository, + NodeMetrics nodeMetrics, + NodeMetricsDb nodeMetricsDb, + Duration interval) { + super(nodeRepository, interval); + this.nodeMetrics = nodeMetrics; + this.nodeMetricsDb = nodeMetricsDb; + } + + @Override + protected void maintain() { + int warnings = 0; + for (ApplicationId application : activeNodesByApplication().keySet()) { + try { + nodeMetricsDb.add(nodeMetrics.fetchMetrics(application)); + } + catch (Exception e) { + if (warnings++ < maxWarningsPerInvocation) + log.log(Level.WARNING, "Could not update metrics for " + application, e); + } + } + nodeMetricsDb.gc(nodeRepository().clock()); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 37620e17a95..ecc550527fc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -8,9 +8,11 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostLivenessTracker; import com.yahoo.config.provision.InfraDeployer; import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; import com.yahoo.vespa.hosted.provision.provisioning.ProvisionServiceProvider; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.service.monitor.ServiceMonitor; @@ -48,22 +50,25 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final CapacityReportMaintainer capacityReportMaintainer; private final OsUpgradeActivator osUpgradeActivator; private final Rebalancer rebalancer; + private final NodeMetricsDbMaintainer nodeMetricsDbMaintainer; + private final AutoscalingMaintainer autoscalingMaintainer; @SuppressWarnings("unused") @Inject public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, InfraDeployer infraDeployer, HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor, Zone zone, Orchestrator orchestrator, Metric metric, - ProvisionServiceProvider provisionServiceProvider, - FlagSource flagSource) { + ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource, + NodeMetrics nodeMetrics, NodeMetricsDb nodeMetricsDb) { this(nodeRepository, deployer, infraDeployer, hostLivenessTracker, serviceMonitor, zone, Clock.systemUTC(), - orchestrator, metric, provisionServiceProvider, flagSource); + orchestrator, metric, provisionServiceProvider, flagSource, nodeMetrics, nodeMetricsDb); } public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, InfraDeployer infraDeployer, HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor, Zone zone, Clock clock, Orchestrator orchestrator, Metric metric, - ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource) { + ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource, + NodeMetrics nodeMetrics, NodeMetricsDb nodeMetricsDb) { DefaultTimes defaults = new DefaultTimes(zone); nodeFailer = new NodeFailer(deployer, hostLivenessTracker, serviceMonitor, nodeRepository, defaults.failGrace, clock, orchestrator, throttlePolicyFromEnv().orElse(defaults.throttlePolicy), metric); @@ -85,6 +90,8 @@ public class NodeRepositoryMaintenance extends AbstractComponent { capacityReportMaintainer = new CapacityReportMaintainer(nodeRepository, metric, defaults.capacityReportInterval); osUpgradeActivator = new OsUpgradeActivator(nodeRepository, defaults.osUpgradeActivatorInterval); rebalancer = new Rebalancer(deployer, nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), provisionServiceProvider.getHostProvisioner(), metric, clock, defaults.rebalancerInterval); + nodeMetricsDbMaintainer = new NodeMetricsDbMaintainer(nodeRepository, nodeMetrics, nodeMetricsDb, defaults.nodeMetricsCollectionInterval); + autoscalingMaintainer = new AutoscalingMaintainer(nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), nodeMetricsDb, deployer, defaults.autoscalingInterval); // The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now infrastructureProvisioner.maintainButThrowOnException(); @@ -109,6 +116,8 @@ public class NodeRepositoryMaintenance extends AbstractComponent { dynamicProvisioningMaintainer.ifPresent(Maintainer::deconstruct); osUpgradeActivator.deconstruct(); rebalancer.deconstruct(); + nodeMetricsDbMaintainer.deconstruct(); + autoscalingMaintainer.deconstruct(); } private static Optional<NodeFailer.ThrottlePolicy> throttlePolicyFromEnv() { @@ -149,6 +158,8 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final Duration dynamicProvisionerInterval; private final Duration osUpgradeActivatorInterval; private final Duration rebalancerInterval; + private final Duration nodeMetricsCollectionInterval; + private final Duration autoscalingInterval; private final NodeFailer.ThrottlePolicy throttlePolicy; @@ -169,6 +180,8 @@ public class NodeRepositoryMaintenance extends AbstractComponent { dynamicProvisionerInterval = Duration.ofMinutes(5); osUpgradeActivatorInterval = zone.system().isCd() ? Duration.ofSeconds(30) : Duration.ofMinutes(5); rebalancerInterval = Duration.ofMinutes(40); + nodeMetricsCollectionInterval = Duration.ofMinutes(1); + autoscalingInterval = Duration.ofMinutes(5); if (zone.environment().equals(Environment.prod) && ! zone.system().isCd()) { inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index 7c5ff35878b..179d7f2703c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -19,11 +19,15 @@ import java.util.Locale; public class CapacityPolicies { private final Zone zone; + + private final NodeResourceLimits nodeResourceLimits; + /* Deployments must match 1-to-1 the advertised resources of a physical host */ private final boolean isUsingAdvertisedResources; public CapacityPolicies(Zone zone) { this.zone = zone; + this.nodeResourceLimits = new NodeResourceLimits(zone); this.isUsingAdvertisedResources = zone.cloud().value().equals("aws"); } @@ -64,7 +68,7 @@ public class CapacityPolicies { } private void ensureSufficientResources(NodeResources resources, ClusterSpec cluster) { - double minMemoryGb = minMemoryGb(cluster.type()); + double minMemoryGb = nodeResourceLimits.minMemoryGb(cluster.type()); if (resources.memoryGb() >= minMemoryGb) return; throw new IllegalArgumentException(String.format(Locale.ENGLISH, @@ -72,12 +76,6 @@ public class CapacityPolicies { minMemoryGb, cluster.type().name(), cluster.id().value(), resources.memoryGb())); } - private int minMemoryGb(ClusterSpec.Type clusterType) { - if (zone.system() == SystemName.dev) return 1; // Allow small containers in dev system - if (clusterType == ClusterSpec.Type.admin) return 2; - return 4; - } - private NodeResources defaultNodeResources(ClusterSpec.Type clusterType) { if (clusterType == ClusterSpec.Type.admin) { if (zone.system() == SystemName.dev) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 5753bbb3c5a..af6fa8edf64 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -64,7 +64,6 @@ public class GroupPreparer { .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()) .value(); boolean allocateFully = dynamicProvisioningEnabled && preprovisionCapacityFlag.value().isEmpty(); - try (Mutex lock = nodeRepository.lock(application)) { // Lock ready pool to ensure that the same nodes are not simultaneously allocated by others diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java index 394549e4141..0423f762f2b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java @@ -18,7 +18,7 @@ public interface HostProvisioner { /** * Schedule provisioning of a given number of hosts. * - * @param provisionIndexes List of unique provision indexes which will be used to generate the node hostnames + * @param provisionIndexes list of unique provision indexes which will be used to generate the node hostnames * on the form of <code>[prefix][index].[domain]</code> * @param resources the resources needed per node * @param applicationId id of the application that will own the provisioned host diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index ebd6a01e61f..c92f7889496 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -390,4 +390,5 @@ class NodeAllocation { return count; } } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java new file mode 100644 index 00000000000..ca04bf66ce3 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java @@ -0,0 +1,32 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; + +/** + * Defines the resource limits for nodes in various zones + * + * @author bratseth + */ +public class NodeResourceLimits { + + private final Zone zone; + + public NodeResourceLimits(Zone zone) { + this.zone = zone; + } + + public int minMemoryGb(ClusterSpec.Type clusterType) { + if (zone.system() == SystemName.dev) return 1; // Allow small containers in dev system + if (clusterType == ClusterSpec.Type.admin) return 2; + return 4; + } + + public NodeResources enlargeToLegal(NodeResources resources, ClusterSpec.Type clusterType) { + return resources.withMemoryGb(Math.max(minMemoryGb(clusterType), resources.memoryGb())); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index 72be68a7ee3..91c15cdb61b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -121,8 +121,7 @@ class Preparer { */ private int findHighestIndex(ApplicationId application, ClusterSpec cluster) { int highestIndex = -1; - for (Node node : nodeRepository.getNodes(application, - Node.State.active, Node.State.inactive, Node.State.parked, Node.State.failed)) { + for (Node node : nodeRepository.getNodes(application, Node.State.allocatedStates().toArray(new Node.State[0]))) { ClusterSpec nodeCluster = node.allocation().get().membership().cluster(); if ( ! nodeCluster.id().equals(cluster.id())) continue; if ( ! nodeCluster.type().equals(cluster.type())) continue; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java index 49d0ba5cf70..d26accd7a84 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java @@ -25,6 +25,8 @@ public class ContainerConfig { " <component id='com.yahoo.vespa.hosted.provision.testutils.ServiceMonitorStub'/>\n" + " <component id='com.yahoo.vespa.hosted.provision.testutils.MockDuperModel'/>\n" + " <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors'/>\n" + + " <component id='com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb'/>\n" + + " <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeMetrics'/>\n" + " <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository'/>\n" + " <component id='com.yahoo.vespa.hosted.provision.testutils.MockProvisionServiceProvider'/>\n" + " <component id='com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance'/>\n" + diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java index 915ef0d9125..e7ebf049e51 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java @@ -18,6 +18,7 @@ import java.util.concurrent.ConcurrentHashMap; * @author hakonhall */ public class MockDuperModel implements DuperModelInfraApi { + private final Map<ApplicationId, InfraApplicationApi> supportedInfraApps = new HashMap<>(); private final ConcurrentHashMap<ApplicationId, List<HostName>> activeApps = new ConcurrentHashMap<>(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeMetrics.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeMetrics.java new file mode 100644 index 00000000000..d5397aa421c --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeMetrics.java @@ -0,0 +1,20 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.testutils; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics; + +import java.util.ArrayList; +import java.util.Collection; + +/** + * @author bratseth + */ +public class MockNodeMetrics implements NodeMetrics { + + @Override + public Collection<MetricValue> fetchMetrics(ApplicationId application) { + return new ArrayList<>(); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java index ab813ddeb5a..95555185292 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java @@ -59,7 +59,7 @@ public class NodeRepositoryTester { public Node addNode(String id, String hostname, String parentHostname, String flavor, NodeType type) { Node node = nodeRepository.createNode(id, hostname, Optional.of(parentHostname), - nodeFlavors.getFlavorOrThrow(flavor), type); + nodeFlavors.getFlavorOrThrow(flavor), type); return nodeRepository.addNodes(Collections.singletonList(node)).get(0); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java new file mode 100644 index 00000000000..fd0517a6e50 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -0,0 +1,147 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.autoscale; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author bratseth + */ +public class AutoscalingTest { + + @Test + public void testAutoscalingSingleGroup() { + NodeResources resources = new NodeResources(3, 100, 100, 1); + AutoscalingTester tester = new AutoscalingTester(resources); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); + + // deploy + tester.deploy(application1, cluster1, 5, 1, resources); + + assertTrue("No measurements -> No change", tester.autoscale(application1, cluster1).isEmpty()); + + tester.addMeasurements(Resource.cpu, 0.25f, 1f, 60, application1); + assertTrue("Too few measurements -> No change", tester.autoscale(application1, cluster1).isEmpty()); + + tester.addMeasurements(Resource.cpu, 0.25f, 1f, 60, application1); + ClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high", + 15, 1, 1.3, 28.6, 28.6, + tester.autoscale(application1, cluster1)); + + tester.deploy(application1, cluster1, scaledResources); + assertTrue("Cluster in flux -> No further change", tester.autoscale(application1, cluster1).isEmpty()); + + tester.deactivateRetired(application1, cluster1, scaledResources); + tester.addMeasurements(Resource.cpu, 0.8f, 1f, 3, application1); + assertTrue("Load change is large, but insufficient measurements for new config -> No change", + tester.autoscale(application1, cluster1).isEmpty()); + + tester.addMeasurements(Resource.cpu, 0.19f, 1f, 100, application1); + assertEquals("Load change is small -> No change", Optional.empty(), tester.autoscale(application1, cluster1)); + + tester.addMeasurements(Resource.cpu, 0.1f, 1f, 120, application1); + tester.assertResources("Scaling down since resource usage has gone down significantly", + 26, 1, 0.6, 16.0, 16.0, + tester.autoscale(application1, cluster1)); + } + + @Test + public void testAutoscalingGroupSize1() { + NodeResources resources = new NodeResources(3, 100, 100, 1); + AutoscalingTester tester = new AutoscalingTester(resources); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); + + // deploy + tester.deploy(application1, cluster1, 5, 5, resources); + tester.addMeasurements(Resource.cpu, 0.25f, 1f, 120, application1); + tester.assertResources("Scaling up since resource usage is too high", + 7, 7, 2.5, 80.0, 80.0, + tester.autoscale(application1, cluster1)); + } + + @Test + public void testAutoscalingGroupSize3() { + NodeResources resources = new NodeResources(3, 100, 100, 1); + AutoscalingTester tester = new AutoscalingTester(resources); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); + + // deploy + tester.deploy(application1, cluster1, 6, 2, resources); + tester.addMeasurements(Resource.cpu, 0.22f, 1f, 120, application1); + tester.assertResources("Scaling up since resource usage is too high", + 9, 3, 2.7, 83.3, 83.3, + tester.autoscale(application1, cluster1)); + } + + @Test + public void testAutoscalingAvoidsIllegalConfigurations() { + NodeResources resources = new NodeResources(3, 100, 100, 1); + AutoscalingTester tester = new AutoscalingTester(resources); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); + + // deploy + tester.deploy(application1, cluster1, 6, 1, resources); + tester.addMeasurements(Resource.memory, 0.02f, 1f, 120, application1); + tester.assertResources("Scaling down", + 6, 1, 3.0, 4.0, 100.0, + tester.autoscale(application1, cluster1)); + } + + @Test + public void testAutoscalingAws() { + List<Flavor> flavors = new ArrayList<>(); + flavors.add(new Flavor("aws-xlarge", new NodeResources(3, 200, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); + flavors.add(new Flavor("aws-large", new NodeResources(3, 150, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); + flavors.add(new Flavor("aws-medium", new NodeResources(3, 100, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); + flavors.add(new Flavor("aws-small", new NodeResources(3, 80, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); + AutoscalingTester tester = new AutoscalingTester(new Zone(CloudName.from("aws"), SystemName.main, + Environment.prod, RegionName.from("us-east")), + flavors); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); + + // deploy + tester.deploy(application1, cluster1, 5, 1, new NodeResources(3, 100, 100, 1)); + + tester.addMeasurements(Resource.memory, 0.9f, 0.6f, 120, application1); + ClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high." + + "Scaling flavor not count since the latter is more expensive due to " + + "memory charged but taken by aws, see MockHostResourcesCalculator", + 5, 1, 3, 150, 100, + tester.autoscale(application1, cluster1)); + + tester.deploy(application1, cluster1, scaledResources); + tester.deactivateRetired(application1, cluster1, scaledResources); + + tester.addMeasurements(Resource.memory, 0.3f, 0.6f, 1000, application1); + System.out.println("Low memory usage"); + tester.assertResources("Scaling down since resource usage has gone down", + 4, 1, 3, 100, 100, + tester.autoscale(application1, cluster1)); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java new file mode 100644 index 00000000000..f15b7e4220b --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -0,0 +1,248 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.autoscale; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.config.provisioning.FlavorsConfig; +import com.yahoo.test.ManualClock; +import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.node.IP; +import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; +import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +class AutoscalingTester { + + private final ProvisioningTester provisioningTester; + private final Autoscaler autoscaler; + private final NodeMetricsDb db; + private final MockHostResourcesCalculator hostResourcesCalculator; + + /** Creates an autoscaling tester with a single host type ready */ + public AutoscalingTester(NodeResources hostResources) { + this(new Zone(Environment.prod, RegionName.from("us-east")), null, null, asConfig(hostResources)); + provisioningTester.makeReadyNodes(20, "hostFlavor", NodeType.host, 8); // "hostFlavor" generated by asConfig + provisioningTester.deployZoneApp(); + } + + public AutoscalingTester(Zone zone, List<Flavor> flavors) { + this(zone, + new MockHostProvisioner(flavors), + new InMemoryFlagSource().withBooleanFlag(Flags.ENABLE_DYNAMIC_PROVISIONING.id(), true), + asConfig(flavors)); + } + + private AutoscalingTester(Zone zone, MockHostProvisioner hostProvisioner, FlagSource flagSource, FlavorsConfig flavorsConfig) { + provisioningTester = new ProvisioningTester.Builder().zone(zone) + .flavorsConfig(flavorsConfig) + .hostProvisioner(hostProvisioner) + .flagSource(flagSource) + .build(); + + hostResourcesCalculator = new MockHostResourcesCalculator(zone); + db = new NodeMetricsDb(); + autoscaler = new Autoscaler(hostResourcesCalculator, db, nodeRepository()); + } + + public ApplicationId applicationId(String applicationName) { + return ApplicationId.from("tenant1", applicationName, "instance1"); + } + + public ClusterSpec clusterSpec(ClusterSpec.Type type, String clusterId) { + return ClusterSpec.request(type, + ClusterSpec.Id.from(clusterId), + Version.fromString("7"), + false); + } + + public void deploy(ApplicationId application, ClusterSpec cluster, ClusterResources resources) { + deploy(application, cluster, resources.nodes(), resources.groups(), resources.nodeResources()); + } + + public void deploy(ApplicationId application, ClusterSpec cluster, int nodes, int groups, NodeResources resources) { + List<HostSpec> hosts = provisioningTester.prepare(application, cluster, Capacity.fromCount(nodes, resources), groups); + for (HostSpec host : hosts) + makeReady(host.hostname()); + provisioningTester.deployZoneApp(); + provisioningTester.activate(application, hosts); + } + + public void makeReady(String hostname) { + Node node = nodeRepository().getNode(hostname).get(); + nodeRepository().write(node.with(new IP.Config(Set.of("::" + 0 + ":0"), Set.of())), nodeRepository().lock(node)); + Node host = nodeRepository().getNode(node.parentHostname().get()).get(); + host = host.with(new IP.Config(Set.of("::" + 0 + ":0"), Set.of("::" + 0 + ":2"))); + if (host.state() == Node.State.provisioned) + nodeRepository().setReady(List.of(host), Agent.system, getClass().getSimpleName()); + } + + public void deactivateRetired(ApplicationId application, ClusterSpec cluster, ClusterResources resources) { + try (Mutex lock = nodeRepository().lock(application)){ + for (Node node : nodeRepository().getNodes(application, Node.State.active)) { + if (node.allocation().get().membership().retired()) + nodeRepository().write(node.with(node.allocation().get().removable()), lock); + } + } + deploy(application, cluster, resources); + } + + /** + * Adds measurements with the given resource value and ideal values for the other resources, + * scaled to take one node redundancy into account. + * (I.e we adjust to measure a bit lower load than "naively" wanted to offset for the autoscaler + * wanting to see the ideal load with one node missing.) + * + * @param resource the resource we are explicitly setting the value of + * @param otherResourcesLoad the load factor relative to ideal to use for other resources + * @param count the number of measurements + * @param applicationId the application we're adding measurements for all nodes of + */ + public void addMeasurements(Resource resource, float value, float otherResourcesLoad, + int count, ApplicationId applicationId) { + List<Node> nodes = nodeRepository().getNodes(applicationId, Node.State.active); + float oneExtraNodeFactor = (float)(nodes.size() - 1.0) / (nodes.size()); + for (int i = 0; i < count; i++) { + clock().advance(Duration.ofMinutes(1)); + for (Node node : nodes) { + for (Resource r : Resource.values()) { + float effectiveValue = (r == resource ? value : (float) r.idealAverageLoad() * otherResourcesLoad) + * oneExtraNodeFactor; + db.add(List.of(new NodeMetrics.MetricValue(node.hostname(), + r.metricName(), + clock().instant().toEpochMilli(), + effectiveValue))); + } + } + } + } + + public Optional<ClusterResources> autoscale(ApplicationId application, ClusterSpec cluster) { + return autoscaler.autoscale(application, cluster, nodeRepository().getNodes(application, Node.State.active)); + } + + public ClusterResources assertResources(String message, + int nodeCount, int groupCount, + double approxCpu, double approxMemory, double approxDisk, + Optional<ClusterResources> actualResources) { + double delta = 0.0000000001; + assertTrue(message, actualResources.isPresent()); + assertEquals("Node count: " + message, nodeCount, actualResources.get().nodes()); + assertEquals("Group count: " + message, groupCount, actualResources.get().groups()); + assertEquals("Cpu: " + message, approxCpu, Math.round(actualResources.get().nodeResources().vcpu() * 10) / 10.0, delta); + assertEquals("Memory: " + message, approxMemory, Math.round(actualResources.get().nodeResources().memoryGb() * 10) / 10.0, delta); + assertEquals("Disk: " + message, approxDisk, Math.round(actualResources.get().nodeResources().diskGb() * 10) / 10.0, delta); + return actualResources.get(); + } + + public ManualClock clock() { + return provisioningTester.clock(); + } + + public NodeRepository nodeRepository() { + return provisioningTester.nodeRepository(); + } + + private static FlavorsConfig asConfig(NodeResources hostResources) { + FlavorsConfig.Builder b = new FlavorsConfig.Builder(); + b.flavor(asFlavorConfig("hostFlavor", hostResources)); + return b.build(); + } + + private static FlavorsConfig asConfig(List<Flavor> flavors) { + FlavorsConfig.Builder b = new FlavorsConfig.Builder(); + for (Flavor flavor : flavors) + b.flavor(asFlavorConfig(flavor.name(), flavor.resources())); + return b.build(); + } + + private static FlavorsConfig.Flavor.Builder asFlavorConfig(String flavorName, NodeResources resources) { + FlavorsConfig.Flavor.Builder flavor = new FlavorsConfig.Flavor.Builder(); + flavor.name(flavorName); + flavor.minCpuCores(resources.vcpu()); + flavor.minMainMemoryAvailableGb(resources.memoryGb()); + flavor.minDiskAvailableGb(resources.diskGb()); + flavor.bandwidth(resources.bandwidthGbps() * 1000); + return flavor; + } + + private static class MockHostResourcesCalculator implements HostResourcesCalculator { + + private final Zone zone; + + public MockHostResourcesCalculator(Zone zone) { + this.zone = zone; + } + + @Override + public NodeResources availableCapacityOf(String flavorName, NodeResources hostResources) { + if (zone.cloud().value().equals("aws")) + return hostResources.withMemoryGb(hostResources.memoryGb() + 3); + else + return hostResources; + } + + } + + private static class MockHostProvisioner implements HostProvisioner { + + private final List<Flavor> hostFlavors; + + public MockHostProvisioner(List<Flavor> hostFlavors) { + this.hostFlavors = hostFlavors; + } + + @Override + public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources, ApplicationId applicationId) { + Flavor hostFlavor = hostFlavors.stream().filter(f -> f.resources().justNumbers().equals(resources.justNumbers())).findAny() + .orElseThrow(() -> new RuntimeException("No flavor matching " + resources + ". Flavors: " + hostFlavors)); + + List<ProvisionedHost> hosts = new ArrayList<>(); + for (int index : provisionIndexes) { + hosts.add(new ProvisionedHost("host" + index, + "hostname" + index, + hostFlavor, + "nodename" + index, + resources)); + } + return hosts; + } + + @Override + public List<Node> provision(Node host, Set<Node> children) throws FatalProvisioningException { + throw new RuntimeException("Not implemented"); + } + + @Override + public void deprovision(Node host) { + throw new RuntimeException("Not implemented"); + } + + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java new file mode 100644 index 00000000000..519235857f1 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java @@ -0,0 +1,33 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.autoscale; + +import com.yahoo.test.ManualClock; +import org.junit.Test; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class NodeMetricsDbTest { + + @Test + public void testNodeMetricsDb() { + ManualClock clock = new ManualClock(); + NodeMetricsDb db = new NodeMetricsDb(); + List<NodeMetrics.MetricValue> values = new ArrayList<>(); + for (int i = 0; i < 40; i++) { + values.add(new NodeMetrics.MetricValue("host0", "cpu.util", clock.instant().toEpochMilli(), 0.9f)); + clock.advance(Duration.ofHours(1)); + } + db.add(values); + + assertEquals(30, db.getWindow(clock.instant().minus(Duration.ofHours(30)), Resource.cpu, List.of("host0")).measurementCount()); + assertEquals( 0, db.getWindow(clock.instant().minus(Duration.ofHours(30)), Resource.memory, List.of("host0")).measurementCount()); + db.gc(clock); + assertEquals(24, db.getWindow(clock.instant().minus(Duration.ofHours(30)), Resource.cpu, List.of("host0")).measurementCount()); + assertEquals( 0, db.getWindow(clock.instant().minus(Duration.ofHours(30)), Resource.memory, List.of("host0")).measurementCount()); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java new file mode 100644 index 00000000000..4376bfd38b0 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java @@ -0,0 +1,147 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.autoscale; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; +import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock; +import com.yahoo.vespa.applicationmodel.HostName; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class NodeMetricsFetcherTest { + + @Test + public void testMetricsFetch() { + NodeResources resources = new NodeResources(1, 10, 100, 1); + ProvisioningTester tester = new ProvisioningTester.Builder().build(); + OrchestratorMock orchestrator = new OrchestratorMock(); + MockHttpClient httpClient = new MockHttpClient(); + NodeMetricsFetcher fetcher = new NodeMetricsFetcher(tester.nodeRepository(), orchestrator, httpClient); + + tester.makeReadyNodes(4, resources); // Creates (in order) host-1.yahoo.com, host-2.yahoo.com, host-3.yahoo.com, host-4.yahoo.com + tester.deployZoneApp(); + + ApplicationId application1 = tester.makeApplicationId(); + ApplicationId application2 = tester.makeApplicationId(); + tester.deploy(application1, Capacity.fromCount(2, resources)); // host-1.yahoo.com, host-2.yahoo.com + tester.deploy(application2, Capacity.fromCount(2, resources)); // host-4.yahoo.com, host-3.yahoo.com + + orchestrator.suspend(new HostName("host-4.yahoo.com")); + + { + httpClient.cannedResponse = cannedResponseForApplication1; + List<NodeMetrics.MetricValue> values = new ArrayList<>(fetcher.fetchMetrics(application1)); + assertEquals("http://host-1.yahoo.com:4080/metrics/v2/values?consumer=vespa-consumer-metrics", + httpClient.requestsReceived.get(0)); + assertEquals(5, values.size()); + assertEquals("metric value cpu.util: 16.2 at 1234 for host-1.yahoo.com", values.get(0).toString()); + assertEquals("metric value memory.util: 23.1 at 1234 for host-1.yahoo.com", values.get(1).toString()); + assertEquals("metric value disk.util: 82.0 at 1234 for host-1.yahoo.com", values.get(2).toString()); + assertEquals("metric value cpu.util: 20.0 at 1200 for host-2.yahoo.com", values.get(3).toString()); + assertEquals("metric value disk.util: 40.0 at 1200 for host-2.yahoo.com", values.get(4).toString()); + } + + { + httpClient.cannedResponse = cannedResponseForApplication2; + List<NodeMetrics.MetricValue> values = new ArrayList<>(fetcher.fetchMetrics(application2)); + assertEquals("http://host-3.yahoo.com:4080/metrics/v2/values?consumer=vespa-consumer-metrics", + httpClient.requestsReceived.get(1)); + assertEquals(3, values.size()); + assertEquals("metric value cpu.util: 10.0 at 1300 for host-3.yahoo.com", values.get(0).toString()); + assertEquals("metric value memory.util: 15.0 at 1300 for host-3.yahoo.com", values.get(1).toString()); + assertEquals("metric value disk.util: 20.0 at 1300 for host-3.yahoo.com", values.get(2).toString()); + } + } + + private static class MockHttpClient implements NodeMetricsFetcher.HttpClient { + + List<String> requestsReceived = new ArrayList<>(); + + String cannedResponse = null; + @Override + public String get(String url) { + requestsReceived.add(url); + return cannedResponse; + } + + @Override + public void close() { } + + } + + final String cannedResponseForApplication1 = + "{\n" + + " \"nodes\": [\n" + + " {\n" + + " \"hostname\": \"host-1.yahoo.com\",\n" + + " \"role\": \"role0\",\n" + + " \"node\": {\n" + + " \"timestamp\": 1234,\n" + + " \"metrics\": [\n" + + " {\n" + + " \"values\": {\n" + + " \"cpu.util\": 16.2,\n" + + " \"memory.util\": 23.1,\n" + + " \"disk.util\": 82\n" + + " },\n" + + " \"dimensions\": {\n" + + " \"state\": \"active\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " },\n" + + " {\n" + + " \"hostname\": \"host-2.yahoo.com\",\n" + + " \"role\": \"role1\",\n" + + " \"node\": {\n" + + " \"timestamp\": 1200,\n" + + " \"metrics\": [\n" + + " {\n" + + " \"values\": {\n" + + " \"cpu.util\": 20,\n" + + " \"disk.util\": 40\n" + + " },\n" + + " \"dimensions\": {\n" + + " \"state\": \"active\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " }\n" + + " ]\n" + + "}\n"; + + + final String cannedResponseForApplication2 = + "{\n" + + " \"nodes\": [\n" + + " {\n" + + " \"hostname\": \"host-3.yahoo.com\",\n" + + " \"role\": \"role0\",\n" + + " \"node\": {\n" + + " \"timestamp\": 1300,\n" + + " \"metrics\": [\n" + + " {\n" + + " \"values\": {\n" + + " \"cpu.util\": 10,\n" + + " \"memory.util\": 15,\n" + + " \"disk.util\": 20\n" + + " },\n" + + " \"dimensions\": {\n" + + " \"state\": \"active\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " }\n" + + " ]\n" + + "}\n"; + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java index 2b770625060..8706661f261 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java @@ -6,10 +6,13 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; @@ -18,11 +21,15 @@ import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import org.junit.Test; +import java.time.Instant; import java.util.List; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.IntStream; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -62,7 +69,7 @@ public class DynamicDockerProvisionTest { @Test public void does_not_allocate_to_available_empty_hosts() { tester.makeReadyNodes(3, "small", NodeType.host, 10); - deployZoneApp(tester); + tester.deployZoneApp(); ApplicationId application = tester.makeApplicationId(); NodeResources flavor = new NodeResources(1, 4, 10, 1); @@ -82,7 +89,7 @@ public class DynamicDockerProvisionTest { tester.prepare(application, clusterSpec("myContent.t2.a2"), 2, 1, flavor); verify(hostProvisioner).provisionHosts(expectedProvisionIndexes, flavor, application); - // Ready the provisioned hosts, add an IP addreses to pool and activate them + // Ready the provisioned hosts, add an IP addresses to pool and activate them for (Integer i : expectedProvisionIndexes) { String hostname = "host-" + i; var ipConfig = new IP.Config(Set.of("::" + i + ":0"), Set.of("::" + i + ":2")); @@ -90,7 +97,7 @@ public class DynamicDockerProvisionTest { tester.nodeRepository().setReady(List.of(host), Agent.system, getClass().getSimpleName()); nameResolver.addRecord(hostname + "-2", "::" + i + ":2"); } - deployZoneApp(tester); + tester.deployZoneApp(); mockHostProvisioner(hostProvisioner, tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("small")); tester.prepare(application, clusterSpec("another-id"), 2, 1, flavor); @@ -103,19 +110,41 @@ public class DynamicDockerProvisionTest { assertEquals(4, tester.nodeRepository().getNodes(NodeType.tenant, Node.State.reserved).size()); } - private static void deployZoneApp(ProvisioningTester tester) { - ApplicationId applicationId = tester.makeApplicationId(); - List<HostSpec> list = tester.prepare(applicationId, - ClusterSpec.request(ClusterSpec.Type.container, - ClusterSpec.Id.from("node-admin"), - Version.fromString("6.42"), - false), - Capacity.fromRequiredNodeType(NodeType.host), - 1); - tester.activate(applicationId, ImmutableSet.copyOf(list)); + @Test + public void node_indices_are_unique_even_when_a_node_is_left_in_reserved_state() { + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); + NodeResources resources = new NodeResources(10, 10, 10, 10); + ApplicationId app = tester.makeApplicationId(); + + Function<Node, Node> retireNode = node -> + tester.nodeRepository().write(node.withWantToRetire(true, Agent.system, Instant.now()), () -> {}); + Function<Integer, Node> getNodeInGroup = group -> tester.nodeRepository().getNodes(app).stream() + .filter(node -> node.allocation().get().membership().cluster().group().get().index() == group) + .findAny().orElseThrow(); + + // Allocate 10 hosts + tester.makeReadyNodes(10, resources, NodeType.host, 1); + tester.deployZoneApp(); + + // Prepare & activate an application with 8 nodes and 2 groups + tester.activate(app, tester.prepare(app, clusterSpec("content"), 8, 2, resources)); + + // Retire a node in group 1 and prepare the application + retireNode.apply(getNodeInGroup.apply(1)); + tester.prepare(app, clusterSpec("content"), 8, 2, resources); + // App is not activated, to leave node '8' in reserved state + + // Retire a node in group 0 and prepare the application + retireNode.apply(getNodeInGroup.apply(0)); + tester.prepare(app, clusterSpec("content"), 8, 2, resources); + + // Verify that nodes have unique indices from 0..9 + var indices = tester.nodeRepository().getNodes(app).stream() + .map(node -> node.allocation().get().membership().index()) + .collect(Collectors.toSet()); + assertTrue(indices.containsAll(IntStream.range(0, 10).boxed().collect(Collectors.toList()))); } - private static ClusterSpec clusterSpec(String clusterId) { return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId), Version.fromString("6.42"), false); } @@ -130,4 +159,5 @@ public class DynamicDockerProvisionTest { .collect(Collectors.toList()); }).when(hostProvisioner).provisionHosts(any(), any(), any()); } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index e464ed07472..85a6ed31073 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -447,6 +447,7 @@ public class ProvisioningTester { } public static final class Builder { + private Curator curator; private FlavorsConfig flavorsConfig; private Zone zone; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json index 02746f1c79a..ab608bac2b4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json @@ -1,6 +1,9 @@ { "jobs": [ { + "name": "AutoscalingMaintainer" + }, + { "name": "CapacityReportMaintainer" }, { @@ -25,6 +28,9 @@ "name": "NodeFailer" }, { + "name": "NodeMetricsDbMaintainer" + }, + { "name": "NodeRebooter" }, { diff --git a/orchestrator/pom.xml b/orchestrator/pom.xml index 5dd4e7ea87d..fed0e617c79 100644 --- a/orchestrator/pom.xml +++ b/orchestrator/pom.xml @@ -81,12 +81,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.easytesting</groupId> - <artifactId>fest-assert</artifactId> - <version>1.4</version> - <scope>test</scope> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>config-provisioning</artifactId> <version>${project.version}</version> diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java index e9bb4984c2e..65c45c8df76 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java @@ -26,7 +26,6 @@ public interface ClusterApi { Optional<StorageNode> storageNodeInGroup(); Optional<StorageNode> upStorageNodeInGroup(); - String servicesDownAndNotInGroupDescription(); - String nodesAllowedToBeDownNotInGroupDescription(); + String downDescription(); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java index b747d8c2e22..24f56eac85d 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java @@ -131,25 +131,56 @@ class ClusterApiImpl implements ClusterApi { return numberOfServicesDown * 100 / (serviceCluster.serviceInstances().size() + missingServices); } + /** + * A description of the hosts outside the group that are allowed to be down, + * and a description of the services outside the group and outside of the allowed services + * that are down. + */ @Override - public String servicesDownAndNotInGroupDescription() { - // Sort these for readability and testing stability - return Stream - .concat(servicesDownAndNotInGroup.stream().map(ServiceInstance::toString).sorted(), - missingServices > 0 ? Stream.of(descriptionOfMissingServices) : Stream.of()) - .collect(Collectors.toList()) - .toString(); - } + public String downDescription() { + StringBuilder description = new StringBuilder(); - @Override - public String nodesAllowedToBeDownNotInGroupDescription() { - return servicesNotInGroup.stream() + Set<HostName> suspended = servicesNotInGroup.stream() .map(ServiceInstance::hostName) .filter(hostName -> hostStatus(hostName).isSuspended()) - .sorted() - .distinct() - .collect(Collectors.toList()) - .toString(); + .collect(Collectors.toSet()); + + if (suspended.size() > 0) { + description.append(" "); + + final int nodeLimit = 3; + description.append("Suspended hosts: "); + description.append(suspended.stream().sorted().distinct().limit(nodeLimit).collect(Collectors.toList()).toString()); + if (suspended.size() > nodeLimit) { + description.append(", and " + (suspended.size() - nodeLimit) + " more"); + } + description.append("."); + } + + Set<ServiceInstance> downElsewhere = servicesDownAndNotInGroup.stream() + .filter(serviceInstance -> !suspended.contains(serviceInstance.hostName())) + .collect(Collectors.toSet()); + + final int downElsewhereTotal = downElsewhere.size() + missingServices; + if (downElsewhereTotal > 0) { + description.append(" "); + + final int serviceLimit = 2; // services info is verbose + description.append("Services down on resumed hosts: "); + description.append(Stream.concat( + downElsewhere.stream().map(ServiceInstance::toString).sorted(), + missingServices > 0 ? Stream.of(descriptionOfMissingServices) : Stream.of()) + .limit(serviceLimit) + .collect(Collectors.toList()) + .toString()); + + if (downElsewhereTotal > serviceLimit) { + description.append(", and " + (downElsewhereTotal - serviceLimit) + " more"); + } + description.append("."); + } + + return description.toString(); } private Optional<StorageNode> storageNodeInGroup(Predicate<ServiceInstance> storageServicePredicate) { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java index e13cf17d420..a6b3cbc87dc 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java @@ -2,55 +2,43 @@ package com.yahoo.vespa.orchestrator.policy; import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.orchestrator.OrchestrationException; import com.yahoo.vespa.orchestrator.model.NodeGroup; -import java.util.Optional; - /** * @author bakksjo */ public class HostStateChangeDeniedException extends OrchestrationException { private final String constraintName; - private final Optional<ServiceType> serviceType; public HostStateChangeDeniedException(HostName hostName, String constraintName, String message) { this(hostName, constraintName, message, null); } public HostStateChangeDeniedException(HostName hostName, String constraintName, String message, Exception e) { - this(hostName.s(), constraintName, Optional.empty(), message, e); + this(hostName.s(), constraintName, message, e); } public HostStateChangeDeniedException(NodeGroup nodeGroup, String constraintName, String message) { - this(nodeGroup.toCommaSeparatedString(), constraintName, Optional.empty(), message, null); + this(nodeGroup.toCommaSeparatedString(), constraintName, message, null); } private HostStateChangeDeniedException(String nodes, String constraintName, - Optional<ServiceType> serviceType, String message, Throwable cause) { - super(createMessage(nodes, constraintName, serviceType, message), cause); + super(createMessage(nodes, constraintName, message), cause); this.constraintName = constraintName; - this.serviceType = serviceType; } private static String createMessage(String nodes, String constraintName, - Optional<ServiceType> serviceType, String message) { return "Changing the state of " + nodes + " would violate " + constraintName - + (serviceType.isPresent() ? " for service type " + serviceType.get() : "") + ": " + message; } public String getConstraintName() { return constraintName; } - - public Optional<ServiceType> getServiceType() { - return serviceType; - } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java index 1e895d0e757..ccb0bb57186 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java @@ -26,13 +26,11 @@ public class HostedVespaClusterPolicy implements ClusterPolicy { throw new HostStateChangeDeniedException( clusterApi.getNodeGroup(), ENOUGH_SERVICES_UP_CONSTRAINT, - "Suspension percentage for service type " + clusterApi.serviceType() + "Suspension for service type " + clusterApi.serviceType() + " would increase from " + clusterApi.percentageOfServicesDown() + "% to " + clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() + "%, over the limit of " + percentageOfServicesAllowedToBeDown + "%." - + " These instances may be down: " + clusterApi.servicesDownAndNotInGroupDescription() - + " and these hosts are allowed to be down: " - + clusterApi.nodesAllowedToBeDownNotInGroupDescription()); + + clusterApi.downDescription()); } @Override @@ -56,9 +54,7 @@ public class HostedVespaClusterPolicy implements ClusterPolicy { "Down percentage for service type " + clusterApi.serviceType() + " would increase to " + clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() + "%, over the limit of " + percentageOfServicesAllowedToBeDown + "%." - + " These instances may be down: " + clusterApi.servicesDownAndNotInGroupDescription() - + " and these hosts are allowed to be down: " - + clusterApi.nodesAllowedToBeDownNotInGroupDescription()); + + clusterApi.downDescription()); } // Non-private for testing purposes diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java index 62925dc003e..a5cb5cfa630 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; -import java.util.HashMap; import java.util.List; import java.util.Optional; import java.util.Set; @@ -81,15 +80,10 @@ public class ClusterApiImplTest { assertEquals("{ clusterId=cluster, serviceType=service-type }", clusterApi.clusterInfo()); assertFalse(clusterApi.isStorageCluster()); - assertEquals("[ServiceInstance{configId=service-2, hostName=host2, serviceStatus=" + - "ServiceStatusInfo{status=DOWN, since=Optional.empty, lastChecked=Optional.empty}}, " - + "ServiceInstance{configId=service-3, hostName=host3, serviceStatus=" + - "ServiceStatusInfo{status=UP, since=Optional.empty, lastChecked=Optional.empty}}, " - + "ServiceInstance{configId=service-4, hostName=host4, serviceStatus=" + - "ServiceStatusInfo{status=DOWN, since=Optional.empty, lastChecked=Optional.empty}}]", - clusterApi.servicesDownAndNotInGroupDescription()); - assertEquals("[host3, host4]", - clusterApi.nodesAllowedToBeDownNotInGroupDescription()); + assertEquals(" Suspended hosts: [host3, host4]. Services down on resumed hosts: [" + + "ServiceInstance{configId=service-2, hostName=host2, serviceStatus=" + + "ServiceStatusInfo{status=DOWN, since=Optional.empty, lastChecked=Optional.empty}}].", + clusterApi.downDescription()); assertEquals(60, clusterApi.percentageOfServicesDown()); assertEquals(80, clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()); } @@ -110,9 +104,9 @@ public class ClusterApiImplTest { fail(); } catch (HostStateChangeDeniedException e) { assertThat(e.getMessage(), - containsString("Changing the state of cfg1 would violate enough-services-up: Suspension percentage " + - "for service type configserver would increase from 33% to 66%, over the limit of 10%. " + - "These instances may be down: [1 missing config server] and these hosts are allowed to be down: []")); + containsString("Changing the state of cfg1 would violate enough-services-up: " + + "Suspension for service type configserver would increase from 33% to 66%, " + + "over the limit of 10%. Services down on resumed hosts: [1 missing config server].")); } } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java index d834034c9a8..4462e886d1b 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java @@ -101,8 +101,7 @@ public class HostedVespaClusterPolicyTest { when(clusterApi.serviceType()).thenReturn(new ServiceType("service-type")); when(clusterApi.percentageOfServicesDown()).thenReturn(5); when(clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()).thenReturn(percentageOfServicesDownIfGroupIsAllowedToBeDown); - when(clusterApi.servicesDownAndNotInGroupDescription()).thenReturn("services-down-and-not-in-group"); - when(clusterApi.nodesAllowedToBeDownNotInGroupDescription()).thenReturn("allowed-to-be-down"); + when(clusterApi.downDescription()).thenReturn(" Down description"); NodeGroup nodeGroup = mock(NodeGroup.class); when(clusterApi.getNodeGroup()).thenReturn(nodeGroup); @@ -116,11 +115,9 @@ public class HostedVespaClusterPolicyTest { } } catch (HostStateChangeDeniedException e) { if (!expectSuccess) { - assertEquals("Changing the state of node-group would violate enough-services-up: " - + "Suspension percentage for service type service-type would increase from " - + "5% to 13%, over the limit of 10%. These instances may be down: " - + "services-down-and-not-in-group and these hosts are allowed to be down: " - + "allowed-to-be-down", e.getMessage()); + assertEquals("Changing the state of node-group would violate enough-services-up: " + + "Suspension for service type service-type would increase from 5% to 13%, " + + "over the limit of 10%. Down description", e.getMessage()); assertEquals("enough-services-up", e.getConstraintName()); } } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java index ff7413cd3bb..bfa68145828 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java @@ -60,9 +60,9 @@ import java.util.Optional; import java.util.Set; import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceClusterSet; -import static org.fest.assertions.Assertions.assertThat; -import static org.fest.assertions.Fail.fail; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -167,7 +167,7 @@ public class HostResourceTest { UpdateHostResponse response = hostResource.suspend(hostName); - assertThat(response.hostname()).isEqualTo(hostName); + assertEquals(hostName, response.hostname()); } @Test @@ -175,14 +175,14 @@ public class HostResourceTest { HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(alwaysAllowOrchestrator); BatchOperationResult response = hostSuspensionResource.suspendAll("parentHostname", Arrays.asList("hostname1", "hostname2")); - assertThat(response.success()); + assertTrue(response.success()); } @Test public void returns_200_empty_batch() { HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(alwaysAllowOrchestrator); BatchOperationResult response = hostSuspensionResource.suspendAll("parentHostname", List.of()); - assertThat(response.success()); + assertTrue(response.success()); } @Test @@ -193,7 +193,7 @@ public class HostResourceTest { hostResource.suspend("hostname"); fail(); } catch (WebApplicationException w) { - assertThat(w.getResponse().getStatus()).isEqualTo(404); + assertEquals(404, w.getResponse().getStatus()); } } @@ -207,7 +207,7 @@ public class HostResourceTest { hostSuspensionResource.suspendAll("parentHostname", Arrays.asList("hostname1", "hostname2")); fail(); } catch (WebApplicationException w) { - assertThat(w.getResponse().getStatus()).isEqualTo(400); + assertEquals(400, w.getResponse().getStatus()); } } @@ -259,7 +259,7 @@ public class HostResourceTest { hostResource.suspend("hostname"); fail(); } catch (WebApplicationException w) { - assertThat(w.getResponse().getStatus()).isEqualTo(409); + assertEquals(409, w.getResponse().getStatus()); } } @@ -280,7 +280,7 @@ public class HostResourceTest { hostSuspensionResource.suspendAll("parentHostname", Arrays.asList("hostname1", "hostname2")); fail(); } catch (WebApplicationException w) { - assertThat(w.getResponse().getStatus()).isEqualTo(409); + assertEquals(409, w.getResponse().getStatus()); } } @@ -377,7 +377,7 @@ public class HostResourceTest { hostResource.resume("hostname"); fail(); } catch (WebApplicationException w) { - assertThat(w.getResponse().getStatus()).isEqualTo(409); + assertEquals(409, w.getResponse().getStatus()); assertEquals("resume failed: Timeout Message [deadline]", w.getMessage()); } } @@ -392,7 +392,7 @@ public class HostResourceTest { resource.suspendAll("parenthost", Arrays.asList("h1", "h2", "h3")); fail(); } catch (WebApplicationException w) { - assertThat(w.getResponse().getStatus()).isEqualTo(409); + assertEquals(409, w.getResponse().getStatus()); } } } diff --git a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp index 340619f09bd..37ac5f0d65c 100644 --- a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp +++ b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp @@ -1,12 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("threading_service_config_test"); #include <vespa/searchcore/config/config-proton.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/server/threading_service_config.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/log/log.h> +LOG_SETUP("threading_service_config_test"); + using namespace proton; using ProtonConfig = vespa::config::search::core::ProtonConfig; using ProtonConfigBuilder = vespa::config::search::core::ProtonConfigBuilder; @@ -42,6 +43,7 @@ TEST_F("require that indexing threads are set based on cpu cores and feeding con TEST_DO(f.assertIndexingThreads(3, 18)); TEST_DO(f.assertIndexingThreads(4, 19)); TEST_DO(f.assertIndexingThreads(4, 24)); + TEST_DO(f.assertIndexingThreads(4, 64)); // Ensure it is capped at 4 } TEST_F("require that indexing threads is always >= 1", Fixture(0)) diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp index 7c5e7584eed..0e80d31a063 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp @@ -35,7 +35,7 @@ namespace { struct WaitTimer { double &wait_time_s; vespalib::Timer wait_time; - WaitTimer(double &wait_time_s_in) + explicit WaitTimer(double &wait_time_s_in) : wait_time_s(wait_time_s_in), wait_time() { } void done() { diff --git a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp index 8f1c3560e9b..55aa1a20ef6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp @@ -22,7 +22,10 @@ namespace { uint32_t calculateIndexingThreads(uint32_t cfgIndexingThreads, double concurrency, const HwInfo::Cpu &cpuInfo) { - double scaledCores = cpuInfo.cores() * concurrency; + // We are capping at 12 threads to reduce cost of waking up threads + // to achieve a better throughput. + // TODO: Fix this in a simpler/better way. + double scaledCores = std::min(12.0, cpuInfo.cores() * concurrency); uint32_t indexingThreads = std::max((uint32_t)std::ceil(scaledCores / 3), cfgIndexingThreads); return std::max(indexingThreads, 1u); } diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index a7689cd6b9f..60f3a6b7664 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -16,7 +16,6 @@ #include <vespa/searchlib/index/i_field_length_inspector.h> #include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/query/tree/simplequery.h> -#include <vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h> #include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/searchlib/queryeval/fake_search.h> #include <vespa/searchlib/queryeval/fake_searchable.h> @@ -24,7 +23,6 @@ #include <vespa/searchlib/test/index/mock_field_length_inspector.h> #include <vespa/searchlib/util/rand48.h> #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/log/log.h> diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java index 18f6c6f2ca2..1f27bc8750e 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java @@ -10,6 +10,7 @@ import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; +import com.yahoo.text.Text; import java.io.File; import java.io.FileNotFoundException; @@ -115,7 +116,7 @@ public class RankingExpression implements Serializable { root = parse(new StringReader(expression)); } catch (ParseException e) { - ParseException p = new ParseException("Could not parse '" + expression + "'"); + ParseException p = new ParseException("Could not parse '" + Text.truncate(expression, 50) + "'"); p.initCause(e); throw p; } diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 5089743a54a..1f3ad7c2fec 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -6,6 +6,7 @@ #include <vespa/eval/tensor/tensor.h> #include <vespa/fastos/file.h> #include <vespa/searchlib/attribute/attributeguard.h> +#include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/tensor/doc_vector_access.h> @@ -41,6 +42,7 @@ using vespalib::tensor::DenseTensor; using vespalib::tensor::Tensor; using DoubleVector = std::vector<double>; +using generation_t = vespalib::GenerationHandler::generation_t; namespace vespalib::tensor { @@ -80,12 +82,16 @@ private: const DocVectorAccess& _vectors; EntryVector _adds; EntryVector _removes; + generation_t _transfer_gen; + generation_t _trim_gen; public: MockNearestNeighborIndex(const DocVectorAccess& vectors) : _vectors(vectors), _adds(), - _removes() + _removes(), + _transfer_gen(std::numeric_limits<generation_t>::max()), + _trim_gen(std::numeric_limits<generation_t>::max()) { } void clear() { @@ -111,6 +117,9 @@ public: EXPECT_EQUAL(exp_docid, _removes.back().first); EXPECT_EQUAL(exp_vector, _removes.back().second); } + generation_t get_transfer_gen() const { return _transfer_gen; } + generation_t get_trim_gen() const { return _trim_gen; } + void add_document(uint32_t docid) override { auto vector = _vectors.get_vector(docid).typify<double>(); _adds.emplace_back(docid, DoubleVector(vector.begin(), vector.end())); @@ -119,6 +128,15 @@ public: auto vector = _vectors.get_vector(docid).typify<double>(); _removes.emplace_back(docid, DoubleVector(vector.begin(), vector.end())); } + void transfer_hold_lists(generation_t current_gen) override { + _transfer_gen = current_gen; + } + void trim_hold_lists(generation_t first_used_gen) override { + _trim_gen = first_used_gen; + } + vespalib::MemoryUsage memory_usage() const override { + return vespalib::MemoryUsage(); + } std::vector<Neighbor> find_top_k(uint32_t k, vespalib::tensor::TypedCells vector, uint32_t explore_k) const override { (void) k; (void) vector; @@ -232,6 +250,10 @@ struct Fixture _attr->commit(); } + generation_t get_current_gen() const { + return _attr->getCurrentGeneration(); + } + search::attribute::Status getStatus() { _attr->commit(true); return _attr->getStatus(); @@ -531,4 +553,33 @@ TEST_F("onLoad() updates nearest neighbor index", DenseTensorAttributeMockIndex) index.expect_adds({{1, {3, 5}}, {2, {7, 9}}}); } + +TEST_F("commit() ensures transfer and trim hold lists on nearest neighbor index", DenseTensorAttributeMockIndex) +{ + auto& index = f.mock_index(); + TensorSpec spec = vec_2d(3, 5); + + f.set_tensor(1, spec); + generation_t gen_1 = f.get_current_gen(); + EXPECT_EQUAL(gen_1 - 1, index.get_transfer_gen()); + EXPECT_EQUAL(gen_1, index.get_trim_gen()); + + generation_t gen_2 = 0; + { + // Takes guard on gen_1 + auto guard = f._attr->makeReadGuard(false); + f.set_tensor(2, spec); + gen_2 = f.get_current_gen(); + EXPECT_GREATER(gen_2, gen_1); + EXPECT_EQUAL(gen_2 - 1, index.get_transfer_gen()); + EXPECT_EQUAL(gen_1, index.get_trim_gen()); + } + + f.set_tensor(3, spec); + generation_t gen_3 = f.get_current_gen(); + EXPECT_GREATER(gen_3, gen_2); + EXPECT_EQUAL(gen_3 - 1, index.get_transfer_gen()); + EXPECT_EQUAL(gen_3, index.get_trim_gen()); +} + TEST_MAIN() { TEST_RUN_ALL(); vespalib::unlink("test.dat"); } diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/.gitignore b/searchlib/src/tests/common/sequencedtaskexecutor/.gitignore index 35d038b0b7c..4bd94f124fb 100644 --- a/searchlib/src/tests/common/sequencedtaskexecutor/.gitignore +++ b/searchlib/src/tests/common/sequencedtaskexecutor/.gitignore @@ -1 +1,2 @@ searchlib_sequencedtaskexecutor_test_app +searchlib_sequencedtaskexecutor_benchmark_app diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt b/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt index 6ba30a1647f..6c593d20683 100644 --- a/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt +++ b/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt @@ -1,4 +1,11 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_sequencedtaskexecutor_benchmark_app TEST + SOURCES + sequencedtaskexecutor_benchmark.cpp + DEPENDS + searchlib +) + vespa_add_executable(searchlib_sequencedtaskexecutor_test_app TEST SOURCES sequencedtaskexecutor_test.cpp diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp b/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp new file mode 100644 index 00000000000..a51becfbf13 --- /dev/null +++ b/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp @@ -0,0 +1,24 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchlib/common/sequencedtaskexecutor.h> +#include <vespa/vespalib/util/lambdatask.h> +#include <atomic> + +using search::SequencedTaskExecutor; +using ExecutorId = search::ISequencedTaskExecutor::ExecutorId; + +int main(int argc, char *argv[]) { + unsigned long numTasks = 1000000; + unsigned numThreads = 4; + std::atomic<long> counter(0); + if (argc > 1) + numTasks = atol(argv[1]); + if (argc > 2) + numThreads = atoi(argv[2]); + + SequencedTaskExecutor executor(numThreads); + for (unsigned long tid(0); tid < numTasks; tid++) { + executor.executeTask(ExecutorId(tid%numThreads), vespalib::makeLambdaTask([&counter] { counter++; })); + } + return 0; +} diff --git a/searchlib/src/tests/features/prod_features.cpp b/searchlib/src/tests/features/prod_features.cpp index 56aaf2dcbc9..de436dffff1 100644 --- a/searchlib/src/tests/features/prod_features.cpp +++ b/searchlib/src/tests/features/prod_features.cpp @@ -9,16 +9,15 @@ #include <vespa/searchlib/attribute/floatbase.h> #include <vespa/searchlib/attribute/integerbase.h> #include <vespa/searchlib/attribute/stringbase.h> +#include <vespa/searchlib/attribute/singleboolattribute.h> #include <vespa/searchlib/features/agefeature.h> #include <vespa/searchlib/features/array_parser.hpp> #include <vespa/searchlib/features/attributefeature.h> -#include <vespa/searchlib/features/attributematchfeature.h> #include <vespa/searchlib/features/closenessfeature.h> #include <vespa/searchlib/features/distancefeature.h> #include <vespa/searchlib/features/dotproductfeature.h> #include <vespa/searchlib/features/fieldlengthfeature.h> #include <vespa/searchlib/features/fieldmatchfeature.h> -#include <vespa/searchlib/features/fieldtermmatchfeature.h> #include <vespa/searchlib/features/firstphasefeature.h> #include <vespa/searchlib/features/foreachfeature.h> #include <vespa/searchlib/features/freshnessfeature.h> @@ -35,7 +34,6 @@ #include <vespa/searchlib/features/setup.h> #include <vespa/searchlib/features/termfeature.h> #include <vespa/searchlib/features/utils.h> -#include <vespa/searchlib/features/valuefeature.h> #include <vespa/searchlib/features/weighted_set_parser.hpp> #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/indexproperties.h> @@ -60,6 +58,7 @@ using search::AttributeFactory; using search::IntegerAttribute; using search::FloatingPointAttribute; using search::StringAttribute; +using search::SingleBoolAttribute; using search::WeightedSetStringExtAttribute; using search::attribute::WeightedEnumContent; @@ -212,7 +211,7 @@ Test::setupForAgeTest(FtFeatureTest & ft, uint64_t docTime) doctime->addReservedDoc(); doctime->addDocs(1); ft.getIndexEnv().getAttributeMap().add(doctime); - (static_cast<IntegerAttribute *>(doctime.get()))->update(1, docTime); + (dynamic_cast<IntegerAttribute *>(doctime.get()))->update(1, docTime); doctime->commit(); } @@ -240,7 +239,12 @@ Test::testAttribute() RankResult exp; exp.addScore("attribute(sint)", 10). addScore("attribute(sint,0)", 10). + addScore("attribute(slong)", 20). + addScore("attribute(sbyte)", 37). + addScore("attribute(sbool)", 1). + addScore("attribute(sebool)", 0). addScore("attribute(sfloat)", 60.5f). + addScore("attribute(sdouble)", 67.5f). addScore("attribute(sstr)", (feature_t)vespalib::hash_code("foo")). addScore("attribute(sint).count", 1). addScore("attribute(sfloat).count", 1). @@ -250,12 +254,18 @@ Test::testAttribute() addScore("attribute(udefstr)", (feature_t)vespalib::hash_code("")); FtFeatureTest ft(_factory, exp.getKeys()); - ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sint"). - addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sfloat"). - addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sstr"). - addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefint"). - addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udeffloat"). - addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefstr"); + ft.getIndexEnv().getBuilder() + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sint") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "slong") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sbyte") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sbool") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sebool") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sfloat") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sdouble") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sstr") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefint") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udeffloat") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefstr"); setupForAttributeTest(ft); ASSERT_TRUE(ft.setup()); ASSERT_TRUE(ft.execute(exp)); @@ -370,6 +380,11 @@ Test::setupForAttributeTest(FtFeatureTest &ft, bool setup_env) avs.push_back(AttributeFactory::createAttribute("udefint", AVC(AVBT::INT32, AVCT::SINGLE))); // 9 avs.push_back(AttributeFactory::createAttribute("udeffloat", AVC(AVBT::FLOAT, AVCT::SINGLE))); // 10 avs.push_back(AttributeFactory::createAttribute("udefstr", AVC(AVBT::STRING, AVCT::SINGLE))); // 11 + avs.push_back(AttributeFactory::createAttribute("sbyte", AVC(AVBT::INT64, AVCT::SINGLE))); // 12 + avs.push_back(AttributeFactory::createAttribute("slong", AVC(AVBT::INT64, AVCT::SINGLE))); // 13 + avs.push_back(AttributeFactory::createAttribute("sbool", AVC(AVBT::BOOL, AVCT::SINGLE))); // 14 + avs.push_back(AttributeFactory::createAttribute("sebool", AVC(AVBT::BOOL, AVCT::SINGLE))); // 15 + avs.push_back(AttributeFactory::createAttribute("sdouble", AVC(AVBT::DOUBLE, AVCT::SINGLE))); // 16 // simulate a unique only attribute as specified in sd AVC cfg(AVBT::INT32, AVCT::SINGLE); @@ -391,36 +406,46 @@ Test::setupForAttributeTest(FtFeatureTest &ft, bool setup_env) .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefint") .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udeffloat") .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefstr") - .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "unique"); + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "unique") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "slong") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sdouble") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sbyte") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sbool") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sebool"); } - for (uint32_t i = 0; i < avs.size(); ++i) { - avs[i]->addReservedDoc(); - avs[i]->addDocs(1); - ft.getIndexEnv().getAttributeMap().add(avs[i]); + for (const auto & attr : avs) { + attr->addReservedDoc(); + attr->addDocs(1); + ft.getIndexEnv().getAttributeMap().add(attr); } // integer attributes - (static_cast<IntegerAttribute *>(avs[0].get()))->update(1, 10); - (static_cast<IntegerAttribute *>(avs[1].get()))->append(1, 20, 0); - (static_cast<IntegerAttribute *>(avs[1].get()))->append(1, 30, 0); - (static_cast<IntegerAttribute *>(avs[2].get()))->append(1, 40, 10); - (static_cast<IntegerAttribute *>(avs[2].get()))->append(1, 50, 20); - (static_cast<IntegerAttribute *>(avs[9].get()))->update(1, search::attribute::getUndefined<int32_t>()); + (dynamic_cast<IntegerAttribute *>(avs[0].get()))->update(1, 10); + (dynamic_cast<IntegerAttribute *>(avs[12].get()))->update(1, 37); + (dynamic_cast<IntegerAttribute *>(avs[13].get()))->update(1, 20); + (dynamic_cast<SingleBoolAttribute *>(avs[14].get()))->update(1, 1); + (dynamic_cast<SingleBoolAttribute *>(avs[15].get()))->update(1, 0); + (dynamic_cast<IntegerAttribute *>(avs[1].get()))->append(1, 20, 0); + (dynamic_cast<IntegerAttribute *>(avs[1].get()))->append(1, 30, 0); + (dynamic_cast<IntegerAttribute *>(avs[2].get()))->append(1, 40, 10); + (dynamic_cast<IntegerAttribute *>(avs[2].get()))->append(1, 50, 20); + (dynamic_cast<IntegerAttribute *>(avs[9].get()))->update(1, search::attribute::getUndefined<int32_t>()); // feature_t attributes - (static_cast<FloatingPointAttribute *>(avs[3].get()))->update(1, 60.5f); - (static_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 70.5f, 0); - (static_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 80.5f, 0); - (static_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 90.5f, -30); - (static_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 100.5f, -40); - (static_cast<FloatingPointAttribute *>(avs[10].get()))->update(1, search::attribute::getUndefined<float>()); + (dynamic_cast<FloatingPointAttribute *>(avs[3].get()))->update(1, 60.5f); + (dynamic_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 70.5f, 0); + (dynamic_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 80.5f, 0); + (dynamic_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 90.5f, -30); + (dynamic_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 100.5f, -40); + (dynamic_cast<FloatingPointAttribute *>(avs[10].get()))->update(1, search::attribute::getUndefined<float>()); + (dynamic_cast<FloatingPointAttribute *>(avs[16].get()))->update(1, 67.5); // string attributes - (static_cast<StringAttribute *>(avs[6].get()))->update(1, "foo"); - (static_cast<StringAttribute *>(avs[7].get()))->append(1, "bar", 0); - (static_cast<StringAttribute *>(avs[7].get()))->append(1, "baz", 0); - (static_cast<StringAttribute *>(avs[8].get()))->append(1, "qux", 11); - (static_cast<StringAttribute *>(avs[8].get()))->append(1, "quux", 12); - (static_cast<StringAttribute *>(avs[11].get()))->update(1, ""); + (dynamic_cast<StringAttribute *>(avs[6].get()))->update(1, "foo"); + (dynamic_cast<StringAttribute *>(avs[7].get()))->append(1, "bar", 0); + (dynamic_cast<StringAttribute *>(avs[7].get()))->append(1, "baz", 0); + (dynamic_cast<StringAttribute *>(avs[8].get()))->append(1, "qux", 11); + (dynamic_cast<StringAttribute *>(avs[8].get()))->append(1, "quux", 12); + (dynamic_cast<StringAttribute *>(avs[11].get()))->update(1, ""); for (uint32_t i = 0; i < avs.size() - 1; ++i) { // do not commit the noupdate attribute avs[i]->commit(); @@ -475,7 +500,7 @@ Test::assertCloseness(feature_t exp, const vespalib::string & attr, double dista FtFeatureTest ft(_factory, feature); std::vector<std::pair<int32_t, int32_t> > positions; int32_t x = 0; - positions.push_back(std::make_pair(x, x)); + positions.emplace_back(x, x); setupForDistanceTest(ft, "pos", positions, false); ft.getQueryEnv().getLocation().setXPosition((int)distance); ft.getQueryEnv().getLocation().setValid(true); @@ -572,7 +597,7 @@ Test::assertFieldMatch(const vespalib::string & spec, const vespalib::string & field, uint32_t totalTermWeight) { - assertFieldMatch(spec, query, field, NULL, totalTermWeight); + assertFieldMatch(spec, query, field, nullptr, totalTermWeight); } void @@ -581,7 +606,7 @@ Test::assertFieldMatchTS(const vespalib::string & spec, const vespalib::string & field, feature_t totalSignificance) { - assertFieldMatch(spec, query, field, NULL, 0, totalSignificance); + assertFieldMatch(spec, query, field, nullptr, 0, totalSignificance); } @@ -871,12 +896,12 @@ Test::setupForDistanceTest(FtFeatureTest &ft, const vespalib::string & attrName, pos->addDocs(1); ft.getIndexEnv().getAttributeMap().add(pos); - IntegerAttribute * ia = static_cast<IntegerAttribute *>(pos.get()); - for (uint32_t i = 0; i < positions.size(); ++i) { + auto ia = dynamic_cast<IntegerAttribute *>(pos.get()); + for (const auto & p : positions) { if (zcurve) { - ia->append(1, vespalib::geo::ZCurve::encode(positions[i].first, positions[i].second), 0); + ia->append(1, vespalib::geo::ZCurve::encode(p.first, p.second), 0); } else { - ia->append(1, positions[i].first, 0); + ia->append(1, p.first, 0); } } @@ -891,11 +916,11 @@ Test::assert2DZDistance(feature_t exp, const vespalib::string & positions, FtFeatureTest ft(_factory, "distance(pos)"); std::vector<vespalib::string> ta = FtUtil::tokenize(positions, ","); std::vector<std::pair<int32_t, int32_t> > pos; - for (uint32_t i = 0; i < ta.size(); ++i) { - std::vector<vespalib::string> tb = FtUtil::tokenize(ta[i], ":"); - int32_t x = util::strToNum<int32_t>(tb[0]); - int32_t y = util::strToNum<int32_t>(tb[1]); - pos.push_back(std::make_pair(x, y)); + for (const auto & s : ta) { + std::vector<vespalib::string> tb = FtUtil::tokenize(s, ":"); + auto x = util::strToNum<int32_t>(tb[0]); + auto y = util::strToNum<int32_t>(tb[1]); + pos.emplace_back(x, y); } setupForDistanceTest(ft, "pos", pos, true); ft.getQueryEnv().getLocation().setXPosition(xquery); @@ -927,7 +952,7 @@ Test::testDistanceToPath() { // Test executor. std::vector<std::pair<int32_t, int32_t> > pos; - pos.push_back(std::make_pair(0, 0)); + pos.emplace_back(0, 0); // invalid path assertDistanceToPath(pos, "a"); @@ -965,7 +990,7 @@ Test::testDistanceToPath() assertDistanceToPath(pos, "(-3,2,2,2,2,-1,0,-1)", 1, 1, 2); // multiple document locations - pos.push_back(std::make_pair(0, 1)); + pos.emplace_back(0, 1); assertDistanceToPath(pos, "(-1,1,1,1)", 0, 0.5); assertDistanceToPath(pos, "(-2,-1,-1,1)", 1, 1, 2); assertDistanceToPath(pos, "(-1,0.25,1,0.25)", 0.25, 0.5, 0.5); @@ -1017,7 +1042,7 @@ Test::testDistanceToPath() } void -Test::assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > pos, +Test::assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > & pos, const vespalib::string &path, feature_t distance, feature_t traveled, feature_t product) { LOG(info, "Testing distance to path '%s' with %zd document locations.", path.c_str(), pos.size()); @@ -1033,20 +1058,6 @@ Test::assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > pos, .addScore("distanceToPath(pos).product", product))); } -void -Test::setupForDocumentTest(FtFeatureTest &ft, const vespalib::string & attrName, const vespalib::string & docType) -{ - AttributePtr type = AttributeFactory::createAttribute(attrName, AVC(AVBT::STRING, AVCT::SINGLE)); - - type->addReservedDoc(); - type->addDocs(1); - ft.getIndexEnv().getAttributeMap().add(type); - - (static_cast<StringAttribute *>(type.get()))->update(1, docType); - type->commit(); -} - - namespace { void @@ -1264,6 +1275,8 @@ void Test::setupForDotProductTest(FtFeatureTest & ft) { struct Config { + Config() : name(nullptr), dataType(AVBT::BOOL), collectionType(AVCT::SINGLE), fastSearch(false) {} + Config(const char *n, AVBT dt, AVCT ct, bool fs) : name(n), dataType(dt), collectionType(ct), fastSearch(fs) {} const char * name; AVBT dataType; AVCT collectionType; @@ -1296,11 +1309,11 @@ Test::setupForDotProductTest(FtFeatureTest & ft) baf->addDocs(2); ft.getIndexEnv().getAttributeMap().add(baf); for (size_t i(1); i < 6; i++) { - IntegerAttribute * ia = dynamic_cast<IntegerAttribute *>(baf.get()); + auto ia = dynamic_cast<IntegerAttribute *>(baf.get()); if (ia) { ia->append(1, i, i); } else { - FloatingPointAttribute * fa = dynamic_cast<FloatingPointAttribute *>(baf.get()); + auto fa = dynamic_cast<FloatingPointAttribute *>(baf.get()); fa->append(1, i, i); } } @@ -1315,14 +1328,14 @@ Test::setupForDotProductTest(FtFeatureTest & ft) ft.getIndexEnv().getAttributeMap().add(c); ft.getIndexEnv().getAttributeMap().add(d); - StringAttribute * sa = static_cast<StringAttribute *>(a.get()); + auto sa = dynamic_cast<StringAttribute *>(a.get()); sa->append(1, "a", 1); sa->append(1, "b", 2); sa->append(1, "c", 3); sa->append(1, "d", 4); sa->append(1, "e", 5); - WeightedSetStringExtAttribute * ea = static_cast<WeightedSetStringExtAttribute *>(d.get()); + auto ea = dynamic_cast<WeightedSetStringExtAttribute *>(d.get()); EXPECT_TRUE(!ea->hasEnum()); uint32_t docId; ea->addDoc(docId); // reserved doc @@ -1517,9 +1530,9 @@ Test::testMatchCount() ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "foo"); ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "bar"); ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "baz"); - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL); // query term 0, hit in foo - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != NULL); // query term 1, hit in bar - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL); // query term 2, hit in foo + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr); // query term 0, hit in foo + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != nullptr); // query term 1, hit in bar + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr); // query term 2, hit in foo ASSERT_TRUE(ft.setup()); MatchDataBuilder::UP mdb = ft.createMatchDataBuilder(); @@ -1577,9 +1590,9 @@ Test::testMatches() ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "foo"); ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "bar"); ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "baz"); - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL); // query term 0, hit in foo - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != NULL); // query term 1, hit in bar - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL); // query term 2, hit in foo + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr); // query term 0, hit in foo + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != nullptr); // query term 1, hit in bar + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr); // query term 2, hit in foo ASSERT_TRUE(ft.setup()); MatchDataBuilder::UP mdb = ft.createMatchDataBuilder(); @@ -1613,11 +1626,9 @@ Test::assertMatches(uint32_t output, ASSERT_TRUE(ft.execute(output, EPS, docId)); // Execute and compare results. - if (!EXPECT_TRUE(ft.execute(output, EPS, docId))) return false; - return true; + return EXPECT_TRUE(ft.execute(output, EPS, docId)); } - void Test::testQuery() { @@ -1721,7 +1732,7 @@ Test::testRandom() search::Rand48 rnd; rnd.srand48(100); for (uint32_t i = 0; i < 5; ++i) { - feature_t exp = rnd.lrand48() / (feature_t)0x80000000u; + feature_t exp = static_cast<feature_t>(rnd.lrand48()) / static_cast<feature_t>(0x80000000u); ASSERT_TRUE(ft.execute(exp, EPS, i + 1)); } } @@ -1744,7 +1755,7 @@ Test::testRandom() search::Rand48 rnd; for (uint32_t i = 1; i <= 5; ++i) { rnd.srand48(100 + i); // seed + lid - feature_t exp = rnd.lrand48() / (feature_t)0x80000000u; + feature_t exp = static_cast<feature_t>(rnd.lrand48()) / static_cast<feature_t>(0x80000000u); ASSERT_TRUE(ft.execute(exp, EPS, i)); } } @@ -2067,10 +2078,7 @@ Test::assertTermDistance(const TermDistanceCalculator::Result & exp, rr.addScore(feature + ".forwardTermPosition", exp.forwardTermPos); rr.addScore(feature + ".reverse", exp.reverseDist); rr.addScore(feature + ".reverseTermPosition", exp.reverseTermPos); - if (!EXPECT_TRUE(ft.execute(rr, docId))) { - return false; - } - return true; + return EXPECT_TRUE(ft.execute(rr, docId)); } void diff --git a/searchlib/src/tests/features/prod_features.h b/searchlib/src/tests/features/prod_features.h index 8e6578f34bd..6d30dd3fcd8 100644 --- a/searchlib/src/tests/features/prod_features.h +++ b/searchlib/src/tests/features/prod_features.h @@ -10,10 +10,10 @@ class Test : public FtTestApp { public: Test(); - ~Test(); + ~Test() override; int Main() override; void testFramework(); - void testFtLib(); + static void testFtLib(); void testAge(); void testAttribute(); void testAttributeMatch(); @@ -39,10 +39,9 @@ public: void testRankingExpression(); void testTerm(); void testTermDistance(); - void testUtils(); + static void testUtils(); static void setupForDotProductTest(FtFeatureTest & ft); - static void setupForDocumentTest(FtFeatureTest &ft, const vespalib::string & attrName, const vespalib::string & docType); private: void testFieldMatchBluePrint(); @@ -81,21 +80,21 @@ private: void testFieldMatchExecutorRemaining(); void assertAge(feature_t expAge, const vespalib::string & attr, uint64_t now, uint64_t docTime); - void setupForAgeTest(FtFeatureTest & ft, uint64_t docTime); - void setupForAttributeTest(FtFeatureTest &ft, bool setup_env = true); + static void setupForAgeTest(FtFeatureTest & ft, uint64_t docTime); + static void setupForAttributeTest(FtFeatureTest &ft, bool setup_env = true); void assertCloseness(feature_t exp, const vespalib::string & attr, double distance, double maxDistance = 0, double halfResponse = 0); - void setupForDistanceTest(FtFeatureTest & ft, const vespalib::string & attrName, - const std::vector<std::pair<int32_t, int32_t> > & positions, bool zcurve); + static void setupForDistanceTest(FtFeatureTest & ft, const vespalib::string & attrName, + const std::vector<std::pair<int32_t, int32_t> > & positions, bool zcurve); void assert2DZDistance(feature_t exp, const vespalib::string & positions, int32_t xquery, int32_t yquery, uint32_t xAspect = 0); - void assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > pos, const vespalib::string &path, + void assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > & pos, const vespalib::string &path, feature_t distance = search::features::DistanceToPathExecutor::DEFAULT_DISTANCE, feature_t traveled = 1, feature_t product = 0); void assertDotProduct(feature_t exp, const vespalib::string & vector, uint32_t docId = 1, const vespalib::string & attribute = "wsstr", const vespalib::string & attributeOverride=""); void assertFieldMatch(const vespalib::string & spec, const vespalib::string & query, const vespalib::string & field, - const search::features::fieldmatch::Params * params = NULL, uint32_t totalTermWeight = 0, feature_t totalSignificance = 0.0f); + const search::features::fieldmatch::Params * params = nullptr, uint32_t totalTermWeight = 0, feature_t totalSignificance = 0.0f); void assertFieldMatch(const vespalib::string & spec, const vespalib::string & query, const vespalib::string & field, uint32_t totalTermWeight); void assertFieldMatchTS(const vespalib::string & spec, const vespalib::string & query, const vespalib::string & field, diff --git a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp index 691e80aeb9f..ee8d4b787bd 100644 --- a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp +++ b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp @@ -195,7 +195,7 @@ TEST("require that NnsIndexIterator works as expected") { std::vector<NnsIndexIterator::Hit> hits{{2,4.0}, {3,9.0}, {5,1.0}, {8,16.0}, {9,36.0}}; auto md = MatchData::makeTestInstance(2, 2); auto &tfmd = *(md->resolveTermField(0)); - auto search = NnsIndexIterator::create(true, tfmd, hits); + auto search = NnsIndexIterator::create(tfmd, hits); uint32_t docid = 1; search->initFullRange(); bool match = search->seek(docid); diff --git a/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp b/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp index b10da86dd8c..861af3527ca 100644 --- a/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp +++ b/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.h> #include <vespa/searchlib/fef/test/indexenvironment.h> #include <vespa/searchlib/fef/test/queryenvironment.h> +#include <vespa/vespalib/util/stash.h> #include <set> using namespace search::features::rankingexpression; diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp index 1204ae1e9bc..37c4d02017f 100644 --- a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -6,11 +6,14 @@ #include <vespa/searchlib/tensor/hnsw_index.h> #include <vespa/searchlib/tensor/random_level_generator.h> #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/util/generationhandler.h> #include <vector> #include <vespa/log/log.h> LOG_SETUP("hnsw_index_test"); +using vespalib::GenerationHandler; +using vespalib::MemoryUsage; using namespace search::tensor; template <typename FloatType> @@ -49,11 +52,13 @@ class HnswIndexTest : public ::testing::Test { public: FloatVectors vectors; LevelGenerator* level_generator; + GenerationHandler gen_handler; HnswIndexUP index; HnswIndexTest() : vectors(), level_generator(), + gen_handler(), index() { vectors.set(1, {2, 2}).set(2, {3, 2}).set(3, {2, 3}) @@ -70,9 +75,23 @@ public: void add_document(uint32_t docid, uint32_t max_level = 0) { level_generator->level = max_level; index->add_document(docid); + commit(); } void remove_document(uint32_t docid) { index->remove_document(docid); + commit(); + } + void commit() { + index->transfer_hold_lists(gen_handler.getCurrentGeneration()); + gen_handler.incGeneration(); + gen_handler.updateFirstUsedGeneration(); + index->trim_hold_lists(gen_handler.getFirstUsedGeneration()); + } + GenerationHandler::Guard take_read_guard() { + return gen_handler.takeGuard(); + } + MemoryUsage memory_usage() const { + return index->memory_usage(); } void expect_entry_point(uint32_t exp_docid, uint32_t exp_level) { EXPECT_EQ(exp_docid, index->get_entry_docid()); @@ -83,6 +102,10 @@ public: ASSERT_EQ(1, node.size()); EXPECT_EQ(exp_links, node.level(0)); } + void expect_empty_level_0(uint32_t docid) { + auto node = index->get_node(docid); + EXPECT_TRUE(node.empty()); + } void expect_levels(uint32_t docid, const HnswNode::LevelArray& exp_levels) { auto act_node = index->get_node(docid); ASSERT_EQ(exp_levels.size(), act_node.size()); @@ -266,5 +289,83 @@ TEST_F(HnswIndexTest, 2d_vectors_inserted_in_hierarchic_graph_with_heuristic_sel expect_level_0(7, {3, 6}); } +TEST_F(HnswIndexTest, manual_insert) +{ + init(false); + + std::vector<uint32_t> nbl; + HnswNode empty{nbl}; + index->set_node(1, empty); + index->set_node(2, empty); + + HnswNode three{{1,2}}; + index->set_node(3, three); + expect_level_0(1, {3}); + expect_level_0(2, {3}); + expect_level_0(3, {1,2}); + + expect_entry_point(1, 0); + + HnswNode twolevels{{{1},nbl}}; + index->set_node(4, twolevels); + + expect_entry_point(4, 1); + expect_level_0(1, {3,4}); + + HnswNode five{{{1,2}, {4}}}; + index->set_node(5, five); + + expect_levels(1, {{3,4,5}}); + expect_levels(2, {{3,5}}); + expect_levels(3, {{1,2}}); + expect_levels(4, {{1}, {5}}); + expect_levels(5, {{1,2}, {4}}); +} + +TEST_F(HnswIndexTest, memory_is_reclaimed_when_doing_changes_to_graph) +{ + init(true); + + add_document(1); + add_document(3); + auto mem_1 = memory_usage(); + + add_document(2); + expect_level_0(1, {2,3}); + expect_level_0(2, {1,3}); + expect_level_0(3, {1,2}); + auto mem_2 = memory_usage(); + // We should use more memory with larger link arrays and extra document. + EXPECT_GT((mem_2.usedBytes() - mem_2.deadBytes()), (mem_1.usedBytes() - mem_1.deadBytes())); + EXPECT_EQ(0, mem_2.allocatedBytesOnHold()); + + remove_document(2); + expect_level_0(1, {3}); + expect_empty_level_0(2); + expect_level_0(3, {1}); + auto mem_3 = memory_usage(); + // We end up in the same state as before document 2 was added and effectively use the same amount of memory. + EXPECT_EQ((mem_1.usedBytes() - mem_1.deadBytes()), (mem_3.usedBytes() - mem_3.deadBytes())); + EXPECT_EQ(0, mem_3.allocatedBytesOnHold()); +} + +TEST_F(HnswIndexTest, memory_is_put_on_hold_while_read_guard_is_held) +{ + init(true); + + add_document(1); + add_document(3); + { + auto guard = take_read_guard(); + add_document(2); + auto mem = memory_usage(); + // As read guard is held memory to reclaim is put on hold + EXPECT_GT(mem.allocatedBytesOnHold(), 0); + } + commit(); + auto mem = memory_usage(); + EXPECT_EQ(0, mem.allocatedBytesOnHold()); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp index bb779b659ab..617e4bf5c85 100644 --- a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp +++ b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp @@ -2,7 +2,6 @@ #include "sequencedtaskexecutor.h" #include <vespa/vespalib/util/blockingthreadstackexecutor.h> -#include <vespa/vespalib/stllike/hash_map.hpp> using vespalib::BlockingThreadStackExecutor; diff --git a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h index 2b7e70d69c7..9337f393150 100644 --- a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h +++ b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h @@ -2,7 +2,6 @@ #pragma once #include "isequencedtaskexecutor.h" -#include <vespa/vespalib/stllike/hash_map.h> #include <vector> namespace vespalib { @@ -16,7 +15,7 @@ namespace search { * Class to run multiple tasks in parallel, but tasks with same * id has to be run in sequence. */ -class SequencedTaskExecutor : public ISequencedTaskExecutor +class SequencedTaskExecutor final : public ISequencedTaskExecutor { using Stats = vespalib::ExecutorStats; std::vector<std::shared_ptr<vespalib::BlockingThreadStackExecutor>> _executors; diff --git a/searchlib/src/vespa/searchlib/features/agefeature.cpp b/searchlib/src/vespa/searchlib/features/agefeature.cpp index 258648408f8..e93691e7241 100644 --- a/searchlib/src/vespa/searchlib/features/agefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/agefeature.cpp @@ -4,6 +4,7 @@ #include "valuefeature.h" #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/matchdata.h> +#include <vespa/vespalib/util/stash.h> using search::attribute::IAttributeVector; @@ -18,20 +19,18 @@ AgeExecutor::AgeExecutor(const IAttributeVector *attribute) : _attribute(attribute), _buf() { - if (_attribute != NULL) { + if (_attribute != nullptr) { _buf.allocate(attribute->getMaxValueCount()); } } -AgeBlueprint::~AgeBlueprint() -{ -} +AgeBlueprint::~AgeBlueprint() = default; void AgeExecutor::execute(uint32_t docId) { feature_t age = 10000000000.0; - if (_attribute != NULL) { + if (_attribute != nullptr) { _buf.fill(*_attribute, docId); int64_t docTime = _buf[0]; feature_t currTime = inputs().get_number(0); @@ -65,7 +64,7 @@ AgeBlueprint::setup(const search::fef::IIndexEnvironment &env, search::fef::Blueprint::UP AgeBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new AgeBlueprint()); + return std::make_unique<AgeBlueprint>(); } search::fef::FeatureExecutor & diff --git a/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp b/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp index 4776437a14b..c26d18eb11c 100644 --- a/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp @@ -8,6 +8,7 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/parameterdescriptions.h> #include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/vespalib/util/stash.h> #include <vespa/log/log.h> LOG_SETUP(".features.attributematchfeature"); diff --git a/searchlib/src/vespa/searchlib/features/bm25_feature.cpp b/searchlib/src/vespa/searchlib/features/bm25_feature.cpp index e16b4bba996..64e365b25ac 100644 --- a/searchlib/src/vespa/searchlib/features/bm25_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/bm25_feature.cpp @@ -6,8 +6,8 @@ #include <vespa/searchlib/fef/itermfielddata.h> #include <vespa/searchlib/fef/objectstore.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <cmath> -#include <memory> #include <stdexcept> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/features/closenessfeature.cpp b/searchlib/src/vespa/searchlib/features/closenessfeature.cpp index 2358e54e9f5..ccd76f5435f 100644 --- a/searchlib/src/vespa/searchlib/features/closenessfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/closenessfeature.cpp @@ -3,14 +3,14 @@ #include "closenessfeature.h" #include "utils.h" #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <vespa/log/log.h> LOG_SETUP(".features.closenessfeature"); using namespace search::fef; -namespace search { -namespace features { +namespace search::features { ClosenessExecutor::ClosenessExecutor(feature_t maxDistance, feature_t scaleDistance) : FeatureExecutor(), @@ -96,7 +96,7 @@ ClosenessBlueprint::setup(const IIndexEnvironment & env, Blueprint::UP ClosenessBlueprint::createInstance() const { - return Blueprint::UP(new ClosenessBlueprint()); + return std::make_unique<ClosenessBlueprint>(); } FeatureExecutor & @@ -105,6 +105,4 @@ ClosenessBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash &s return stash.create<ClosenessExecutor>(_maxDistance, _scaleDistance); } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/constant_feature.cpp b/searchlib/src/vespa/searchlib/features/constant_feature.cpp index ced9d95fb33..7fc0c5c05fc 100644 --- a/searchlib/src/vespa/searchlib/features/constant_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/constant_feature.cpp @@ -4,14 +4,14 @@ #include "valuefeature.h" #include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/eval/eval/value_cache/constant_value.h> +#include <vespa/vespalib/util/stash.h> #include <vespa/log/log.h> LOG_SETUP(".features.constant_feature"); using namespace search::fef; -namespace search { -namespace features { +namespace search::features { /** * Feature executor that returns a constant value. @@ -25,8 +25,8 @@ public: ConstantFeatureExecutor(const vespalib::eval::Value &value) : _value(value) {} - virtual bool isPure() override { return true; } - virtual void execute(uint32_t) override { + bool isPure() override { return true; } + void execute(uint32_t) override { outputs().set_object(0, _value); } static FeatureExecutor &create(const vespalib::eval::Value &value, vespalib::Stash &stash) { @@ -41,9 +41,7 @@ ConstantBlueprint::ConstantBlueprint() { } -ConstantBlueprint::~ConstantBlueprint() -{ -} +ConstantBlueprint::~ConstantBlueprint() = default; void ConstantBlueprint::visitDumpFeatures(const IIndexEnvironment &, @@ -54,7 +52,7 @@ ConstantBlueprint::visitDumpFeatures(const IIndexEnvironment &, Blueprint::UP ConstantBlueprint::createInstance() const { - return Blueprint::UP(new ConstantBlueprint()); + return std::make_unique<ConstantBlueprint>(); } bool @@ -88,5 +86,4 @@ ConstantBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash } } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h b/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h index 2ab7b98fabe..785eb357795 100644 --- a/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h +++ b/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h @@ -7,7 +7,7 @@ #include <vespa/eval/eval/value.h> #include <vespa/eval/eval/value_type.h> #include <vespa/eval/tensor/default_tensor_engine.h> -#include <memory> +#include <vespa/vespalib/util/stash.h> namespace search::features { diff --git a/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp b/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp index 86a71184f43..8cc75a9a424 100644 --- a/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp +++ b/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp @@ -2,6 +2,8 @@ #include "debug_attribute_wait.h" #include <vespa/vespalib/util/time.h> +#include <vespa/vespalib/util/stash.h> + using search::attribute::IAttributeVector; using namespace search::fef; diff --git a/searchlib/src/vespa/searchlib/features/debug_wait.cpp b/searchlib/src/vespa/searchlib/features/debug_wait.cpp index fb002564572..57d19618ba4 100644 --- a/searchlib/src/vespa/searchlib/features/debug_wait.cpp +++ b/searchlib/src/vespa/searchlib/features/debug_wait.cpp @@ -2,6 +2,7 @@ #include "debug_wait.h" #include <vespa/vespalib/util/time.h> +#include <vespa/vespalib/util/stash.h> using namespace search::fef; diff --git a/searchlib/src/vespa/searchlib/features/distancefeature.cpp b/searchlib/src/vespa/searchlib/features/distancefeature.cpp index 501bfd7cd14..4d7d77fe315 100644 --- a/searchlib/src/vespa/searchlib/features/distancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/distancefeature.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/fef/matchdata.h> #include <vespa/document/datatype/positiondatatype.h> #include <vespa/vespalib/geo/zcurve.h> +#include <vespa/vespalib/util/stash.h> #include <cmath> #include <limits> @@ -13,13 +14,12 @@ LOG_SETUP(".features.distancefeature"); using namespace search::fef; -namespace search { -namespace features { +namespace search::features { feature_t DistanceExecutor::calculateDistance(uint32_t docId) { - if (_location.isValid() && _pos != NULL) { + if (_location.isValid() && _pos != nullptr) { return calculate2DZDistance(docId); } return DEFAULT_DISTANCE; @@ -66,7 +66,7 @@ DistanceExecutor::DistanceExecutor(const Location & location, _pos(pos), _intBuf() { - if (_pos != NULL) { + if (_pos != nullptr) { _intBuf.allocate(_pos->getMaxValueCount()); } } @@ -86,9 +86,7 @@ DistanceBlueprint::DistanceBlueprint() : { } -DistanceBlueprint::~DistanceBlueprint() -{ -} +DistanceBlueprint::~DistanceBlueprint() = default; void DistanceBlueprint::visitDumpFeatures(const IIndexEnvironment &, @@ -99,7 +97,7 @@ DistanceBlueprint::visitDumpFeatures(const IIndexEnvironment &, Blueprint::UP DistanceBlueprint::createInstance() const { - return Blueprint::UP(new DistanceBlueprint()); + return std::make_unique<DistanceBlueprint>(); } bool @@ -116,26 +114,26 @@ DistanceBlueprint::setup(const IIndexEnvironment & env, FeatureExecutor & DistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const { - const search::attribute::IAttributeVector * pos = NULL; + const search::attribute::IAttributeVector * pos = nullptr; const Location & location = env.getLocation(); LOG(debug, "DistanceBlueprint::createExecutor location.valid='%s', '%s', alternatively '%s'", location.isValid() ? "true" : "false", _posAttr.c_str(), document::PositionDataType::getZCurveFieldName(_posAttr).c_str()); if (location.isValid()) { pos = env.getAttributeContext().getAttribute(_posAttr); - if (pos == NULL) { + if (pos == nullptr) { LOG(debug, "Failed to find attribute '%s', resorting too '%s'", _posAttr.c_str(), document::PositionDataType::getZCurveFieldName(_posAttr).c_str()); pos = env.getAttributeContext().getAttribute(document::PositionDataType::getZCurveFieldName(_posAttr)); } - if (pos != NULL) { + if (pos != nullptr) { if (!pos->isIntegerType()) { LOG(warning, "The position attribute '%s' is not an integer attribute. Will use default distance.", pos->getName().c_str()); - pos = NULL; + pos = nullptr; } else if (pos->getCollectionType() == attribute::CollectionType::WSET) { LOG(warning, "The position attribute '%s' is a weighted set attribute. Will use default distance.", pos->getName().c_str()); - pos = NULL; + pos = nullptr; } } else { LOG(warning, "The position attribute '%s' was not found. Will use default distance.", _posAttr.c_str()); @@ -145,7 +143,4 @@ DistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash return stash.create<DistanceExecutor>(location, pos); } - - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp b/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp index 834f5913af9..a2f9225d3c4 100644 --- a/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp @@ -6,10 +6,10 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/document/datatype/positiondatatype.h> #include <vespa/vespalib/geo/zcurve.h> +#include <vespa/vespalib/util/stash.h> #include <boost/algorithm/string/split.hpp> #include <boost/algorithm/string/classification.hpp> #include <cmath> -#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".features.distancetopathfeature"); @@ -25,7 +25,7 @@ DistanceToPathExecutor::DistanceToPathExecutor(std::vector<Vector2> &path, _path(), _pos(pos) { - if (_pos != NULL) { + if (_pos != nullptr) { _intBuf.allocate(_pos->getMaxValueCount()); } _path.swap(path); // avoid copy @@ -34,7 +34,7 @@ DistanceToPathExecutor::DistanceToPathExecutor(std::vector<Vector2> &path, void DistanceToPathExecutor::execute(uint32_t docId) { - if (_path.size() > 1 && _pos != NULL) { + if (_path.size() > 1 && _pos != nullptr) { double pos = -1, trip = 0, product = 0; double minSqDist = std::numeric_limits<double>::max(); _intBuf.fill(*_pos, docId); @@ -145,21 +145,21 @@ DistanceToPathBlueprint::createExecutor(const search::fef::IQueryEnvironment &en } // Lookup the attribute vector that holds document positions. - const search::attribute::IAttributeVector *pos = NULL; + const search::attribute::IAttributeVector *pos = nullptr; if (path.size() > 1) { pos = env.getAttributeContext().getAttribute(_posAttr); - if (pos == NULL) { + if (pos == nullptr) { pos = env.getAttributeContext().getAttribute(document::PositionDataType::getZCurveFieldName(_posAttr)); } - if (pos != NULL) { + if (pos != nullptr) { if (!pos->isIntegerType()) { LOG(warning, "The position attribute '%s' is not an integer attribute. Will use default distance.", pos->getName().c_str()); - pos = NULL; + pos = nullptr; } else if (pos->getCollectionType() == attribute::CollectionType::WSET) { LOG(warning, "The position attribute '%s' is a weighted set attribute. Will use default distance.", pos->getName().c_str()); - pos = NULL; + pos = nullptr; } } else { LOG(warning, "The position attribute '%s' was not found. Will use default distance.", _posAttr.c_str()); diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp index 8998f01b59e..1072607aa8a 100644 --- a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp @@ -10,12 +10,11 @@ #include <vespa/searchlib/attribute/floatbase.h> #include <vespa/searchlib/attribute/multinumericattribute.h> #include <vespa/searchlib/attribute/multienumattribute.h> -#include <type_traits> - -#include <vespa/log/log.h> #include <vespa/eval/tensor/serialization/typed_binary_format.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/stash.h> +#include <vespa/log/log.h> LOG_SETUP(".features.dotproduct"); using namespace search::attribute; diff --git a/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp b/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp index 1622a87e733..1014fe4679a 100644 --- a/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/locale/c.h> +#include <vespa/vespalib/util/stash.h> #include <cassert> namespace search::features { @@ -116,7 +117,7 @@ ElementCompletenessBlueprint::visitDumpFeatures(const fef::IIndexEnvironment &en fef::Blueprint::UP ElementCompletenessBlueprint::createInstance() const { - return Blueprint::UP(new ElementCompletenessBlueprint()); + return std::make_unique<ElementCompletenessBlueprint>(); } bool diff --git a/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp b/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp index 0676b0a46c4..45fcd013fbf 100644 --- a/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/eval/eval/llvm/compiled_function.h> #include <vespa/eval/eval/llvm/compile_cache.h> +#include <vespa/vespalib/util/stash.h> #include <vespa/log/log.h> LOG_SETUP(".features.elementsimilarity"); diff --git a/searchlib/src/vespa/searchlib/features/euclidean_distance_feature.cpp b/searchlib/src/vespa/searchlib/features/euclidean_distance_feature.cpp index cd007f1396f..9d71d358b46 100644 --- a/searchlib/src/vespa/searchlib/features/euclidean_distance_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/euclidean_distance_feature.cpp @@ -4,9 +4,9 @@ #include "euclidean_distance_feature.h" #include "array_parser.hpp" #include <vespa/searchlib/attribute/integerbase.h> -#include <vespa/searchlib/attribute/floatbase.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/vespalib/util/stash.h> #include <cmath> #include <vespa/log/log.h> @@ -15,8 +15,7 @@ LOG_SETUP(".features.euclidean_distance_feature"); using namespace search::attribute; using namespace search::fef; -namespace search { -namespace features { +namespace search::features { template <typename DataType> @@ -57,7 +56,7 @@ EuclideanDistanceBlueprint::EuclideanDistanceBlueprint() : { } -EuclideanDistanceBlueprint::~EuclideanDistanceBlueprint() {} +EuclideanDistanceBlueprint::~EuclideanDistanceBlueprint() = default; void EuclideanDistanceBlueprint::visitDumpFeatures(const IIndexEnvironment &, IDumpFeatureVisitor &) const @@ -78,7 +77,7 @@ EuclideanDistanceBlueprint::setup(const IIndexEnvironment &env, const ParameterL Blueprint::UP EuclideanDistanceBlueprint::createInstance() const { - return Blueprint::UP(new EuclideanDistanceBlueprint()); + return std::make_unique<EuclideanDistanceBlueprint>(); } namespace { @@ -97,7 +96,7 @@ FeatureExecutor & EuclideanDistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const { const IAttributeVector * attribute = env.getAttributeContext().getAttribute(_attributeName); - if (attribute == NULL) { + if (attribute == nullptr) { LOG(warning, "The attribute vector '%s' was not found in the attribute manager, returning executor with default value.", _attributeName.c_str()); return stash.create<SingleZeroValueExecutor>(); @@ -118,6 +117,5 @@ EuclideanDistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespali } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp b/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp index d7e17187ff4..18a15ccf541 100644 --- a/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp @@ -7,12 +7,11 @@ #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/fieldtype.h> #include <vespa/searchlib/fef/featurenamebuilder.h> -#include <vespa/searchlib/fef/itermdata.h> #include <vespa/searchlib/fef/handle.h> +#include <vespa/vespalib/util/stash.h> #include <sstream> -namespace search { -namespace features { +namespace search::features { IndexFieldInfoExecutor::IndexFieldInfoExecutor(feature_t type, feature_t isFilter, [[maybe_unused]] uint32_t field, uint32_t fieldHandle) @@ -238,5 +237,4 @@ FieldInfoBlueprint::createExecutor(const fef::IQueryEnvironment &queryEnv, vespa return stash.create<ValueExecutor>(values); } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp b/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp index d0680e8fc19..74bfa156b5f 100644 --- a/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp @@ -6,13 +6,12 @@ #include <vespa/searchlib/fef/itermdata.h> #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> -#include <vespa/searchlib/fef/fieldtype.h> -#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> + using namespace search::fef; -namespace search { -namespace features { +namespace search::features { FieldLengthExecutor:: FieldLengthExecutor(const IQueryEnvironment &env, @@ -63,7 +62,7 @@ FieldLengthExecutor::handle_bind_match_data(const MatchData &md) FieldLengthBlueprint::FieldLengthBlueprint() : Blueprint("fieldLength"), - _field(NULL) + _field(nullptr) { } @@ -86,7 +85,7 @@ FieldLengthBlueprint::setup(const IIndexEnvironment &env, Blueprint::UP FieldLengthBlueprint::createInstance() const { - return Blueprint::UP(new FieldLengthBlueprint()); + return std::make_unique<FieldLengthBlueprint>(); } FeatureExecutor & @@ -100,4 +99,4 @@ FieldLengthBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Sta return stash.create<FieldLengthExecutor>(env, _field->id()); } -}} +} diff --git a/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp b/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp index 583afa6e698..7ad9eef0506 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp @@ -7,6 +7,7 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/locale/c.h> +#include <vespa/vespalib/util/stash.h> using namespace search::fef; using CollectionType = FieldInfo::CollectionType; @@ -92,14 +93,12 @@ FieldMatchExecutor::handle_bind_match_data(const fef::MatchData &md) FieldMatchBlueprint::FieldMatchBlueprint() : Blueprint("fieldMatch"), - _field(NULL), + _field(nullptr), _params() { } -FieldMatchBlueprint::~FieldMatchBlueprint() -{ -} +FieldMatchBlueprint::~FieldMatchBlueprint() = default; void FieldMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env, @@ -158,7 +157,7 @@ FieldMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env, Blueprint::UP FieldMatchBlueprint::createInstance() const { - return Blueprint::UP(new FieldMatchBlueprint()); + return std::make_unique<FieldMatchBlueprint>(); } bool @@ -306,5 +305,4 @@ FieldMatchBlueprint::createExecutor(const IQueryEnvironment & env, vespalib::Sta return stash.create<FieldMatchExecutor>(env, *_field, _params); } - } diff --git a/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp b/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp index a7a00bee956..2f065fee289 100644 --- a/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp @@ -8,9 +8,10 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/itermdata.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> -namespace search { -namespace features { + +namespace search::features { FieldTermMatchExecutor::FieldTermMatchExecutor(const search::fef::IQueryEnvironment &env, uint32_t fieldId, uint32_t termId) : @@ -122,7 +123,7 @@ FieldTermMatchBlueprint::setup(const search::fef::IIndexEnvironment &env, search::fef::Blueprint::UP FieldTermMatchBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new FieldTermMatchBlueprint()); + return std::make_unique<FieldTermMatchBlueprint>(); } search::fef::FeatureExecutor & @@ -131,4 +132,4 @@ FieldTermMatchBlueprint::createExecutor(const search::fef::IQueryEnvironment &en return stash.create<FieldTermMatchExecutor>(env, _fieldId, _termId); } -}} +} diff --git a/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp b/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp index 2e6bae14a44..9d1831c6102 100644 --- a/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp @@ -4,11 +4,12 @@ #include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> + using namespace search::fef; -namespace search { -namespace features { +namespace search::features { void FirstPhaseExecutor::execute(uint32_t) @@ -34,7 +35,7 @@ FirstPhaseBlueprint::visitDumpFeatures(const IIndexEnvironment &, Blueprint::UP FirstPhaseBlueprint::createInstance() const { - return Blueprint::UP(new FirstPhaseBlueprint()); + return std::make_unique<FirstPhaseBlueprint>(); } bool @@ -54,5 +55,4 @@ FirstPhaseBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash & } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp b/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp index eda83b991bf..e43faaec4e1 100644 --- a/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp @@ -7,6 +7,8 @@ #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/locale/c.h> +#include <vespa/vespalib/util/stash.h> + #include <cassert> #include <vespa/log/log.h> @@ -285,7 +287,7 @@ FlowCompletenessBlueprint::visitDumpFeatures(const fef::IIndexEnvironment &env, fef::Blueprint::UP FlowCompletenessBlueprint::createInstance() const { - return Blueprint::UP(new FlowCompletenessBlueprint()); + return std::make_unique<FlowCompletenessBlueprint>(); } bool @@ -318,6 +320,4 @@ FlowCompletenessBlueprint::createExecutor(const fef::IQueryEnvironment &env, ves return stash.create<FlowCompletenessExecutor>(env, _params); } -//----------------------------------------------------------------------------- - } diff --git a/searchlib/src/vespa/searchlib/features/foreachfeature.cpp b/searchlib/src/vespa/searchlib/features/foreachfeature.cpp index a67d8001a36..21167dd23d4 100644 --- a/searchlib/src/vespa/searchlib/features/foreachfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/foreachfeature.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> #include <boost/algorithm/string/replace.hpp> #include <vespa/log/log.h> @@ -120,9 +121,7 @@ ForeachBlueprint::ForeachBlueprint() : { } -ForeachBlueprint::~ForeachBlueprint() -{ -} +ForeachBlueprint::~ForeachBlueprint() = default; void ForeachBlueprint::visitDumpFeatures(const IIndexEnvironment &, @@ -171,13 +170,13 @@ ForeachBlueprint::setup(const IIndexEnvironment & env, Blueprint::UP ForeachBlueprint::createInstance() const { - return Blueprint::UP(new ForeachBlueprint()); + return std::make_unique<ForeachBlueprint>(); } FeatureExecutor & ForeachBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash &stash) const { - if (_executorCreator.get() != NULL) { + if (_executorCreator) { return _executorCreator->create(_num_inputs, stash); } return stash.create<SingleZeroValueExecutor>(); diff --git a/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp b/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp index 11ae8305e16..6e621f61034 100644 --- a/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp @@ -3,14 +3,14 @@ #include "freshnessfeature.h" #include "utils.h" #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <vespa/log/log.h> LOG_SETUP(".features.freshnessfeature"); using namespace search::fef; -namespace search { -namespace features { +namespace search::features { FreshnessExecutor::FreshnessExecutor(feature_t maxAge, feature_t scaleAge) : FeatureExecutor(), @@ -86,7 +86,7 @@ FreshnessBlueprint::setup(const IIndexEnvironment & env, Blueprint::UP FreshnessBlueprint::createInstance() const { - return Blueprint::UP(new FreshnessBlueprint()); + return std::make_unique<FreshnessBlueprint>(); } fef::ParameterDescriptions @@ -101,7 +101,4 @@ FreshnessBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash &s return stash.create<FreshnessExecutor>(_maxAge, _scaleAge); } - -} // namespace features -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp index fd1faeae5ea..e2e8a206099 100644 --- a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp @@ -5,12 +5,13 @@ #include "weighted_set_parser.h" #include "dotproductfeature.h" -#include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/attribute/imported_attribute_vector_read_guard.h> #include <vespa/searchlib/attribute/multinumericattribute.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/searchcommon/common/datatype.h> +#include <vespa/vespalib/util/stash.h> + #include <vespa/log/log.h> LOG_SETUP(".features.internalmaxreduceprodjoin"); diff --git a/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp b/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp index 45baf646656..89f7751a369 100644 --- a/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp @@ -3,11 +3,11 @@ #include "item_raw_score_feature.h" #include "valuefeature.h" #include "utils.h" +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { void ItemRawScoreExecutor::execute(uint32_t docId) @@ -89,6 +89,4 @@ ItemRawScoreBlueprint::resolve(const IQueryEnvironment &env, return handles; } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp b/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp index a5e3e2da5ba..dc689459ff3 100644 --- a/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp @@ -6,6 +6,8 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/locale/c.h> +#include <vespa/vespalib/util/stash.h> + namespace search::features { @@ -168,7 +170,7 @@ JaroWinklerDistanceBlueprint::setup(const search::fef::IIndexEnvironment &env, search::fef::Blueprint::UP JaroWinklerDistanceBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new JaroWinklerDistanceBlueprint()); + return std::make_unique<JaroWinklerDistanceBlueprint>(); } search::fef::FeatureExecutor & diff --git a/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp b/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp index fd453e17eb1..c061ace4854 100644 --- a/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp @@ -3,11 +3,11 @@ #include "matchcountfeature.h" #include "utils.h" #include "valuefeature.h" +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { MatchCountExecutor::MatchCountExecutor(uint32_t fieldId, const IQueryEnvironment &env) : FeatureExecutor(), @@ -43,7 +43,7 @@ MatchCountExecutor::handle_bind_match_data(const MatchData &md) MatchCountBlueprint::MatchCountBlueprint() : Blueprint("matchCount"), - _field(NULL) + _field(nullptr) { } @@ -63,7 +63,7 @@ MatchCountBlueprint::setup(const IIndexEnvironment &, const ParameterList & para Blueprint::UP MatchCountBlueprint::createInstance() const { - return Blueprint::UP(new MatchCountBlueprint()); + return std::make_unique<MatchCountBlueprint>(); } FeatureExecutor & @@ -75,5 +75,4 @@ MatchCountBlueprint::createExecutor(const IQueryEnvironment & queryEnv, vespalib return stash.create<MatchCountExecutor>(_field->id(), queryEnv); } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/matchesfeature.cpp b/searchlib/src/vespa/searchlib/features/matchesfeature.cpp index f4788ee74c8..a99c2330ee3 100644 --- a/searchlib/src/vespa/searchlib/features/matchesfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/matchesfeature.cpp @@ -4,11 +4,12 @@ #include "utils.h" #include "valuefeature.h" #include <vespa/searchlib/fef/fieldinfo.h> +#include <vespa/vespalib/util/stash.h> + using namespace search::fef; -namespace search { -namespace features { +namespace search::features { MatchesExecutor::MatchesExecutor(uint32_t fieldId, const search::fef::IQueryEnvironment &env, @@ -47,7 +48,7 @@ MatchesExecutor::handle_bind_match_data(const MatchData &md) MatchesBlueprint::MatchesBlueprint() : Blueprint("matches"), - _field(NULL), + _field(nullptr), _termIdx(std::numeric_limits<uint32_t>::max()) { } @@ -73,7 +74,7 @@ MatchesBlueprint::setup(const IIndexEnvironment &, Blueprint::UP MatchesBlueprint::createInstance() const { - return Blueprint::UP(new MatchesBlueprint()); + return std::make_unique<MatchesBlueprint>(); } FeatureExecutor & @@ -89,5 +90,4 @@ MatchesBlueprint::createExecutor(const IQueryEnvironment & queryEnv, vespalib::S } } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/matchfeature.cpp b/searchlib/src/vespa/searchlib/features/matchfeature.cpp index 7210b8b67e9..f6843df1a2f 100644 --- a/searchlib/src/vespa/searchlib/features/matchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/matchfeature.cpp @@ -2,17 +2,16 @@ #include "matchfeature.h" #include "utils.h" -#include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/properties.h> -#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> + using namespace search::fef; using CollectionType = FieldInfo::CollectionType; -namespace search { -namespace features { +namespace search::features { MatchExecutor::MatchExecutor(const MatchParams & params) : FeatureExecutor(), @@ -46,9 +45,7 @@ MatchBlueprint::MatchBlueprint() : { } -MatchBlueprint::~MatchBlueprint() -{ -} +MatchBlueprint::~MatchBlueprint() = default; void MatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env, @@ -61,7 +58,7 @@ MatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env, Blueprint::UP MatchBlueprint::createInstance() const { - return Blueprint::UP(new MatchBlueprint()); + return std::make_unique<MatchBlueprint>(); } bool @@ -101,6 +98,4 @@ MatchBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &st return stash.create<MatchExecutor>(_params); } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp b/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp index 7865e32849f..5ede14130ec 100644 --- a/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp @@ -2,11 +2,11 @@ #include "native_dot_product_feature.h" #include "utils.h" +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { NativeDotProductExecutor::NativeDotProductExecutor(const search::fef::IQueryEnvironment &env) : FeatureExecutor(), @@ -80,5 +80,4 @@ NativeDotProductBlueprint::createExecutor(const IQueryEnvironment &queryEnv, ves } } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp b/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp index 5dda159f629..865ea9fc3c4 100644 --- a/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp @@ -7,11 +7,11 @@ #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/itablemanager.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { feature_t NativeAttributeMatchExecutor::calculateScore(const CachedTermData &td, const TermFieldMatchData &tfmd) @@ -115,7 +115,7 @@ NativeAttributeMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env, Blueprint::UP NativeAttributeMatchBlueprint::createInstance() const { - return Blueprint::UP(new NativeAttributeMatchBlueprint()); + return std::make_unique<NativeAttributeMatchBlueprint>(); } fef::ParameterDescriptions @@ -160,6 +160,4 @@ NativeAttributeMatchBlueprint::createExecutor(const IQueryEnvironment &env, vesp return NativeAttributeMatchExecutor::createExecutor(env, _params, stash); } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp b/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp index 089a8102d6e..2b6841750ad 100644 --- a/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp @@ -7,11 +7,11 @@ #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/itablemanager.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { const uint32_t NativeFieldMatchParam::NOT_DEF_FIELD_LENGTH(std::numeric_limits<uint32_t>::max()); @@ -95,9 +95,7 @@ NativeFieldMatchBlueprint::NativeFieldMatchBlueprint() : { } -NativeFieldMatchBlueprint::~NativeFieldMatchBlueprint() -{ -} +NativeFieldMatchBlueprint::~NativeFieldMatchBlueprint() = default; void NativeFieldMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env, @@ -110,7 +108,7 @@ NativeFieldMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env, Blueprint::UP NativeFieldMatchBlueprint::createInstance() const { - return Blueprint::UP(new NativeFieldMatchBlueprint()); + return std::make_unique<NativeFieldMatchBlueprint>(); } bool @@ -181,5 +179,4 @@ NativeFieldMatchBlueprint::createExecutor(const IQueryEnvironment &env, vespalib } } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp b/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp index a31d9207e05..2809417e382 100644 --- a/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp @@ -7,12 +7,12 @@ #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/itablemanager.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <map> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { feature_t NativeProximityExecutor::calculateScoreForField(const FieldSetup & fs, uint32_t docId) @@ -136,9 +136,7 @@ NativeProximityBlueprint::NativeProximityBlueprint() : { } -NativeProximityBlueprint::~NativeProximityBlueprint() -{ -} +NativeProximityBlueprint::~NativeProximityBlueprint() = default; void NativeProximityBlueprint::visitDumpFeatures(const IIndexEnvironment & env, @@ -151,7 +149,7 @@ NativeProximityBlueprint::visitDumpFeatures(const IIndexEnvironment & env, Blueprint::UP NativeProximityBlueprint::createInstance() const { - return Blueprint::UP(new NativeProximityBlueprint()); + return std::make_unique<NativeProximityBlueprint>(); } bool @@ -168,12 +166,12 @@ NativeProximityBlueprint::setup(const IIndexEnvironment & env, NativeProximityParam & param = _params.vector[fieldId]; param.field = true; if ((param.proximityTable = - util::lookupTable(env, getBaseName(), "proximityTable", info->name(), _defaultProximityBoost)) == NULL) + util::lookupTable(env, getBaseName(), "proximityTable", info->name(), _defaultProximityBoost)) == nullptr) { return false; } if ((param.revProximityTable = - util::lookupTable(env, getBaseName(), "reverseProximityTable", info->name(), _defaultRevProximityBoost)) == NULL) + util::lookupTable(env, getBaseName(), "reverseProximityTable", info->name(), _defaultRevProximityBoost)) == nullptr) { return false; } @@ -190,7 +188,7 @@ NativeProximityBlueprint::setup(const IIndexEnvironment & env, if (NativeRankBlueprint::useTableNormalization(env)) { const Table * fp = param.proximityTable; const Table * rp = param.revProximityTable; - if (fp != NULL && rp != NULL) { + if (fp != nullptr && rp != nullptr) { double value = (fp->max() * param.proximityImportance) + (rp->max() * (1 - param.proximityImportance)); _params.setMaxTableSums(fieldId, value); @@ -217,6 +215,4 @@ NativeProximityBlueprint::createExecutor(const IQueryEnvironment &env, vespalib: } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp b/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp index b519c4f4b7f..a980c265484 100644 --- a/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp @@ -4,6 +4,7 @@ #include "valuefeature.h" #include "utils.h" #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <sstream> #include <vespa/log/log.h> @@ -30,8 +31,7 @@ buildFeatureName(const vespalib::string & baseName, const search::features::Fiel } -namespace search { -namespace features { +namespace search::features { FieldWrapper::FieldWrapper(const IIndexEnvironment & env, const ParameterList & fields, @@ -93,7 +93,7 @@ NativeRankBlueprint::visitDumpFeatures(const IIndexEnvironment & env, Blueprint::UP NativeRankBlueprint::createInstance() const { - return Blueprint::UP(new NativeRankBlueprint()); + return std::make_unique<NativeRankBlueprint>(); } bool @@ -168,6 +168,4 @@ NativeRankBlueprint::useTableNormalization(const search::fef::IIndexEnvironment return (!(norm.found() && (norm.get() == vespalib::string("false")))); } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/nowfeature.cpp b/searchlib/src/vespa/searchlib/features/nowfeature.cpp index 074acb2e890..d6059592cf3 100644 --- a/searchlib/src/vespa/searchlib/features/nowfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nowfeature.cpp @@ -3,6 +3,7 @@ #include "nowfeature.h" #include <vespa/searchlib/fef/queryproperties.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <chrono> namespace search::features { diff --git a/searchlib/src/vespa/searchlib/features/proximityfeature.cpp b/searchlib/src/vespa/searchlib/features/proximityfeature.cpp index f625e30f378..daeb4af6569 100644 --- a/searchlib/src/vespa/searchlib/features/proximityfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/proximityfeature.cpp @@ -2,13 +2,11 @@ #include "proximityfeature.h" #include "utils.h" -#include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/itermdata.h> -#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> -namespace search { -namespace features { +namespace search::features { ProximityConfig::ProximityConfig() : fieldId(search::fef::IllegalHandle), @@ -139,7 +137,7 @@ ProximityBlueprint::setup(const search::fef::IIndexEnvironment &env, search::fef::Blueprint::UP ProximityBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new ProximityBlueprint()); + return std::make_unique<ProximityBlueprint>(); } search::fef::FeatureExecutor & @@ -148,4 +146,4 @@ ProximityBlueprint::createExecutor(const search::fef::IQueryEnvironment &env, ve return stash.create<ProximityExecutor>(env, _config); } -}} +} diff --git a/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp b/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp index b4b6a1b0eb4..56e8810e14b 100644 --- a/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp @@ -3,15 +3,14 @@ #include "querycompletenessfeature.h" #include "utils.h" #include <vespa/searchlib/fef/featurenamebuilder.h> -#include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/itermdata.h> +#include <vespa/vespalib/util/stash.h> #include <limits> #include <vespa/log/log.h> LOG_SETUP(".features.querycompleteness"); -namespace search { -namespace features { +namespace search::features { QueryCompletenessConfig::QueryCompletenessConfig() : fieldId(search::fef::IllegalHandle), @@ -106,7 +105,7 @@ QueryCompletenessBlueprint::setup(const search::fef::IIndexEnvironment &env, search::fef::Blueprint::UP QueryCompletenessBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new QueryCompletenessBlueprint()); + return std::make_unique<QueryCompletenessBlueprint>(); } search::fef::FeatureExecutor & @@ -115,4 +114,4 @@ QueryCompletenessBlueprint::createExecutor(const search::fef::IQueryEnvironment return stash.create<QueryCompletenessExecutor>(env, _config); } -}} +} diff --git a/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp b/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp index dfc4af059a1..cbaf9e97cb6 100644 --- a/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp @@ -4,15 +4,14 @@ #include "valuefeature.h" #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/fieldinfo.h> -#include <vespa/searchlib/fef/fieldtype.h> #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/itermdata.h> -#include <vespa/searchlib/fef/handle.h> +#include <vespa/vespalib/util/stash.h> + using namespace search::fef; -namespace search { -namespace features { +namespace search::features { QueryTermCountBlueprint::QueryTermCountBlueprint() : Blueprint("queryTermCount") @@ -30,7 +29,7 @@ QueryTermCountBlueprint::visitDumpFeatures(const IIndexEnvironment & env, Blueprint::UP QueryTermCountBlueprint::createInstance() const { - return Blueprint::UP(new QueryTermCountBlueprint()); + return std::make_unique<QueryTermCountBlueprint>(); } bool @@ -49,6 +48,4 @@ QueryTermCountBlueprint::createExecutor(const IQueryEnvironment &env, vespalib:: return stash.create<ValueExecutor>(values); } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp index dd0c67df45c..8d4af8fd88d 100644 --- a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp @@ -3,6 +3,7 @@ #include "random_normal_feature.h" #include "utils.h" #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <chrono> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp index bde0cedaed0..f3d33c7dc29 100644 --- a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp @@ -3,6 +3,7 @@ #include "random_normal_stable_feature.h" #include "utils.h" #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <vespa/log/log.h> LOG_SETUP(".features.randomnormalstablefeature"); @@ -41,7 +42,7 @@ RandomNormalStableBlueprint::visitDumpFeatures(const search::fef::IIndexEnvironm search::fef::Blueprint::UP RandomNormalStableBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new RandomNormalStableBlueprint()); + return std::make_unique<RandomNormalStableBlueprint>(); } bool @@ -75,5 +76,4 @@ RandomNormalStableBlueprint::createExecutor(const search::fef::IQueryEnvironment return stash.create<RandomNormalStableExecutor>(seed, _mean, _stddev); } - } diff --git a/searchlib/src/vespa/searchlib/features/randomfeature.cpp b/searchlib/src/vespa/searchlib/features/randomfeature.cpp index 18b0cf616d4..95daebd0452 100644 --- a/searchlib/src/vespa/searchlib/features/randomfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/randomfeature.cpp @@ -3,7 +3,9 @@ #include "randomfeature.h" #include "utils.h" #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <chrono> + #include <vespa/log/log.h> LOG_SETUP(".features.randomfeature"); @@ -75,5 +77,4 @@ RandomBlueprint::createExecutor(const fef::IQueryEnvironment &env, vespalib::Sta return stash.create<RandomExecutor>(seed, matchSeed); } - } diff --git a/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp b/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp index 61355581214..ea696c75eff 100644 --- a/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp @@ -2,6 +2,7 @@ #include "raw_score_feature.h" #include "utils.h" +#include <vespa/vespalib/util/stash.h> using namespace search::fef; diff --git a/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp b/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp index c27936332d2..436e9bc0a0c 100644 --- a/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp @@ -2,14 +2,11 @@ #include "reverseproximityfeature.h" #include "utils.h" -#include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> -#include <vespa/searchlib/fef/fieldtype.h> #include <vespa/searchlib/fef/itermdata.h> -#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> -namespace search { -namespace features { +namespace search::features { ReverseProximityConfig::ReverseProximityConfig() : fieldId(search::fef::IllegalHandle), @@ -126,7 +123,7 @@ ReverseProximityBlueprint::setup(const search::fef::IIndexEnvironment &env, search::fef::Blueprint::UP ReverseProximityBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new ReverseProximityBlueprint()); + return std::make_unique<ReverseProximityBlueprint>(); } search::fef::FeatureExecutor & @@ -135,4 +132,4 @@ ReverseProximityBlueprint::createExecutor(const search::fef::IQueryEnvironment & return stash.create<ReverseProximityExecutor>(env, _config); } -}} +} diff --git a/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp b/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp index 6c52b6edb76..aaff46af93e 100644 --- a/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp @@ -2,11 +2,11 @@ #include "subqueries_feature.h" #include "utils.h" +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { SubqueriesExecutor::SubqueriesExecutor(const IQueryEnvironment &env, uint32_t fieldId) @@ -59,5 +59,4 @@ SubqueriesBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib: return stash.create<SubqueriesExecutor>(queryEnv, _field->id()); } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp b/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp index a4ca8524140..7fd487d8bdd 100644 --- a/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp @@ -2,10 +2,10 @@ #include "term_field_md_feature.h" #include "utils.h" -#include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/itablemanager.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <cassert> using namespace search::fef; @@ -83,7 +83,7 @@ TermFieldMdBlueprint::visitDumpFeatures(const IIndexEnvironment &, Blueprint::UP TermFieldMdBlueprint::createInstance() const { - return Blueprint::UP(new TermFieldMdBlueprint()); + return std::make_unique<TermFieldMdBlueprint>(); } bool diff --git a/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp b/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp index fb38a49d6eb..e9f48421fcf 100644 --- a/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp @@ -5,11 +5,11 @@ #include "utils.h" #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { TermDistanceExecutor::TermDistanceExecutor(const IQueryEnvironment & env, @@ -61,7 +61,7 @@ TermDistanceBlueprint::visitDumpFeatures(const IIndexEnvironment &, Blueprint::UP TermDistanceBlueprint::createInstance() const { - return Blueprint::UP(new TermDistanceBlueprint()); + return std::make_unique<TermDistanceBlueprint>(); } bool @@ -97,6 +97,4 @@ TermDistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::St } } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp b/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp index 5990d62cb25..433bf6134b8 100644 --- a/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp @@ -6,6 +6,8 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/locale/c.h> +#include <vespa/vespalib/util/stash.h> + #include <vespa/log/log.h> LOG_SETUP(".features.termeditdistance"); @@ -219,7 +221,7 @@ TermEditDistanceBlueprint::setup(const search::fef::IIndexEnvironment &env, search::fef::Blueprint::UP TermEditDistanceBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new TermEditDistanceBlueprint()); + return std::make_unique<TermEditDistanceBlueprint>(); } search::fef::FeatureExecutor & diff --git a/searchlib/src/vespa/searchlib/features/termfeature.cpp b/searchlib/src/vespa/searchlib/features/termfeature.cpp index d6df25cc2b9..90540726227 100644 --- a/searchlib/src/vespa/searchlib/features/termfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/termfeature.cpp @@ -4,15 +4,14 @@ #include "utils.h" #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> -#include <vespa/searchlib/fef/fieldtype.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/itermdata.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { TermExecutor::TermExecutor(const search::fef::IQueryEnvironment &env, uint32_t termId) : @@ -21,7 +20,7 @@ TermExecutor::TermExecutor(const search::fef::IQueryEnvironment &env, _connectedness(util::lookupConnectedness(env, termId)), _significance(0) { - if (_termData != NULL) { + if (_termData != nullptr) { feature_t fallback = util::getSignificance(*_termData); _significance = util::lookupSignificance(env, termId, fallback); } @@ -30,7 +29,7 @@ TermExecutor::TermExecutor(const search::fef::IQueryEnvironment &env, void TermExecutor::execute(uint32_t) { - if (_termData == NULL) { // this query term is not present in the query + if (_termData == nullptr) { // this query term is not present in the query outputs().set_number(0, 0.0f); // connectedness outputs().set_number(1, 0.0f); // significance (1 - frequency) outputs().set_number(2, 0.0f); // weight @@ -76,7 +75,7 @@ TermBlueprint::setup(const search::fef::IIndexEnvironment &, search::fef::Blueprint::UP TermBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new TermBlueprint()); + return std::make_unique<TermBlueprint>(); } search::fef::FeatureExecutor & @@ -85,4 +84,4 @@ TermBlueprint::createExecutor(const search::fef::IQueryEnvironment &env, vespali return stash.create<TermExecutor>(env, _termId); } -}} +} diff --git a/searchlib/src/vespa/searchlib/features/terminfofeature.cpp b/searchlib/src/vespa/searchlib/features/terminfofeature.cpp index 4f32cda8c86..ca7a5d8e248 100644 --- a/searchlib/src/vespa/searchlib/features/terminfofeature.cpp +++ b/searchlib/src/vespa/searchlib/features/terminfofeature.cpp @@ -3,30 +3,28 @@ #include "terminfofeature.h" #include "valuefeature.h" #include <vespa/searchlib/fef/properties.h> -#include <vespa/searchlib/fef/fieldinfo.h> -#include <vespa/searchlib/fef/fieldtype.h> #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/itermdata.h> -#include <vespa/searchlib/fef/handle.h> +#include <vespa/vespalib/util/stash.h> -namespace search { -namespace features { +using namespace search::fef; +namespace search::features { TermInfoBlueprint::TermInfoBlueprint() - : search::fef::Blueprint("termInfo"), + : Blueprint("termInfo"), _termIdx(0) { } void -TermInfoBlueprint::visitDumpFeatures(const search::fef::IIndexEnvironment &, - search::fef::IDumpFeatureVisitor &) const +TermInfoBlueprint::visitDumpFeatures(const IIndexEnvironment &, + IDumpFeatureVisitor &) const { } bool -TermInfoBlueprint::setup(const search::fef::IIndexEnvironment &, - const search::fef::ParameterList & params) +TermInfoBlueprint::setup(const IIndexEnvironment &, + const ParameterList & params) { _termIdx = params[0].asInteger(); describeOutput("queryidx", "The index of the first term with the given " @@ -34,8 +32,8 @@ TermInfoBlueprint::setup(const search::fef::IIndexEnvironment &, return true; } -search::fef::FeatureExecutor & -TermInfoBlueprint::createExecutor(const search::fef::IQueryEnvironment &queryEnv, vespalib::Stash &stash) const +FeatureExecutor & +TermInfoBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stash &stash) const { feature_t queryIdx = -1.0; if (queryEnv.getNumTerms() > _termIdx) { @@ -46,5 +44,4 @@ TermInfoBlueprint::createExecutor(const search::fef::IQueryEnvironment &queryEnv return stash.create<ValueExecutor>(values); } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp b/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp index 3cdb20a16e5..a0e0a8759a0 100644 --- a/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp @@ -3,6 +3,7 @@ #include "text_similarity_feature.h" #include <vespa/searchlib/fef/itermdata.h> #include <vespa/searchlib/fef/featurenamebuilder.h> +#include <vespa/vespalib/util/stash.h> namespace search::features { @@ -194,7 +195,7 @@ TextSimilarityBlueprint::visitDumpFeatures(const fef::IIndexEnvironment &env, fef::Blueprint::UP TextSimilarityBlueprint::createInstance() const { - return Blueprint::UP(new TextSimilarityBlueprint()); + return std::make_unique<TextSimilarityBlueprint>(); } bool @@ -218,6 +219,4 @@ TextSimilarityBlueprint::createExecutor(const fef::IQueryEnvironment &env, vespa return stash.create<TextSimilarityExecutor>(env, _field_id); } -//----------------------------------------------------------------------------- - } diff --git a/searchlib/src/vespa/searchlib/features/valuefeature.cpp b/searchlib/src/vespa/searchlib/features/valuefeature.cpp index 339cc5431f1..2b91cf1688b 100644 --- a/searchlib/src/vespa/searchlib/features/valuefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/valuefeature.cpp @@ -2,12 +2,13 @@ #include "valuefeature.h" #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/stash.h> -namespace search { -namespace features { +using namespace search::fef; +namespace search::features { ValueExecutor::ValueExecutor(const std::vector<feature_t> & values) : - search::fef::FeatureExecutor(), + FeatureExecutor(), _values(values) { } @@ -27,22 +28,20 @@ SingleZeroValueExecutor::execute(uint32_t) } ValueBlueprint::ValueBlueprint() : - search::fef::Blueprint("value"), + Blueprint("value"), _values() { } -ValueBlueprint::~ValueBlueprint() {} +ValueBlueprint::~ValueBlueprint() = default; void -ValueBlueprint::visitDumpFeatures(const search::fef::IIndexEnvironment &, - search::fef::IDumpFeatureVisitor &) const +ValueBlueprint::visitDumpFeatures(const IIndexEnvironment &, IDumpFeatureVisitor &) const { } bool -ValueBlueprint::setup(const search::fef::IIndexEnvironment &, - const search::fef::ParameterList & params) +ValueBlueprint::setup(const IIndexEnvironment &, const ParameterList & params) { for (uint32_t i = 0; i < params.size(); ++i) { _values.push_back(params[i].asDouble()); @@ -56,13 +55,12 @@ ValueBlueprint::setup(const search::fef::IIndexEnvironment &, return true; } -search::fef::FeatureExecutor & -ValueBlueprint::createExecutor(const search::fef::IQueryEnvironment &queryEnv, vespalib::Stash &stash) const +FeatureExecutor & +ValueBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stash &stash) const { (void) queryEnv; return stash.create<ValueExecutor>(_values); } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/rank_program.cpp b/searchlib/src/vespa/searchlib/fef/rank_program.cpp index d39be693806..bd8abe41afb 100644 --- a/searchlib/src/vespa/searchlib/fef/rank_program.cpp +++ b/searchlib/src/vespa/searchlib/fef/rank_program.cpp @@ -6,6 +6,9 @@ #include <algorithm> #include <cassert> +#include <vespa/log/log.h> +LOG_SETUP(".fef.rankprogram"); + using vespalib::Stash; namespace search::fef { @@ -42,7 +45,7 @@ struct OverrideVisitor : public IPropertiesVisitor { auto pos = feature_map.find(key); if (pos != feature_map.end()) { - overrides.push_back(Override(pos->second, vespalib::locale::c::strtod(values.get().c_str(), nullptr))); + overrides.emplace_back(pos->second, vespalib::locale::c::strtod(values.get().c_str(), nullptr)); } } }; @@ -175,6 +178,7 @@ RankProgram::setup(const MatchData &md, auto override_end = overrides.end(); const auto &specs = _resolver->getExecutorSpecs(); + _executors.reserve(specs.size()); for (uint32_t i = 0; i < specs.size(); ++i) { vespalib::ArrayRef<NumberOrObject> outputs = _hot_stash.create_array<NumberOrObject>(specs[i].output_types.size()); StashSelector stash(_hot_stash, _cold_stash); @@ -216,6 +220,8 @@ RankProgram::setup(const MatchData &md, } } assert(_executors.size() == specs.size()); + LOG(debug, "Num executors = %ld, hot stash = %ld, cold stash = %ld, match data fields = %d", + _executors.size(), _hot_stash.count_used(), _cold_stash.count_used(), md.getNumTermFields()); } FeatureResolver diff --git a/searchlib/src/vespa/searchlib/fef/rank_program.h b/searchlib/src/vespa/searchlib/fef/rank_program.h index 0e5c390162a..aa6f77abce4 100644 --- a/searchlib/src/vespa/searchlib/fef/rank_program.h +++ b/searchlib/src/vespa/searchlib/fef/rank_program.h @@ -9,8 +9,8 @@ #include "feature_resolver.h" #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/array.h> +#include <vespa/vespalib/util/stash.h> #include <set> -#include <vector> namespace search::fef { diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp index ee88be8ad00..88f4a07d95d 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp @@ -12,7 +12,7 @@ class VisitorAdapter : public search::fef::IDumpFeatureVisitor { search::fef::BlueprintResolver &_resolver; public: - VisitorAdapter(search::fef::BlueprintResolver &resolver) + explicit VisitorAdapter(search::fef::BlueprintResolver &resolver) : _resolver(resolver) {} void visitDumpFeature(const vespalib::string &name) override { _resolver.addSeed(name); @@ -60,7 +60,8 @@ RankSetup::RankSetup(const BlueprintFactory &factory, const IIndexEnvironment &i _diversityCutoffFactor(10.0), _diversityCutoffStrategy("loose"), _softTimeoutEnabled(false), - _softTimeoutTailCost(0.1) + _softTimeoutTailCost(0.1), + _softTimeoutFactor(0.5) { } RankSetup::~RankSetup() = default; @@ -71,13 +72,13 @@ RankSetup::configure() setFirstPhaseRank(rank::FirstPhase::lookup(_indexEnv.getProperties())); setSecondPhaseRank(rank::SecondPhase::lookup(_indexEnv.getProperties())); std::vector<vespalib::string> summaryFeatures = summary::Feature::lookup(_indexEnv.getProperties()); - for (uint32_t i = 0; i < summaryFeatures.size(); ++i) { - addSummaryFeature(summaryFeatures[i]); + for (const auto & feature : summaryFeatures) { + addSummaryFeature(feature); } setIgnoreDefaultRankFeatures(dump::IgnoreDefaultFeatures::check(_indexEnv.getProperties())); std::vector<vespalib::string> dumpFeatures = dump::Feature::lookup(_indexEnv.getProperties()); - for (uint32_t i = 0; i < dumpFeatures.size(); ++i) { - addDumpFeature(dumpFeatures[i]); + for (const auto & feature : dumpFeatures) { + addDumpFeature(feature); } split_unpacking_iterators(matching::SplitUnpackingIterators::check(_indexEnv.getProperties())); delay_unpacking_iterators(matching::DelayUnpackingIterators::check(_indexEnv.getProperties())); @@ -159,15 +160,15 @@ RankSetup::compile() _compileError = true; } } - for (uint32_t i = 0; i < _summaryFeatures.size(); ++i) { - _summary_resolver->addSeed(_summaryFeatures[i]); + for (const auto & feature :_summaryFeatures) { + _summary_resolver->addSeed(feature); } if (!_ignoreDefaultRankFeatures) { VisitorAdapter adapter(*_dumpResolver); _factory.visitDumpFeatures(_indexEnv, adapter); } - for (uint32_t i = 0; i < _dumpFeatures.size(); ++i) { - _dumpResolver->addSeed(_dumpFeatures[i]); + for (const auto & feature : _dumpFeatures) { + _dumpResolver->addSeed(feature); } _indexEnv.hintFeatureMotivation(IIndexEnvironment::RANK); _compileError |= !_first_phase_resolver->compile(); diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.h b/searchlib/src/vespa/searchlib/fef/ranksetup.h index d543794b347..e1cd78d41a9 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.h +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.h @@ -397,10 +397,10 @@ public: // them to be ready to use. Also keep in mind that creating a rank // program is cheap while setting it up is more expensive. - RankProgram::UP create_first_phase_program() const { return RankProgram::UP(new RankProgram(_first_phase_resolver)); } - RankProgram::UP create_second_phase_program() const { return RankProgram::UP(new RankProgram(_second_phase_resolver)); } - RankProgram::UP create_summary_program() const { return RankProgram::UP(new RankProgram(_summary_resolver)); } - RankProgram::UP create_dump_program() const { return RankProgram::UP(new RankProgram(_dumpResolver)); } + RankProgram::UP create_first_phase_program() const { return std::make_unique<RankProgram>(_first_phase_resolver); } + RankProgram::UP create_second_phase_program() const { return std::make_unique<RankProgram>(_second_phase_resolver); } + RankProgram::UP create_summary_program() const { return std::make_unique<RankProgram>(_summary_resolver); } + RankProgram::UP create_dump_program() const { return std::make_unique<RankProgram>(_dumpResolver); } /** * Here you can do some preprocessing. State must be stored in the IObjectStore. diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp index 31e99ef9953..8be0961f999 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp @@ -2,11 +2,10 @@ #include "cfgvalue.h" #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <sstream> -namespace search { -namespace fef { -namespace test { +namespace search::fef::test { CfgValueBlueprint::CfgValueBlueprint() : Blueprint("test_cfgvalue"), @@ -14,9 +13,7 @@ CfgValueBlueprint::CfgValueBlueprint() : { } -CfgValueBlueprint::~CfgValueBlueprint() -{ -} +CfgValueBlueprint::~CfgValueBlueprint() = default; void CfgValueBlueprint::visitDumpFeatures(const IIndexEnvironment &indexEnv, IDumpFeatureVisitor &visitor) const @@ -59,6 +56,4 @@ CfgValueBlueprint::createExecutor(const IQueryEnvironment & queryEnv, vespalib:: return stash.create<search::features::ValueExecutor>(_values); } -} // namespace test -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp index 86754c2c22d..017b916ad76 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp @@ -1,11 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "chain.h" +#include <vespa/vespalib/util/stash.h> #include <sstream> -namespace search { -namespace fef { -namespace test { +namespace search::fef::test { ChainExecutor::ChainExecutor() : FeatureExecutor() @@ -67,6 +66,4 @@ ChainBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stas return stash.create<ChainExecutor>(); } -} // namespace test -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp index d9ec8b13e57..6f8ebd57fb0 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp @@ -3,6 +3,7 @@ #include "double.h" #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/stash.h> #include <cassert> namespace search::fef::test { diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp index 4b4b10c4d25..1e57d6252ee 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp @@ -4,6 +4,7 @@ #include <vespa/searchlib/features/valuefeature.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/locale/c.h> +#include <vespa/vespalib/util/stash.h> #include <sstream> namespace search::fef::test { @@ -32,7 +33,7 @@ QueryBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stas { std::vector<feature_t> values; std::string val = queryEnv.getProperties().lookup(_key).get("0.0"); - values.push_back(vespalib::locale::c::strtod(val.data(), NULL)); + values.push_back(vespalib::locale::c::strtod(val.data(), nullptr)); return stash.create<search::features::ValueExecutor>(values); } diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp index c1b8b940245..c5871b23e77 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp @@ -2,10 +2,9 @@ #include "staticrank.h" #include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/vespalib/util/stash.h> -namespace search { -namespace fef { -namespace test { +namespace search::fef::test { StaticRankExecutor::StaticRankExecutor(const search::attribute::IAttributeVector * attribute) : FeatureExecutor(), @@ -17,7 +16,7 @@ void StaticRankExecutor::execute(uint32_t docId) { search::attribute::FloatContent staticRank; - if (_attribute != NULL) { + if (_attribute != nullptr) { staticRank.allocate(_attribute->getMaxValueCount()); staticRank.fill(*_attribute, docId); } @@ -31,9 +30,7 @@ StaticRankBlueprint::StaticRankBlueprint() : { } -StaticRankBlueprint::~StaticRankBlueprint() -{ -} +StaticRankBlueprint::~StaticRankBlueprint() = default; bool StaticRankBlueprint::setup(const IIndexEnvironment & indexEnv, const StringVector & params) @@ -54,6 +51,4 @@ StaticRankBlueprint::createExecutor(const IQueryEnvironment & queryEnv, vespalib return stash.create<StaticRankExecutor>(av); } -} // namespace test -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp index b5025d53cbd..4362a7f0860 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp @@ -2,10 +2,9 @@ #include "sum.h" #include <vespa/searchlib/fef/featurenamebuilder.h> +#include <vespa/vespalib/util/stash.h> -namespace search { -namespace fef { -namespace test { +namespace search::fef::test { void SumExecutor::execute(uint32_t) @@ -73,6 +72,4 @@ SumBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stash return stash.create<SumExecutor>(); } -} // namespace test -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp index 7b3876fada0..e30b4893e15 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "unbox.h" +#include <vespa/vespalib/util/stash.h> namespace search::fef::test { diff --git a/searchlib/src/vespa/searchlib/fef/test/test_features.cpp b/searchlib/src/vespa/searchlib/fef/test/test_features.cpp index f924acd65de..c32776dc88d 100644 --- a/searchlib/src/vespa/searchlib/fef/test/test_features.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/test_features.cpp @@ -3,6 +3,8 @@ #include <vespa/vespalib/testkit/test_kit.h> #include "test_features.h" #include <vespa/vespalib/locale/c.h> +#include <vespa/vespalib/util/stash.h> + using vespalib::eval::DoubleValue; using vespalib::eval::ValueType; diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp index f9bce4bf7d1..d4aa2aaa1d7 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp @@ -21,7 +21,11 @@ NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& f _distance_heap(target_num_hits), _found_hits() { - setEstimate(HitEstimate(_attr_tensor.getNumDocs(), false)); + uint32_t est_hits = _attr_tensor.getNumDocs(); + if (_attr_tensor.nearest_neighbor_index()) { + est_hits = std::min(target_num_hits, est_hits); + } + setEstimate(HitEstimate(est_hits, false)); } NearestNeighborBlueprint::~NearestNeighborBlueprint() = default; @@ -56,7 +60,7 @@ NearestNeighborBlueprint::createLeafSearch(const search::fef::TermFieldMatchData assert(tfmda.size() == 1); fef::TermFieldMatchData &tfmd = *tfmda[0]; // always search in only one field if (strict && ! _found_hits.empty()) { - return NnsIndexIterator::create(strict, tfmd, _found_hits); + return NnsIndexIterator::create(tfmd, _found_hits); } const vespalib::tensor::DenseTensorView &qT = *_query_tensor; return NearestNeighborIterator::create(strict, tfmd, qT, _attr_tensor, _distance_heap); diff --git a/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp index 7ee985a0ba5..18e0213e092 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp @@ -4,7 +4,7 @@ #include <vespa/searchlib/tensor/nearest_neighbor_index.h> #include <cmath> -using Hit = search::tensor::NearestNeighborIndex::Neighbor; +using Neighbor = search::tensor::NearestNeighborIndex::Neighbor; namespace search::queryeval { @@ -17,12 +17,12 @@ class NeighborVectorIterator : public NnsIndexIterator { private: fef::TermFieldMatchData &_tfmd; - const std::vector<Hit> &_hits; + const std::vector<Neighbor> &_hits; uint32_t _idx; double _last_sq_dist; public: NeighborVectorIterator(fef::TermFieldMatchData &tfmd, - const std::vector<Hit> &hits) + const std::vector<Neighbor> &hits) : _tfmd(tfmd), _hits(hits), _idx(0), @@ -59,11 +59,9 @@ public: std::unique_ptr<NnsIndexIterator> NnsIndexIterator::create( - bool strict, fef::TermFieldMatchData &tfmd, - const std::vector<Hit> &hits) + const std::vector<Neighbor> &hits) { - assert(strict); return std::make_unique<NeighborVectorIterator>(tfmd, hits); } diff --git a/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h index 62fa49aac46..9ffd0df94eb 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h @@ -13,7 +13,6 @@ class NnsIndexIterator : public SearchIterator public: using Hit = search::tensor::NearestNeighborIndex::Neighbor; static std::unique_ptr<NnsIndexIterator> create( - bool strict, fef::TermFieldMatchData &tfmd, const std::vector<Hit> &hits); }; diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index 171340e07f1..229c6c2c34f 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -183,6 +183,26 @@ DenseTensorAttribute::getVersion() const return DENSE_TENSOR_ATTRIBUTE_VERSION; } +void +DenseTensorAttribute::onGenerationChange(generation_t next_gen) +{ + // TODO: Change onGenerationChange() to send current generation instead of next generation. + // This applies for entire attribute vector code. + TensorAttribute::onGenerationChange(next_gen); + if (_index) { + _index->transfer_hold_lists(next_gen - 1); + } +} + +void +DenseTensorAttribute::removeOldGenerations(generation_t first_used_gen) +{ + TensorAttribute::removeOldGenerations(first_used_gen); + if (_index) { + _index->trim_hold_lists(first_used_gen); + } +} + vespalib::tensor::TypedCells DenseTensorAttribute::get_vector(uint32_t docid) const { diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h index f9a8a81b56b..84a60376fe7 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h @@ -29,7 +29,7 @@ public: DenseTensorAttribute(vespalib::stringref baseFileName, const Config& cfg, const NearestNeighborIndexFactory& index_factory = DefaultNearestNeighborIndexFactory()); virtual ~DenseTensorAttribute(); - // Implements TensorAttribute + // Implements AttributeVector and ITensorAttribute uint32_t clearDoc(DocId docId) override; void setTensor(DocId docId, const Tensor &tensor) override; std::unique_ptr<Tensor> getTensor(DocId docId) const override; @@ -38,6 +38,8 @@ public: std::unique_ptr<AttributeSaver> onInitSave(vespalib::stringref fileName) override; void compactWorst() override; uint32_t getVersion() const override; + void onGenerationChange(generation_t next_gen) override; + void removeOldGenerations(generation_t first_used_gen) override; // Implements DocVectorAccess vespalib::tensor::TypedCells get_vector(uint32_t docid) const override; diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index 0d308206761..467e41d83fc 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -9,6 +9,8 @@ namespace search::tensor { +using search::datastore::EntryRef; + namespace { // TODO: Move this to MemoryAllocator, with name PAGE_SIZE. @@ -44,6 +46,9 @@ HnswIndex::max_links_for_level(uint32_t level) const uint32_t HnswIndex::make_node_for_document(uint32_t docid) { + // A document cannot be added twice. + assert(!_node_refs[docid].load_acquire().valid()); + uint32_t max_level = _level_generator->max_level(); // TODO: Add capping on num_levels uint32_t num_levels = max_level + 1; @@ -54,6 +59,15 @@ HnswIndex::make_node_for_document(uint32_t docid) return max_level; } +void +HnswIndex::remove_node_for_document(uint32_t docid) +{ + auto node_ref = _node_refs[docid].load_acquire(); + _nodes.remove(node_ref); + EntryRef invalid; + _node_refs[docid].store_release(invalid); +} + HnswIndex::LevelArrayRef HnswIndex::get_level_array(uint32_t docid) const { @@ -72,10 +86,12 @@ HnswIndex::get_link_array(uint32_t docid, uint32_t level) const void HnswIndex::set_link_array(uint32_t docid, uint32_t level, const LinkArrayRef& links) { - auto links_ref = _links.add(links); + auto new_links_ref = _links.add(links); auto node_ref = _node_refs[docid].load_acquire(); auto levels = _nodes.get_writable(node_ref); - levels[level].store_release(links_ref); + auto old_links_ref = levels[level].load_acquire(); + levels[level].store_release(new_links_ref); + _links.remove(old_links_ref); } bool @@ -136,7 +152,7 @@ HnswIndex::select_neighbors(const HnswCandidateVector& neighbors, uint32_t max_l } void -HnswIndex::connect_new_node(uint32_t docid, const LinkArray& neighbors, uint32_t level) +HnswIndex::connect_new_node(uint32_t docid, const LinkArrayRef &neighbors, uint32_t level) { set_link_array(docid, level, neighbors); for (uint32_t neighbor_docid : neighbors) { @@ -248,8 +264,6 @@ HnswIndex::add_document(uint32_t docid) { auto input = get_vector(docid); _node_refs.ensure_size(docid + 1, AtomicEntryRef()); - // A document cannot be added twice. - assert(!_node_refs[docid].load_acquire().valid()); int level = make_node_for_document(docid); if (_entry_docid == 0) { _entry_docid = docid; @@ -306,8 +320,35 @@ HnswIndex::remove_document(uint32_t docid) _entry_docid = 0; _entry_level = -1; } - search::datastore::EntryRef invalid; - _node_refs[docid].store_release(invalid); + remove_node_for_document(docid); +} + +void +HnswIndex::transfer_hold_lists(generation_t current_gen) +{ + // Note: RcuVector transfers hold lists as part of reallocation based on current generation. + // We need to set the next generation here, as it is incremented on a higher level right after this call. + _node_refs.setGeneration(current_gen + 1); + _nodes.transferHoldLists(current_gen); + _links.transferHoldLists(current_gen); +} + +void +HnswIndex::trim_hold_lists(generation_t first_used_gen) +{ + _node_refs.removeOldGenerations(first_used_gen); + _nodes.trimHoldLists(first_used_gen); + _links.trimHoldLists(first_used_gen); +} + +vespalib::MemoryUsage +HnswIndex::memory_usage() const +{ + vespalib::MemoryUsage result; + result.merge(_node_refs.getMemoryUsage()); + result.merge(_nodes.getMemoryUsage()); + result.merge(_links.getMemoryUsage()); + return result; } struct NeighborsByDocId { @@ -371,5 +412,28 @@ HnswIndex::get_node(uint32_t docid) const return HnswNode(result); } +void +HnswIndex::set_node(uint32_t docid, const HnswNode &node) +{ + _node_refs.ensure_size(docid + 1, AtomicEntryRef()); + // A document cannot be added twice. + assert(!_node_refs[docid].load_acquire().valid()); + + // make new node + size_t num_levels = node.size(); + assert(num_levels > 0); + LevelArray levels(num_levels, AtomicEntryRef()); + auto node_ref = _nodes.add(levels); + _node_refs[docid].store_release(node_ref); + + for (size_t level = 0; level < num_levels; ++level) { + connect_new_node(docid, node.level(level), level); + } + int max_level = num_levels - 1; + if (_entry_level < max_level) { + _entry_docid = docid; + _entry_level = max_level; + } } +} diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h index 800b88923b5..89c45d6b50c 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -95,6 +95,7 @@ protected: uint32_t max_links_for_level(uint32_t level) const; uint32_t make_node_for_document(uint32_t docid); + void remove_node_for_document(uint32_t docid); LevelArrayRef get_level_array(uint32_t docid) const; LinkArrayRef get_link_array(uint32_t docid, uint32_t level) const; void set_link_array(uint32_t docid, uint32_t level, const LinkArrayRef& links); @@ -110,7 +111,7 @@ protected: LinkArray select_neighbors_heuristic(const HnswCandidateVector& neighbors, uint32_t max_links) const; LinkArray select_neighbors_simple(const HnswCandidateVector& neighbors, uint32_t max_links) const; LinkArray select_neighbors(const HnswCandidateVector& neighbors, uint32_t max_links) const; - void connect_new_node(uint32_t docid, const LinkArray& neighbors, uint32_t level); + void connect_new_node(uint32_t docid, const LinkArrayRef &neighbors, uint32_t level); void remove_link_to(uint32_t remove_from, uint32_t remove_id, uint32_t level); inline TypedCells get_vector(uint32_t docid) const { @@ -133,20 +134,22 @@ public: const Config& config() const { return _cfg; } + // Implements NearestNeighborIndex void add_document(uint32_t docid) override; void remove_document(uint32_t docid) override; + void transfer_hold_lists(generation_t current_gen) override; + void trim_hold_lists(generation_t first_used_gen) override; + vespalib::MemoryUsage memory_usage() const override; std::vector<Neighbor> find_top_k(uint32_t k, TypedCells vector, uint32_t explore_k) const override; - FurthestPriQ top_k_candidates(const TypedCells &vector, uint32_t k) const; - // TODO: Add support for generation handling and cleanup (transfer_hold_lists, trim_hold_lists) + FurthestPriQ top_k_candidates(const TypedCells &vector, uint32_t k) const; uint32_t get_entry_docid() const { return _entry_docid; } uint32_t get_entry_level() const { return _entry_level; } // Should only be used by unit tests. HnswNode get_node(uint32_t docid) const; - - // TODO: Implement set_node() as well for use in unit tests. + void set_node(uint32_t docid, const HnswNode &node); }; } diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h index f933af0147e..bd98623bdd3 100644 --- a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h +++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h @@ -5,6 +5,8 @@ #include <cstdint> #include <vector> #include <vespa/eval/tensor/dense/typed_cells.h> +#include <vespa/vespalib/util/generationhandler.h> +#include <vespa/vespalib/util/memoryusage.h> namespace search::tensor { @@ -13,6 +15,7 @@ namespace search::tensor { */ class NearestNeighborIndex { public: + using generation_t = vespalib::GenerationHandler::generation_t; struct Neighbor { uint32_t docid; double distance; @@ -24,9 +27,14 @@ public: virtual ~NearestNeighborIndex() {} virtual void add_document(uint32_t docid) = 0; virtual void remove_document(uint32_t docid) = 0; + virtual void transfer_hold_lists(generation_t current_gen) = 0; + virtual void trim_hold_lists(generation_t first_used_gen) = 0; + virtual vespalib::MemoryUsage memory_usage() const = 0; + virtual std::vector<Neighbor> find_top_k(uint32_t k, vespalib::tensor::TypedCells vector, uint32_t explore_k) const = 0; + }; } diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp index 89b8e77e136..0b9628e6872 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp @@ -71,7 +71,6 @@ TensorAttribute::TensorAttribute(vespalib::stringref name, const Config &cfg, Te { } - TensorAttribute::~TensorAttribute() = default; const ITensorAttribute * @@ -93,7 +92,6 @@ TensorAttribute::clearDoc(DocId docId) return 0u; } - void TensorAttribute::onCommit() { @@ -110,7 +108,6 @@ TensorAttribute::onCommit() } } - void TensorAttribute::onUpdateStat() { @@ -126,7 +123,6 @@ TensorAttribute::onUpdateStat() total.allocatedBytesOnHold()); } - void TensorAttribute::removeOldGenerations(generation_t firstUsed) { @@ -141,7 +137,6 @@ TensorAttribute::onGenerationChange(generation_t generation) _tensorStore.transferHoldLists(generation - 1); } - bool TensorAttribute::addDoc(DocId &docId) { @@ -209,7 +204,6 @@ TensorAttribute::clearDocs(DocId lidLow, DocId lidLimit) } } - void TensorAttribute::onShrinkLidSpace() { @@ -220,14 +214,12 @@ TensorAttribute::onShrinkLidSpace() setNumDocs(committedDocIdLimit); } - uint32_t TensorAttribute::getVersion() const { return TENSOR_ATTRIBUTE_VERSION; } - TensorAttribute::RefCopyVector TensorAttribute::getRefCopy() const { diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelCache.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelCache.java index c50f5e6c2d5..1b37555a554 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelCache.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelCache.java @@ -17,6 +17,7 @@ public class ServiceModelCache implements Supplier<ServiceModel> { private final Supplier<ServiceModel> expensiveSupplier; private final Timer timer; + private final boolean useCache; private volatile ServiceModel snapshot; private boolean updatePossiblyInProgress = false; @@ -24,13 +25,18 @@ public class ServiceModelCache implements Supplier<ServiceModel> { private final Object updateMonitor = new Object(); private long snapshotMillis; - public ServiceModelCache(Supplier<ServiceModel> expensiveSupplier, Timer timer) { + public ServiceModelCache(Supplier<ServiceModel> expensiveSupplier, Timer timer, boolean useCache) { this.expensiveSupplier = expensiveSupplier; this.timer = timer; + this.useCache = useCache; } @Override public ServiceModel get() { + if (!useCache) { + return expensiveSupplier.get(); + } + if (snapshot == null) { synchronized (updateMonitor) { if (snapshot == null) { diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java index 0a40555036c..67b4e890c29 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java @@ -5,16 +5,12 @@ import com.google.inject.Inject; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Timer; -import com.yahoo.vespa.applicationmodel.ApplicationInstance; -import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.service.duper.DuperModelManager; -import com.yahoo.vespa.service.health.HealthMonitorManager; import com.yahoo.vespa.service.manager.UnionMonitorManager; import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ServiceMonitor; -import com.yahoo.vespa.service.slobrok.SlobrokMonitorManagerImpl; - -import java.util.Map; public class ServiceMonitorImpl implements ServiceMonitor { @@ -25,7 +21,8 @@ public class ServiceMonitorImpl implements ServiceMonitor { UnionMonitorManager monitorManager, Metric metric, Timer timer, - Zone zone) { + Zone zone, + FlagSource flagSource) { duperModelManager.registerListener(monitorManager); ServiceModelProvider uncachedServiceModelProvider = new ServiceModelProvider( @@ -34,7 +31,8 @@ public class ServiceMonitorImpl implements ServiceMonitor { duperModelManager, new ModelGenerator(), zone); - serviceModelProvider = new ServiceModelCache(uncachedServiceModelProvider, timer); + boolean cache = Flags.SERVICE_MODEL_CACHE.bindTo(flagSource).value(); + serviceModelProvider = new ServiceModelCache(uncachedServiceModelProvider, timer, cache); } @Override diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelCacheTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelCacheTest.java index 2d6921df374..c2314be1e0f 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelCacheTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelCacheTest.java @@ -18,7 +18,7 @@ public class ServiceModelCacheTest { @SuppressWarnings("unchecked") private final Supplier<ServiceModel> rawSupplier = mock(Supplier.class); private final Timer timer = mock(Timer.class); - private final ServiceModelCache cache = new ServiceModelCache(rawSupplier, timer); + private final ServiceModelCache cache = new ServiceModelCache(rawSupplier, timer, true); @Test public void sanityCheck() { diff --git a/storage/src/tests/distributor/garbagecollectiontest.cpp b/storage/src/tests/distributor/garbagecollectiontest.cpp index 65c1ac726b5..776cfc14d84 100644 --- a/storage/src/tests/distributor/garbagecollectiontest.cpp +++ b/storage/src/tests/distributor/garbagecollectiontest.cpp @@ -3,6 +3,7 @@ #include <vespa/storageapi/message/removelocation.h> #include <vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h> #include <vespa/storage/distributor/idealstatemanager.h> +#include <vespa/storage/distributor/idealstatemetricsset.h> #include <tests/distributor/distributortestutil.h> #include <vespa/storage/distributor/distributor.h> #include <vespa/document/test/make_document_bucket.h> @@ -35,11 +36,13 @@ struct GarbageCollectionOperationTest : Test, DistributorTestUtil { } // FIXME fragile to assume that send order == node index, but that's the way it currently works - void reply_to_nth_request(GarbageCollectionOperation& op, size_t n, uint32_t bucket_info_checksum) { + void reply_to_nth_request(GarbageCollectionOperation& op, size_t n, + uint32_t bucket_info_checksum, uint32_t n_docs_removed) { auto msg = _sender.command(n); assert(msg->getType() == api::MessageType::REMOVELOCATION); std::shared_ptr<api::StorageReply> reply(msg->makeReply()); auto& gc_reply = dynamic_cast<api::RemoveLocationReply&>(*reply); + gc_reply.set_documents_removed(n_docs_removed); gc_reply.setBucketInfo(api::BucketInfo(bucket_info_checksum, 90, 500)); op.receive(_sender, reply); @@ -56,6 +59,13 @@ struct GarbageCollectionOperationTest : Test, DistributorTestUtil { << entry->getNode(i)->getBucketInfo(); } } + + uint32_t gc_removed_documents_metric() { + auto metric_base = getIdealStateManager().getMetrics().operations[IdealStateOperation::GARBAGE_COLLECTION]; + auto gc_metrics = std::dynamic_pointer_cast<GcMetricSet>(metric_base); + assert(gc_metrics); + return gc_metrics->documents_removed.getValue(); + } }; TEST_F(GarbageCollectionOperationTest, simple) { @@ -63,29 +73,34 @@ TEST_F(GarbageCollectionOperationTest, simple) { op->start(_sender, framework::MilliSecTime(0)); ASSERT_EQ(2, _sender.commands().size()); + EXPECT_EQ(0u, gc_removed_documents_metric()); for (uint32_t i = 0; i < 2; ++i) { std::shared_ptr<api::StorageCommand> msg = _sender.command(i); ASSERT_EQ(msg->getType(), api::MessageType::REMOVELOCATION); auto& tmp = dynamic_cast<api::RemoveLocationCommand&>(*msg); EXPECT_EQ("music.date < 34", tmp.getDocumentSelection()); - reply_to_nth_request(*op, i, 777 + i); + reply_to_nth_request(*op, i, 777 + i, 50); } ASSERT_NO_FATAL_FAILURE(assert_bucket_db_contains({api::BucketInfo(777, 90, 500), api::BucketInfo(778, 90, 500)}, 34)); + EXPECT_EQ(50u, gc_removed_documents_metric()); } TEST_F(GarbageCollectionOperationTest, replica_bucket_info_not_added_to_db_until_all_replies_received) { auto op = create_op(); op->start(_sender, framework::MilliSecTime(0)); ASSERT_EQ(2, _sender.commands().size()); + EXPECT_EQ(0u, gc_removed_documents_metric()); // Respond to 1st request. Should _not_ cause bucket info to be merged into the database yet - reply_to_nth_request(*op, 0, 1234); + reply_to_nth_request(*op, 0, 1234, 70); ASSERT_NO_FATAL_FAILURE(assert_bucket_db_contains({api::BucketInfo(250, 50, 300), api::BucketInfo(250, 50, 300)}, 0)); // Respond to 2nd request. This _should_ cause bucket info to be merged into the database. - reply_to_nth_request(*op, 1, 4567); + reply_to_nth_request(*op, 1, 4567, 60); ASSERT_NO_FATAL_FAILURE(assert_bucket_db_contains({api::BucketInfo(1234, 90, 500), api::BucketInfo(4567, 90, 500)}, 34)); + + EXPECT_EQ(70u, gc_removed_documents_metric()); // Use max of received metrics } TEST_F(GarbageCollectionOperationTest, gc_bucket_info_does_not_overwrite_later_sequenced_bucket_info_writes) { @@ -93,10 +108,10 @@ TEST_F(GarbageCollectionOperationTest, gc_bucket_info_does_not_overwrite_later_s op->start(_sender, framework::MilliSecTime(0)); ASSERT_EQ(2, _sender.commands().size()); - reply_to_nth_request(*op, 0, 1234); + reply_to_nth_request(*op, 0, 1234, 0); // Change to replica on node 0 happens after GC op, but before GC info is merged into the DB. Must not be lost. insertBucketInfo(op->getBucketId(), 0, 7777, 100, 2000); - reply_to_nth_request(*op, 1, 4567); + reply_to_nth_request(*op, 1, 4567, 0); // Bucket info for node 0 is that of the later sequenced operation, _not_ from the earlier GC op. ASSERT_NO_FATAL_FAILURE(assert_bucket_db_contains({api::BucketInfo(7777, 100, 2000), api::BucketInfo(4567, 90, 500)}, 34)); } diff --git a/storage/src/tests/persistence/processalltest.cpp b/storage/src/tests/persistence/processalltest.cpp index 8c0f8853d2d..83f243ed1b2 100644 --- a/storage/src/tests/persistence/processalltest.cpp +++ b/storage/src/tests/persistence/processalltest.cpp @@ -23,11 +23,15 @@ TEST_F(ProcessAllHandlerTest, remove_location) { api::RemoveLocationCommand removeLocation("id.user == 4", makeDocumentBucket(bucketId)); ProcessAllHandler handler(getEnv(), getPersistenceProvider()); spi::Context context(documentapi::LoadType::DEFAULT, 0, 0); - handler.handleRemoveLocation(removeLocation, context); + auto tracker = handler.handleRemoveLocation(removeLocation, context); EXPECT_EQ("DocEntry(1234, 1, id:mail:testdoctype1:n=4:3619.html)\n" "DocEntry(2345, 1, id:mail:testdoctype1:n=4:4008.html)\n", dumpBucket(bucketId)); + + auto reply = std::dynamic_pointer_cast<api::RemoveLocationReply>(tracker->getReply()); + ASSERT_TRUE(reply.get() != nullptr); + EXPECT_EQ(2u, reply->documents_removed()); } TEST_F(ProcessAllHandlerTest, remove_location_document_subset) { @@ -44,7 +48,7 @@ TEST_F(ProcessAllHandlerTest, remove_location_document_subset) { api::RemoveLocationCommand removeLocation("testdoctype1.headerval % 2 == 0", makeDocumentBucket(bucketId)); spi::Context context(documentapi::LoadType::DEFAULT, 0, 0); - handler.handleRemoveLocation(removeLocation, context); + auto tracker = handler.handleRemoveLocation(removeLocation, context); EXPECT_EQ("DocEntry(100, 1, id:mail:testdoctype1:n=4:3619.html)\n" "DocEntry(101, 0, Doc(id:mail:testdoctype1:n=4:33113.html))\n" @@ -57,6 +61,10 @@ TEST_F(ProcessAllHandlerTest, remove_location_document_subset) { "DocEntry(108, 1, id:mail:testdoctype1:n=4:42967.html)\n" "DocEntry(109, 0, Doc(id:mail:testdoctype1:n=4:6925.html))\n", dumpBucket(bucketId)); + + auto reply = std::dynamic_pointer_cast<api::RemoveLocationReply>(tracker->getReply()); + ASSERT_TRUE(reply.get() != nullptr); + EXPECT_EQ(5u, reply->documents_removed()); } TEST_F(ProcessAllHandlerTest, remove_location_throws_exception_on_unknown_doc_type) { diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp index 61e67b40f44..c211e775326 100644 --- a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp +++ b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp @@ -4,8 +4,7 @@ #include <vespa/storage/distributor/distributormetricsset.h> #include <vespa/storage/distributor/idealstatemetricsset.h> -namespace storage { -namespace distributor { +namespace storage::distributor { BucketDBMetricUpdater::Stats::Stats() : _docCount(0), @@ -27,9 +26,7 @@ BucketDBMetricUpdater::BucketDBMetricUpdater() { } -BucketDBMetricUpdater::~BucketDBMetricUpdater() -{ -} +BucketDBMetricUpdater::~BucketDBMetricUpdater() = default; void BucketDBMetricUpdater::resetStats() @@ -148,5 +145,4 @@ BucketDBMetricUpdater::reset() resetStats(); } -} // distributor -} // storage +} // storage::distributor diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h index 6e15ee03d12..7ef8479866f 100644 --- a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h +++ b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h @@ -7,12 +7,9 @@ #include <unordered_map> -namespace storage { +namespace storage::distributor { class DistributorMetricSet; - -namespace distributor { - class IdealStateMetricSet; class BucketDBMetricUpdater { @@ -107,5 +104,4 @@ private: void resetStats(); }; -} // distributor -} // storage +} // storage::distributor diff --git a/storage/src/vespa/storage/distributor/distributorinterface.h b/storage/src/vespa/storage/distributor/distributorinterface.h index aba58e112dc..b17bcd56d19 100644 --- a/storage/src/vespa/storage/distributor/distributorinterface.h +++ b/storage/src/vespa/storage/distributor/distributorinterface.h @@ -12,10 +12,10 @@ namespace storage::api { class MergeBucketReply; } namespace storage::lib { class ClusterStateBundle; } namespace storage { class DistributorConfiguration; - class DistributorMetricSet; } namespace storage::distributor { +class DistributorMetricSet; class PendingMessageTracker; class DistributorInterface : public DistributorMessageSender diff --git a/storage/src/vespa/storage/distributor/distributormetricsset.cpp b/storage/src/vespa/storage/distributor/distributormetricsset.cpp index 244406ca6fb..98e96f9294f 100644 --- a/storage/src/vespa/storage/distributor/distributormetricsset.cpp +++ b/storage/src/vespa/storage/distributor/distributormetricsset.cpp @@ -3,7 +3,7 @@ #include <vespa/metrics/loadmetric.hpp> #include <vespa/metrics/summetric.hpp> -namespace storage { +namespace storage::distributor { using metrics::MetricSet; diff --git a/storage/src/vespa/storage/distributor/distributormetricsset.h b/storage/src/vespa/storage/distributor/distributormetricsset.h index 1e4730b8de6..b5be72e8c14 100644 --- a/storage/src/vespa/storage/distributor/distributormetricsset.h +++ b/storage/src/vespa/storage/distributor/distributormetricsset.h @@ -7,7 +7,7 @@ #include <vespa/metrics/metrics.h> #include <vespa/documentapi/loadtypes/loadtypeset.h> -namespace storage { +namespace storage::distributor { class DistributorMetricSet : public metrics::MetricSet { diff --git a/storage/src/vespa/storage/distributor/externaloperationhandler.h b/storage/src/vespa/storage/distributor/externaloperationhandler.h index 96875a3644a..60cad15a791 100644 --- a/storage/src/vespa/storage/distributor/externaloperationhandler.h +++ b/storage/src/vespa/storage/distributor/externaloperationhandler.h @@ -13,11 +13,11 @@ namespace storage { -class DistributorMetricSet; class PersistenceOperationMetricSet; namespace distributor { +class DistributorMetricSet; class Distributor; class MaintenanceOperationGenerator; class DirectDispatchSender; diff --git a/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp b/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp index d72f4a80ef4..fd193ad6fd8 100644 --- a/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp +++ b/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp @@ -6,7 +6,7 @@ namespace storage { namespace distributor { OperationMetricSet::OperationMetricSet(const std::string& name, metrics::Metric::Tags tags, const std::string& description, MetricSet* owner) - : MetricSet(name, tags, description, owner), + : MetricSet(name, std::move(tags), description, owner), pending("pending", {{"logdefault"},{"yamasdefault"}}, "The number of operations pending", this), @@ -16,14 +16,25 @@ OperationMetricSet::OperationMetricSet(const std::string& name, metrics::Metric: failed("done_failed", {{"logdefault"},{"yamasdefault"}}, "The number of operations that failed", this) -{ } +{} -OperationMetricSet::~OperationMetricSet() { } +OperationMetricSet::~OperationMetricSet() = default; + +GcMetricSet::GcMetricSet(const std::string& name, metrics::Metric::Tags tags, const std::string& description, MetricSet* owner) + : OperationMetricSet(name, std::move(tags), description, owner), + documents_removed("documents_removed", + {{"logdefault"},{"yamasdefault"}}, + "Number of documents removed by GC operations", this) +{} + +GcMetricSet::~GcMetricSet() = default; void IdealStateMetricSet::createOperationMetrics() { typedef IdealStateOperation ISO; operations.resize(ISO::OPERATION_COUNT); + // Note: naked new is used instead of make_shared due to the latter not being + // able to properly transitively deduce the types for the tag initializer lists. operations[ISO::DELETE_BUCKET] = std::shared_ptr<OperationMetricSet>( new OperationMetricSet("delete_bucket", {{"logdefault"},{"yamasdefault"}}, @@ -45,9 +56,9 @@ IdealStateMetricSet::createOperationMetrics() { {{"logdefault"},{"yamasdefault"}}, "Operations to set active/ready state for bucket copies", this)); operations[ISO::GARBAGE_COLLECTION] = std::shared_ptr<OperationMetricSet>( - new OperationMetricSet("garbage_collection", - {{"logdefault"},{"yamasdefault"}}, - "Operations to garbage collect data from buckets", this)); + new GcMetricSet("garbage_collection", + {{"logdefault"},{"yamasdefault"}}, + "Operations to garbage collect data from buckets", this)); } IdealStateMetricSet::IdealStateMetricSet() @@ -81,7 +92,7 @@ IdealStateMetricSet::IdealStateMetricSet() createOperationMetrics(); } -IdealStateMetricSet::~IdealStateMetricSet() { } +IdealStateMetricSet::~IdealStateMetricSet() = default; void IdealStateMetricSet::setPendingOperations(const std::vector<uint64_t>& newMetrics) { for (uint32_t i = 0; i < IdealStateOperation::OPERATION_COUNT; i++) { diff --git a/storage/src/vespa/storage/distributor/idealstatemetricsset.h b/storage/src/vespa/storage/distributor/idealstatemetricsset.h index 7bb472b4a2c..2679da17598 100644 --- a/storage/src/vespa/storage/distributor/idealstatemetricsset.h +++ b/storage/src/vespa/storage/distributor/idealstatemetricsset.h @@ -16,13 +16,21 @@ public: metrics::LongCountMetric failed; OperationMetricSet(const std::string& name, metrics::Metric::Tags tags, const std::string& description, MetricSet* owner); - ~OperationMetricSet(); + ~OperationMetricSet() override; +}; + +struct GcMetricSet : OperationMetricSet { + metrics::LongCountMetric documents_removed; + + GcMetricSet(const std::string& name, metrics::Metric::Tags tags, + const std::string& description, MetricSet* owner); + ~GcMetricSet() override; }; class IdealStateMetricSet : public metrics::MetricSet { public: - std::vector<std::shared_ptr<OperationMetricSet> > operations; + std::vector<std::shared_ptr<OperationMetricSet>> operations; metrics::LongValueMetric idealstate_diff; metrics::LongValueMetric buckets_toofewcopies; metrics::LongValueMetric buckets_toomanycopies; @@ -35,7 +43,7 @@ public: void createOperationMetrics(); IdealStateMetricSet(); - ~IdealStateMetricSet(); + ~IdealStateMetricSet() override; void setPendingOperations(const std::vector<uint64_t>& newMetrics); }; diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp index c674add80f7..fc127c2e0eb 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp @@ -2,6 +2,7 @@ #include "garbagecollectionoperation.h" #include <vespa/storage/distributor/idealstatemanager.h> +#include <vespa/storage/distributor/idealstatemetricsset.h> #include <vespa/storage/distributor/distributor.h> #include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/storageapi/message/removelocation.h> @@ -9,19 +10,18 @@ #include <vespa/log/log.h> LOG_SETUP(".distributor.operation.idealstate.remove"); -using namespace storage::distributor; +namespace storage::distributor { GarbageCollectionOperation::GarbageCollectionOperation(const std::string& clusterName, const BucketAndNodes& nodes) : IdealStateOperation(nodes), _tracker(clusterName), - _replica_info() + _replica_info(), + _max_documents_removed(0) {} GarbageCollectionOperation::~GarbageCollectionOperation() = default; -void -GarbageCollectionOperation::onStart(DistributorMessageSender& sender) -{ +void GarbageCollectionOperation::onStart(DistributorMessageSender& sender) { BucketDatabase::Entry entry = _bucketSpace->getBucketDatabase().get(getBucketId()); std::vector<uint16_t> nodes = entry->getNodes(); @@ -43,7 +43,7 @@ GarbageCollectionOperation::onStart(DistributorMessageSender& sender) void GarbageCollectionOperation::onReceive(DistributorMessageSender&, - const std::shared_ptr<api::StorageReply>& reply) + const std::shared_ptr<api::StorageReply>& reply) { auto* rep = dynamic_cast<api::RemoveLocationReply*>(reply.get()); assert(rep != nullptr); @@ -53,6 +53,7 @@ GarbageCollectionOperation::onReceive(DistributorMessageSender&, if (!rep->getResult().failed()) { _replica_info.emplace_back(_manager->getDistributorComponent().getUniqueTimestamp(), node, rep->getBucketInfo()); + _max_documents_removed = std::max(_max_documents_removed, rep->documents_removed()); } else { _ok = false; } @@ -61,6 +62,7 @@ GarbageCollectionOperation::onReceive(DistributorMessageSender&, if (_ok) { merge_received_bucket_info_into_db(); } + update_gc_metrics(); done(); } } @@ -76,8 +78,16 @@ void GarbageCollectionOperation::merge_received_bucket_info_into_db() { } } +void GarbageCollectionOperation::update_gc_metrics() { + auto metric_base = _manager->getMetrics().operations[IdealStateOperation::GARBAGE_COLLECTION]; + auto gc_metrics = std::dynamic_pointer_cast<GcMetricSet>(metric_base); + assert(gc_metrics); + gc_metrics->documents_removed.inc(_max_documents_removed); +} + bool -GarbageCollectionOperation::shouldBlockThisOperation(uint32_t, uint8_t) const -{ +GarbageCollectionOperation::shouldBlockThisOperation(uint32_t, uint8_t) const { return true; } + +} diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h index 47ea11bb328..28de9592a63 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h @@ -26,8 +26,10 @@ protected: MessageTracker _tracker; private: std::vector<BucketCopy> _replica_info; + uint32_t _max_documents_removed; void merge_received_bucket_info_into_db(); + void update_gc_metrics(); }; } diff --git a/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h b/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h index d951d7ceba2..1299fdad2ad 100644 --- a/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h +++ b/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h @@ -16,7 +16,7 @@ class PersistenceFailuresMetricSet : public metrics::MetricSet { public: explicit PersistenceFailuresMetricSet(metrics::MetricSet* owner); - ~PersistenceFailuresMetricSet(); + ~PersistenceFailuresMetricSet() override; metrics::SumMetric<metrics::LongCountMetric> sum; metrics::LongCountMetric notready; @@ -44,7 +44,7 @@ public: PersistenceFailuresMetricSet failures; PersistenceOperationMetricSet(const std::string& name, metrics::MetricSet* owner = nullptr); - ~PersistenceOperationMetricSet(); + ~PersistenceOperationMetricSet() override; MetricSet * clone(std::vector<Metric::UP>& ownerList, CopyType copyType, metrics::MetricSet* owner, bool includeUnused) const override; diff --git a/storage/src/vespa/storage/persistence/processallhandler.cpp b/storage/src/vespa/storage/persistence/processallhandler.cpp index 8c951a9f50d..5b94a3da027 100644 --- a/storage/src/vespa/storage/persistence/processallhandler.cpp +++ b/storage/src/vespa/storage/persistence/processallhandler.cpp @@ -23,6 +23,7 @@ public: spi::PersistenceProvider& _provider; const spi::Bucket& _bucket; spi::Context& _context; + uint32_t _n_removed; UnrevertableRemoveEntryProcessor( spi::PersistenceProvider& provider, @@ -30,7 +31,9 @@ public: spi::Context& context) : _provider(provider), _bucket(bucket), - _context(context) {} + _context(context), + _n_removed(0) + {} void process(spi::DocEntry& entry) override { spi::RemoveResult removeResult = _provider.remove( @@ -45,13 +48,14 @@ public: << removeResult.getErrorMessage(); throw std::runtime_error(ss.str()); } + ++_n_removed; } }; class StatEntryProcessor : public BucketProcessor::EntryProcessor { public: std::ostream& ost; - StatEntryProcessor(std::ostream& o) + explicit StatEntryProcessor(std::ostream& o) : ost(o) {}; void process(spi::DocEntry& e) override { @@ -97,7 +101,9 @@ ProcessAllHandler::handleRemoveLocation(api::RemoveLocationCommand& cmd, context); spi::Result result = _spi.flush(bucket, context); uint32_t code = _env.convertErrorCode(result); - if (code != 0) { + if (code == 0) { + tracker->setReply(std::make_shared<api::RemoveLocationReply>(cmd, processor._n_removed)); + } else { tracker->fail(code, result.getErrorMessage()); } diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp index 2f959e40e2a..2e5eb115844 100644 --- a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp +++ b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp @@ -522,8 +522,15 @@ TEST_P(StorageProtocolTest, remove_location) { EXPECT_EQ("id.group == \"mygroup\"", cmd2->getDocumentSelection()); EXPECT_EQ(_bucket, cmd2->getBucket()); - auto reply = std::make_shared<RemoveLocationReply>(*cmd2); + uint32_t n_docs_removed = 12345; + auto reply = std::make_shared<RemoveLocationReply>(*cmd2, n_docs_removed); auto reply2 = copyReply(reply); + if (GetParam().getMajor() == 7) { + // Statistics are only available for protobuf-enabled version. + EXPECT_EQ(n_docs_removed, reply2->documents_removed()); + } else { + EXPECT_EQ(0, reply2->documents_removed()); + } } TEST_P(StorageProtocolTest, create_visitor) { diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto b/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto index 810f88f588f..12dbaf59146 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto +++ b/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto @@ -90,7 +90,12 @@ message RemoveLocationRequest { bytes document_selection = 2; } +message RemoveLocationStats { + uint32 documents_removed = 1; +} + message RemoveLocationResponse { BucketInfo bucket_info = 1; BucketId remapped_bucket_id = 2; + RemoveLocationStats stats = 3; } diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp index 9751fd1be98..90c8d1c7d2a 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp @@ -643,7 +643,9 @@ void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveLocationComma } void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveLocationReply& msg) const { - encode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, msg, no_op_encode); + encode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, msg, [&](auto& res) { + res.mutable_stats()->set_documents_removed(msg.documents_removed()); + }); } api::StorageCommand::UP ProtocolSerialization7::onDecodeRemoveLocationCommand(BBuf& buf) const { @@ -653,8 +655,11 @@ api::StorageCommand::UP ProtocolSerialization7::onDecodeRemoveLocationCommand(BB } api::StorageReply::UP ProtocolSerialization7::onDecodeRemoveLocationReply(const SCmd& cmd, BBuf& buf) const { - return decode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, [&]([[maybe_unused]] auto& res) { - return std::make_unique<api::RemoveLocationReply>(static_cast<const api::RemoveLocationCommand&>(cmd)); + return decode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, [&](auto& res) { + uint32_t documents_removed = (res.has_stats() ? res.stats().documents_removed() : 0u); + return std::make_unique<api::RemoveLocationReply>( + static_cast<const api::RemoveLocationCommand&>(cmd), + documents_removed); }); } diff --git a/storageapi/src/vespa/storageapi/message/removelocation.cpp b/storageapi/src/vespa/storageapi/message/removelocation.cpp index b53584601ef..49c9d22f5ee 100644 --- a/storageapi/src/vespa/storageapi/message/removelocation.cpp +++ b/storageapi/src/vespa/storageapi/message/removelocation.cpp @@ -25,8 +25,9 @@ RemoveLocationCommand::print(std::ostream& out, bool verbose, const std::string& BucketInfoCommand::print(out, verbose, indent); } -RemoveLocationReply::RemoveLocationReply(const RemoveLocationCommand& cmd) - : BucketInfoReply(cmd) +RemoveLocationReply::RemoveLocationReply(const RemoveLocationCommand& cmd, uint32_t docs_removed) + : BucketInfoReply(cmd), + _documents_removed(docs_removed) { } diff --git a/storageapi/src/vespa/storageapi/message/removelocation.h b/storageapi/src/vespa/storageapi/message/removelocation.h index 46555497035..812cc8c413b 100644 --- a/storageapi/src/vespa/storageapi/message/removelocation.h +++ b/storageapi/src/vespa/storageapi/message/removelocation.h @@ -11,7 +11,7 @@ class RemoveLocationCommand : public BucketInfoCommand { public: RemoveLocationCommand(vespalib::stringref documentSelection, const document::Bucket &bucket); - ~RemoveLocationCommand(); + ~RemoveLocationCommand() override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; const vespalib::string& getDocumentSelection() const { return _documentSelection; } @@ -22,8 +22,13 @@ private: class RemoveLocationReply : public BucketInfoReply { + uint32_t _documents_removed; public: - RemoveLocationReply(const RemoveLocationCommand& cmd); + explicit RemoveLocationReply(const RemoveLocationCommand& cmd, uint32_t docs_removed = 0); + void set_documents_removed(uint32_t docs_removed) noexcept { + _documents_removed = docs_removed; + } + uint32_t documents_removed() const noexcept { return _documents_removed; } DECLARE_STORAGEREPLY(RemoveLocationReply, onRemoveLocationReply) }; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java index 4cc92828b0e..97a04e21d4b 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java @@ -80,6 +80,7 @@ public abstract class ClientBase implements AutoCloseable { .setRetryHandler(new DefaultHttpRequestRetryHandler(3, /*requestSentRetryEnabled*/true)) .setUserAgent(userAgent) .setSSLSocketFactory(new SSLConnectionSocketFactory(new ServiceIdentitySslSocketFactory(sslContextSupplier), hostnameVerifier)) + .setMaxConnPerRoute(8) .setDefaultRequestConfig(RequestConfig.custom() .setConnectTimeout((int) Duration.ofSeconds(10).toMillis()) .setConnectionRequestTimeout((int)Duration.ofSeconds(10).toMillis()) diff --git a/vespabase/src/rhel-prestart.sh b/vespabase/src/rhel-prestart.sh index 3a770cdd785..d84122898a3 100755 --- a/vespabase/src/rhel-prestart.sh +++ b/vespabase/src/rhel-prestart.sh @@ -96,32 +96,32 @@ fixdir () { # BEGIN directory fixups -fixdir root wheel 1777 logs -fixdir root wheel 1777 tmp -fixdir root wheel 1777 var/run -fixdir ${VESPA_USER} wheel 1777 var/crash -fixdir ${VESPA_USER} wheel 1777 logs/vespa -fixdir ${VESPA_USER} wheel 1777 tmp/vespa -fixdir ${VESPA_USER} wheel 755 var -fixdir ${VESPA_USER} wheel 755 libexec/vespa/plugins/qrs -fixdir ${VESPA_USER} wheel 755 logs/vespa/configserver -fixdir ${VESPA_USER} wheel 755 logs/vespa/qrs -fixdir ${VESPA_USER} wheel 755 logs/vespa/search -fixdir ${VESPA_USER} wheel 755 var/db/vespa -fixdir ${VESPA_USER} wheel 755 var/db/vespa/tmp -fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server -fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server/serverdb -fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server/serverdb/tenants -fixdir ${VESPA_USER} wheel 755 var/db/vespa/filedistribution -fixdir ${VESPA_USER} wheel 755 var/db/vespa/index -fixdir ${VESPA_USER} wheel 755 var/db/vespa/logcontrol -fixdir ${VESPA_USER} wheel 755 var/db/vespa/search -fixdir ${VESPA_USER} wheel 755 var/jdisc_container -fixdir ${VESPA_USER} wheel 755 var/vespa -fixdir ${VESPA_USER} wheel 755 var/vespa/application -fixdir ${VESPA_USER} wheel 755 var/vespa/bundlecache -fixdir ${VESPA_USER} wheel 755 var/vespa/bundlecache/configserver -fixdir ${VESPA_USER} wheel 755 var/vespa/cache/config/ +fixdir root root 1777 logs +fixdir root root 1777 tmp +fixdir root root 1777 var/run +fixdir ${VESPA_USER} root 1777 var/crash +fixdir ${VESPA_USER} root 1777 logs/vespa +fixdir ${VESPA_USER} root 1777 tmp/vespa +fixdir root root 755 var +fixdir ${VESPA_USER} root 755 libexec/vespa/plugins/qrs +fixdir ${VESPA_USER} root 755 logs/vespa/configserver +fixdir ${VESPA_USER} root 755 logs/vespa/qrs +fixdir ${VESPA_USER} root 755 logs/vespa/search +fixdir ${VESPA_USER} root 755 var/db/vespa +fixdir ${VESPA_USER} root 755 var/db/vespa/tmp +fixdir ${VESPA_USER} root 755 var/db/vespa/config_server +fixdir ${VESPA_USER} root 755 var/db/vespa/config_server/serverdb +fixdir ${VESPA_USER} root 755 var/db/vespa/config_server/serverdb/tenants +fixdir ${VESPA_USER} root 755 var/db/vespa/filedistribution +fixdir ${VESPA_USER} root 755 var/db/vespa/index +fixdir ${VESPA_USER} root 755 var/db/vespa/logcontrol +fixdir ${VESPA_USER} root 755 var/db/vespa/search +fixdir ${VESPA_USER} root 755 var/jdisc_container +fixdir ${VESPA_USER} root 755 var/vespa +fixdir ${VESPA_USER} root 755 var/vespa/application +fixdir ${VESPA_USER} root 755 var/vespa/bundlecache +fixdir ${VESPA_USER} root 755 var/vespa/bundlecache/configserver +fixdir ${VESPA_USER} root 755 var/vespa/cache/config/ if [ "${VESPA_UNPRIVILEGED}" != yes ]; then chown -hR ${VESPA_USER} logs/vespa diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index b8b6716d879..04f859e2802 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -3022,7 +3022,8 @@ "public static boolean isTextCharacter(int)", "public static java.util.OptionalInt validateTextString(java.lang.String)", "public static boolean isDisplayable(int)", - "public static java.lang.String stripInvalidCharacters(java.lang.String)" + "public static java.lang.String stripInvalidCharacters(java.lang.String)", + "public static java.lang.String truncate(java.lang.String, int)" ], "fields": [] }, diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/CachedThreadPoolWithFallback.java b/vespajlib/src/main/java/com/yahoo/concurrent/CachedThreadPoolWithFallback.java new file mode 100644 index 00000000000..42e86aad1ba --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/CachedThreadPoolWithFallback.java @@ -0,0 +1,63 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.concurrent; + +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * An executor that will first try a bounded cached threadpool before falling back to a unbounded + * single threaded threadpool that will take over dispatching to the primary pool. + * + */ +public class CachedThreadPoolWithFallback implements AutoCloseable, Executor { + private final ExecutorService primary; + private final ExecutorService secondary; + public CachedThreadPoolWithFallback(String baseName, int corePoolSize, int maximumPoolSize, long keepAlimeTime, TimeUnit timeUnit) { + primary = new ThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAlimeTime, timeUnit, + new SynchronousQueue<>(), ThreadFactoryFactory.getDaemonThreadFactory(baseName + ".primary")); + secondary = Executors.newSingleThreadExecutor(ThreadFactoryFactory.getDaemonThreadFactory(baseName + ".secondary")); + } + @Override + public void execute(Runnable command) { + try { + primary.execute(command); + } catch (RejectedExecutionException e1) { + secondary.execute(() -> retryForever(command)); + } + } + private void retryForever(Runnable command) { + while (true) { + try { + primary.execute(command); + return; + } catch (RejectedExecutionException rejected) { + try { + Thread.sleep(1); + } catch (InterruptedException silenced) { } + } + } + } + + @Override + public void close() { + secondary.shutdown(); + join(secondary); + primary.shutdown(); + join(primary); + } + private static void join(ExecutorService executor) { + while (true) { + try { + if (executor.awaitTermination(60, TimeUnit.SECONDS)) { + return; + } + } catch (InterruptedException e) {} + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/text/Text.java b/vespajlib/src/main/java/com/yahoo/text/Text.java index 706fd1583a3..85b28639d89 100644 --- a/vespajlib/src/main/java/com/yahoo/text/Text.java +++ b/vespajlib/src/main/java/com/yahoo/text/Text.java @@ -174,4 +174,16 @@ public final class Text { return stripped != null ? stripped.toString() : string; } + /** + * Returns a string which is never larger than the given number of characters. + * If the string is longer than the given length it will be truncated. + * If length is 4 or less the string will be truncated to length. + * If length is longer than 4, it will be truncated at length-4 with " ..." added at the end. + */ + public static String truncate(String s, int length) { + if (s.length() <= length) return s; + if (length <= 4) return s.substring(0, length); + return s.substring(0, length - 4) + " ..."; + } + } diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/CachedThreadPoolWithFallbackTest.java b/vespajlib/src/test/java/com/yahoo/concurrent/CachedThreadPoolWithFallbackTest.java new file mode 100644 index 00000000000..52e17631a34 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/concurrent/CachedThreadPoolWithFallbackTest.java @@ -0,0 +1,43 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.concurrent; + +import org.junit.Test; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.Assert.assertEquals; + +public class CachedThreadPoolWithFallbackTest { + private static void countAndBlock(AtomicLong counter, long waitLimit) { + counter.incrementAndGet(); + try { + synchronized (counter) { + while (counter.get() < waitLimit) { + counter.wait(); + } + } + } catch (InterruptedException e) {} + } + + @Test + public void testThatTaskAreQueued() throws InterruptedException { + CachedThreadPoolWithFallback executor = new CachedThreadPoolWithFallback("test", 1, 30, 1, TimeUnit.SECONDS); + AtomicLong counter = new AtomicLong(0); + for (int i = 0; i < 1000; i++) { + executor.execute(() -> countAndBlock(counter, 100)); + } + while (counter.get() < 30) { + Thread.sleep(1); + } + Thread.sleep(1); + assertEquals(30L, counter.get()); + counter.set(100); + synchronized (counter) { + counter.notifyAll(); + } + executor.close(); + assertEquals(1070L, counter.get()); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java b/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java index e733b838c39..8bb8b2aaad5 100644 --- a/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java @@ -61,4 +61,15 @@ public class TextTestCase { assertFalse(Text.isDisplayable(0)); } + @Test + public void testTruncate() { + assertEquals("ab", Text.truncate("ab", 5)); + assertEquals("ab", Text.truncate("ab", 6)); + assertEquals("ab", Text.truncate("ab", 2)); + assertEquals("a", Text.truncate("ab", 1)); + assertEquals("", Text.truncate("ab", 0)); + assertEquals("ab c", Text.truncate("ab cde", 4)); + assertEquals("a ...", Text.truncate("ab cde", 5)); + } + } diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 9339cdacea0..979184acbae 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -30,6 +30,7 @@ vespa_define_module( src/tests/component src/tests/compress src/tests/compression + src/tests/crypto src/tests/data/databuffer src/tests/data/input_reader src/tests/data/lz4_encode_decode @@ -139,6 +140,7 @@ vespa_define_module( src/vespa/vespalib src/vespa/vespalib/btree src/vespa/vespalib/component + src/vespa/vespalib/crypto src/vespa/vespalib/data src/vespa/vespalib/data/slime src/vespa/vespalib/datastore diff --git a/vespalib/src/tests/crypto/CMakeLists.txt b/vespalib/src/tests/crypto/CMakeLists.txt new file mode 100644 index 00000000000..b930b5715b5 --- /dev/null +++ b/vespalib/src/tests/crypto/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_crypto_crypto_test_app TEST + SOURCES + crypto_test.cpp + DEPENDS + vespalib + gtest +) +vespa_add_test(NAME vespalib_crypto_crypto_test_app COMMAND vespalib_crypto_crypto_test_app) + diff --git a/vespalib/src/tests/crypto/crypto_test.cpp b/vespalib/src/tests/crypto/crypto_test.cpp new file mode 100644 index 00000000000..8daba954793 --- /dev/null +++ b/vespalib/src/tests/crypto/crypto_test.cpp @@ -0,0 +1,37 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/crypto/private_key.h> +#include <vespa/vespalib/crypto/x509_certificate.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <gmock/gmock.h> + +using namespace ::testing; + +namespace vespalib::crypto { + +// FIXME these tests are very high level and simple since the current crypto utility API we provide +// is extremely simple and does not support loading PEMs, signing or verifying. + +TEST(CryptoTest, generated_p256_ec_private_key_can_be_exported_to_pem_format) { + auto key = PrivateKey::generate_p256_ec_key(); + auto pem = key->private_to_pem(); + EXPECT_THAT(pem, StartsWith("-----BEGIN PRIVATE KEY-----")); +} + +TEST(CryptoTest, generated_x509_certificate_can_be_exported_to_pem_format) { + auto dn = X509Certificate::DistinguishedName() + .country("NO").locality("Trondheim") + .organization("Cool Unit Test Writers") + .organizational_unit("Only the finest tests, yes") + .add_common_name("cooltests.example.com"); + auto subject = X509Certificate::SubjectInfo(std::move(dn)); + auto key = PrivateKey::generate_p256_ec_key(); + auto params = X509Certificate::Params::self_signed(std::move(subject), key); + auto cert = X509Certificate::generate_from(std::move(params)); + auto pem = cert->to_pem(); + EXPECT_THAT(pem, StartsWith("-----BEGIN CERTIFICATE-----")); +} + +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/net/tls/direct_buffer_bio/direct_buffer_bio_test.cpp b/vespalib/src/tests/net/tls/direct_buffer_bio/direct_buffer_bio_test.cpp index 134df9b17eb..a51cfb688c1 100644 --- a/vespalib/src/tests/net/tls/direct_buffer_bio/direct_buffer_bio_test.cpp +++ b/vespalib/src/tests/net/tls/direct_buffer_bio/direct_buffer_bio_test.cpp @@ -5,6 +5,7 @@ #include <cassert> using namespace vespalib; +using namespace vespalib::crypto; using namespace vespalib::net::tls::impl; struct Fixture { diff --git a/vespalib/src/tests/net/tls/openssl_impl/CMakeLists.txt b/vespalib/src/tests/net/tls/openssl_impl/CMakeLists.txt index e8f77d36e16..799e2291d7c 100644 --- a/vespalib/src/tests/net/tls/openssl_impl/CMakeLists.txt +++ b/vespalib/src/tests/net/tls/openssl_impl/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(vespalib_net_tls_openssl_impl_test_app TEST SOURCES - crypto_utils.cpp openssl_impl_test.cpp DEPENDS vespalib diff --git a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp index 54c8c19fc64..4586beef910 100644 --- a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp +++ b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp @@ -1,5 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "crypto_utils.h" +#include <vespa/vespalib/crypto/private_key.h> +#include <vespa/vespalib/crypto/x509_certificate.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/smart_buffer.h> #include <vespa/vespalib/net/tls/authorization_mode.h> @@ -11,11 +12,11 @@ #include <vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h> #include <vespa/vespalib/test/make_tls_options_for_testing.h> #include <vespa/vespalib/test/peer_policy_utils.h> -#include <iostream> #include <stdexcept> #include <stdlib.h> using namespace vespalib; +using namespace vespalib::crypto; using namespace vespalib::net::tls; using namespace vespalib::net::tls::impl; diff --git a/vespalib/src/vespa/vespalib/CMakeLists.txt b/vespalib/src/vespa/vespalib/CMakeLists.txt index 4249f6333a4..95f6a407914 100644 --- a/vespalib/src/vespa/vespalib/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(vespalib SOURCES $<TARGET_OBJECTS:vespalib_vespalib_btree> $<TARGET_OBJECTS:vespalib_vespalib_component> + $<TARGET_OBJECTS:vespalib_vespalib_crypto> $<TARGET_OBJECTS:vespalib_vespalib_data> $<TARGET_OBJECTS:vespalib_vespalib_data_slime> $<TARGET_OBJECTS:vespalib_vespalib_datastore> diff --git a/vespalib/src/vespa/vespalib/crypto/CMakeLists.txt b/vespalib/src/vespa/vespalib/crypto/CMakeLists.txt new file mode 100644 index 00000000000..6000156fcfa --- /dev/null +++ b/vespalib/src/vespa/vespalib/crypto/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(vespalib_vespalib_crypto OBJECT + SOURCES + crypto_exception.cpp + openssl_crypto_impl.cpp + private_key.cpp + x509_certificate.cpp + DEPENDS +) +find_package(OpenSSL) +target_include_directories(vespalib_vespalib_crypto PUBLIC ${OPENSSL_INCLUDE_DIR}) diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_exception.cpp b/vespalib/src/vespa/vespalib/crypto/crypto_exception.cpp index 41bb2060c04..226d8664de6 100644 --- a/vespalib/src/vespa/vespalib/net/tls/crypto_exception.cpp +++ b/vespalib/src/vespa/vespalib/crypto/crypto_exception.cpp @@ -2,7 +2,7 @@ #include "crypto_exception.h" -namespace vespalib::net::tls { +namespace vespalib::crypto { VESPA_IMPLEMENT_EXCEPTION(CryptoException, Exception); diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_exception.h b/vespalib/src/vespa/vespalib/crypto/crypto_exception.h index 696a158e058..0d0dcc8ceec 100644 --- a/vespalib/src/vespa/vespalib/net/tls/crypto_exception.h +++ b/vespalib/src/vespa/vespalib/crypto/crypto_exception.h @@ -3,7 +3,7 @@ #include <vespa/vespalib/util/exception.h> -namespace vespalib::net::tls { +namespace vespalib::crypto { VESPA_DEFINE_EXCEPTION(CryptoException, Exception); diff --git a/vespalib/src/tests/net/tls/openssl_impl/crypto_utils.cpp b/vespalib/src/vespa/vespalib/crypto/openssl_crypto_impl.cpp index 14755360b51..72efbb841c2 100644 --- a/vespalib/src/tests/net/tls/openssl_impl/crypto_utils.cpp +++ b/vespalib/src/vespa/vespalib/crypto/openssl_crypto_impl.cpp @@ -1,13 +1,12 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "crypto_utils.h" -#include <vespa/vespalib/net/tls/crypto_exception.h> +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "openssl_crypto_impl.h" +#include <vespa/vespalib/crypto/crypto_exception.h> #include <cassert> #include <openssl/bn.h> #include <openssl/rand.h> #include <openssl/x509v3.h> -namespace vespalib::net::tls::impl { +namespace vespalib::crypto::openssl_impl { namespace { @@ -61,6 +60,77 @@ BioPtr new_memory_bio() { return bio; } +} // anonymous namespace + +vespalib::string PrivateKeyImpl::private_to_pem() const { + BioPtr bio = new_memory_bio(); + // TODO this API is const-broken even on 1.1.1, revisit in the future... + auto* mutable_pkey = const_cast<::EVP_PKEY*>(_pkey.get()); + if (::PEM_write_bio_PrivateKey(bio.get(), mutable_pkey, nullptr, nullptr, + 0, nullptr, nullptr) != 1) { + throw CryptoException("PEM_write_bio_PrivateKey"); + } + return bio_to_string(*bio); +} + +std::shared_ptr<PrivateKeyImpl> PrivateKeyImpl::generate_openssl_p256_ec_key() { + // We first have to generate an EVP context for the keygen _parameters_... + EvpPkeyCtxPtr params_ctx(::EVP_PKEY_CTX_new_id(EVP_PKEY_EC, nullptr)); + if (!params_ctx) { + throw CryptoException("EVP_PKEY_CTX_new_id"); + } + if (::EVP_PKEY_paramgen_init(params_ctx.get()) != 1) { + throw CryptoException("EVP_PKEY_paramgen_init"); + } + // Set EC keygen parameters to use prime256v1, which is the same as P-256 + if (EVP_PKEY_CTX_set_ec_paramgen_curve_nid(params_ctx.get(), NID_X9_62_prime256v1) <= 0) { + throw CryptoException("EVP_PKEY_CTX_set_ec_paramgen_curve_nid"); + } +#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) + // Must tag _explicitly_ as a named curve or many won't accept our pretty keys. + // If we don't do this, explicit curve parameters are included with the key, + // and this is not widely supported nor needed since we're generating a key on + // a standardized curve. + if (EVP_PKEY_CTX_set_ec_param_enc(params_ctx.get(), OPENSSL_EC_NAMED_CURVE) <= 0) { + throw CryptoException("EVP_PKEY_CTX_set_ec_param_enc"); + } +#endif + // Note: despite being an EVP_PKEY this is not an actual key, just key parameters! + ::EVP_PKEY* params_raw = nullptr; + if (::EVP_PKEY_paramgen(params_ctx.get(), ¶ms_raw) != 1) { + throw CryptoException("EVP_PKEY_paramgen"); + } + EvpPkeyPtr params(params_raw); + // Now we can create a context for the proper key generation + EvpPkeyCtxPtr key_ctx(::EVP_PKEY_CTX_new(params.get(), nullptr)); + if (!params_ctx) { + throw CryptoException("EVP_PKEY_CTX_new"); + } + if (::EVP_PKEY_keygen_init(key_ctx.get()) != 1) { + throw CryptoException("EVP_PKEY_keygen_init"); + } + // Finally, it's time to generate the key pair itself. + ::EVP_PKEY* pkey_raw = nullptr; + if (::EVP_PKEY_keygen(key_ctx.get(), &pkey_raw) != 1) { + throw CryptoException("EVP_PKEY_keygen"); + } + EvpPkeyPtr generated_key(pkey_raw); +#if (OPENSSL_VERSION_NUMBER < 0x10100000L) + // On OpenSSL versions prior to 1.1.0, we must set the named curve ASN1 flag + // directly on the EC_KEY, as the EVP_PKEY wrapper doesn't exist (this is a + // half truth, as it exists on 1.0.2 stable, but not necessarily on all 1.0.2 + // versions, and certainly not on 1.0.1). + EcKeyPtr ec_key(::EVP_PKEY_get1_EC_KEY(generated_key.get())); // Bumps ref count, needs free + if (!ec_key) { + throw CryptoException("EVP_PKEY_get1_EC_KEY"); + } + ::EC_KEY_set_asn1_flag(ec_key.get(), OPENSSL_EC_NAMED_CURVE); +#endif + return std::make_shared<PrivateKeyImpl>(std::move(generated_key), Type::EC); +} + +namespace { + void assign_random_positive_serial_number(::X509& cert) { /* * From RFC3280, section 4.1.2.2: @@ -165,88 +235,23 @@ void add_any_subject_alternate_names(::X509& subject, ::X509& issuer, } } -} // anon ns - -vespalib::string PrivateKey::private_to_pem() const { - BioPtr bio = new_memory_bio(); - // TODO this API is const-broken even on 1.1.1, revisit in the future... - auto* mutable_pkey = const_cast<::EVP_PKEY*>(_pkey.get()); - if (::PEM_write_bio_PrivateKey(bio.get(), mutable_pkey, nullptr, nullptr, - 0, nullptr, nullptr) != 1) { - throw CryptoException("PEM_write_bio_PrivateKey"); - } - return bio_to_string(*bio); -} - -std::shared_ptr<PrivateKey> PrivateKey::generate_p256_ec_key() { - // We first have to generate an EVP context for the keygen _parameters_... - EvpPkeyCtxPtr params_ctx(::EVP_PKEY_CTX_new_id(EVP_PKEY_EC, nullptr)); - if (!params_ctx) { - throw CryptoException("EVP_PKEY_CTX_new_id"); - } - if (::EVP_PKEY_paramgen_init(params_ctx.get()) != 1) { - throw CryptoException("EVP_PKEY_paramgen_init"); - } - // Set EC keygen parameters to use prime256v1, which is the same as P-256 - if (EVP_PKEY_CTX_set_ec_paramgen_curve_nid(params_ctx.get(), NID_X9_62_prime256v1) <= 0) { - throw CryptoException("EVP_PKEY_CTX_set_ec_paramgen_curve_nid"); - } -#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) - // Must tag _explicitly_ as a named curve or many won't accept our pretty keys. - // If we don't do this, explicit curve parameters are included with the key, - // and this is not widely supported nor needed since we're generating a key on - // a standardized curve. - if (EVP_PKEY_CTX_set_ec_param_enc(params_ctx.get(), OPENSSL_EC_NAMED_CURVE) <= 0) { - throw CryptoException("EVP_PKEY_CTX_set_ec_param_enc"); - } -#endif - // Note: despite being an EVP_PKEY this is not an actual key, just key parameters! - ::EVP_PKEY* params_raw = nullptr; - if (::EVP_PKEY_paramgen(params_ctx.get(), ¶ms_raw) != 1) { - throw CryptoException("EVP_PKEY_paramgen"); - } - EvpPkeyPtr params(params_raw); - // Now we can create a context for the proper key generation - EvpPkeyCtxPtr key_ctx(::EVP_PKEY_CTX_new(params.get(), nullptr)); - if (!params_ctx) { - throw CryptoException("EVP_PKEY_CTX_new"); - } - if (::EVP_PKEY_keygen_init(key_ctx.get()) != 1) { - throw CryptoException("EVP_PKEY_keygen_init"); - } - // Finally, it's time to generate the key pair itself. - ::EVP_PKEY* pkey_raw = nullptr; - if (::EVP_PKEY_keygen(key_ctx.get(), &pkey_raw) != 1) { - throw CryptoException("EVP_PKEY_keygen"); - } - EvpPkeyPtr generated_key(pkey_raw); -#if (OPENSSL_VERSION_NUMBER < 0x10100000L) - // On OpenSSL versions prior to 1.1.0, we must set the named curve ASN1 flag - // directly on the EC_KEY, as the EVP_PKEY wrapper doesn't exist (this is a - // half truth, as it exists on 1.0.2 stable, but not necessarily on all 1.0.2 - // versions, and certainly not on 1.0.1). - EcKeyPtr ec_key(::EVP_PKEY_get1_EC_KEY(generated_key.get())); // Bumps ref count, needs free - if (!ec_key) { - throw CryptoException("EVP_PKEY_get1_EC_KEY"); - } - ::EC_KEY_set_asn1_flag(ec_key.get(), OPENSSL_EC_NAMED_CURVE); -#endif - return std::make_shared<PrivateKey>(std::move(generated_key), Type::EC); -} +} // anonymous namespace // Some references: // https://stackoverflow.com/questions/256405/programmatically-create-x509-certificate-using-openssl // https://opensource.apple.com/source/OpenSSL/OpenSSL-22/openssl/demos/x509/mkcert.c -std::shared_ptr<X509Certificate> X509Certificate::generate_from(Params params) { +std::shared_ptr<X509CertificateImpl> X509CertificateImpl::generate_openssl_x509_from(Params params) { X509Ptr cert(::X509_new()); if (!cert) { throw CryptoException("X509_new"); } + // FIXME make this const, currently is not due to OpenSSL API const issues (ugh). + auto& subject_key_impl = dynamic_cast<PrivateKeyImpl&>(*params.subject_key); ::X509_set_version(cert.get(), 2); // 2 actually means v3 :) assign_random_positive_serial_number(*cert); set_certificate_expires_from_now(*cert, params.valid_for); // Internal key copy; does not take ownership - if (::X509_set_pubkey(cert.get(), params.subject_key->native_key()) != 1) { + if (::X509_set_pubkey(cert.get(), subject_key_impl.native_key()) != 1) { throw CryptoException("X509_set_pubkey"); } // The "subject" is the target entity the certificate is intended to, well, certify. @@ -259,13 +264,14 @@ std::shared_ptr<X509Certificate> X509Certificate::generate_from(Params params) { // If we _do_ have an issuer, we'll record its Subject name as our Issuer name. // Note that it's legal to have a self-signed non-CA certificate, though it obviously // cannot be used to sign any subordinate certificates. - ::X509_NAME* issuer_name = (params.issuer - ? ::X509_get_subject_name(params.issuer->native_cert()) + auto* issuer_cert_impl = dynamic_cast<X509CertificateImpl*>(params.issuer.get()); // May be nullptr. + ::X509_NAME* issuer_name = (issuer_cert_impl + ? ::X509_get_subject_name(issuer_cert_impl->native_cert()) : subj_name); if (::X509_set_issuer_name(cert.get(), issuer_name) != 1) { // Makes internal copy throw CryptoException("X509_set_issuer_name"); } - ::X509& issuer_cert = params.issuer ? *params.issuer->native_cert() : *cert; + ::X509& issuer_cert = issuer_cert_impl ? *issuer_cert_impl->native_cert() : *cert; const char* basic_constraints = params.is_ca ? "critical,CA:TRUE" : "critical,CA:FALSE"; const char* key_usage = params.is_ca ? "critical,keyCertSign,digitalSignature" @@ -278,13 +284,14 @@ std::shared_ptr<X509Certificate> X509Certificate::generate_from(Params params) { add_v3_ext(*cert, issuer_cert, NID_authority_key_identifier, "keyid:always"); add_any_subject_alternate_names(*cert, issuer_cert, params.subject_info.subject_alt_names); - if (::X509_sign(cert.get(), params.issuer_key->native_key(), ::EVP_sha256()) == 0) { + auto& issuer_key_impl = dynamic_cast<PrivateKeyImpl&>(*params.issuer_key); + if (::X509_sign(cert.get(), issuer_key_impl.native_key(), ::EVP_sha256()) == 0) { throw CryptoException("X509_sign"); } - return std::make_shared<X509Certificate>(std::move(cert)); + return std::make_shared<X509CertificateImpl>(std::move(cert)); } -vespalib::string X509Certificate::to_pem() const { +vespalib::string X509CertificateImpl::to_pem() const { BioPtr bio = new_memory_bio(); // TODO this API is const-broken, revisit in the future... auto* mutable_cert = const_cast<::X509*>(_cert.get()); @@ -295,47 +302,4 @@ vespalib::string X509Certificate::to_pem() const { } -X509Certificate::DistinguishedName::DistinguishedName() = default; -X509Certificate::DistinguishedName::DistinguishedName(const DistinguishedName&) = default; -X509Certificate::DistinguishedName& X509Certificate::DistinguishedName::operator=(const DistinguishedName&) = default; -X509Certificate::DistinguishedName::DistinguishedName(DistinguishedName&&) = default; -X509Certificate::DistinguishedName& X509Certificate::DistinguishedName::operator=(DistinguishedName&&) = default; -X509Certificate::DistinguishedName::~DistinguishedName() = default; - -X509Certificate::Params::Params() = default; -X509Certificate::Params::~Params() = default; - -X509Certificate::Params -X509Certificate::Params::self_signed(SubjectInfo subject, - std::shared_ptr<PrivateKey> key) { - Params params; - params.subject_info = std::move(subject); - params.subject_key = key; - params.issuer_key = std::move(key); // self-signed, subject == issuer - params.is_ca = true; - return params; -} - -X509Certificate::Params -X509Certificate::Params::issued_by(SubjectInfo subject, - std::shared_ptr<PrivateKey> subject_key, - std::shared_ptr<X509Certificate> issuer, - std::shared_ptr<PrivateKey> issuer_key) { - Params params; - params.subject_info = std::move(subject); - params.issuer = std::move(issuer); - params.subject_key = std::move(subject_key); - params.issuer_key = std::move(issuer_key); - params.is_ca = false; // By default, caller can change for intermediate CAs - return params; -} - -CertKeyWrapper::CertKeyWrapper(std::shared_ptr<X509Certificate> cert_, - std::shared_ptr<PrivateKey> key_) - : cert(std::move(cert_)), - key(std::move(key_)) -{} - -CertKeyWrapper::~CertKeyWrapper() = default; - } diff --git a/vespalib/src/vespa/vespalib/crypto/openssl_crypto_impl.h b/vespalib/src/vespa/vespalib/crypto/openssl_crypto_impl.h new file mode 100644 index 00000000000..87e4ee14e65 --- /dev/null +++ b/vespalib/src/vespa/vespalib/crypto/openssl_crypto_impl.h @@ -0,0 +1,44 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/crypto/openssl_typedefs.h> +#include "private_key.h" +#include "x509_certificate.h" + +namespace vespalib::crypto::openssl_impl { + +class PrivateKeyImpl : public PrivateKey { + EvpPkeyPtr _pkey; + Type _type; +public: + PrivateKeyImpl(EvpPkeyPtr pkey, Type type) + : _pkey(std::move(pkey)), + _type(type) + {} + ~PrivateKeyImpl() override = default; + + ::EVP_PKEY* native_key() noexcept { return _pkey.get(); } + const ::EVP_PKEY* native_key() const noexcept { return _pkey.get(); } + + Type type() const noexcept override { return _type; } + vespalib::string private_to_pem() const override; + + static std::shared_ptr<PrivateKeyImpl> generate_openssl_p256_ec_key(); +}; + +class X509CertificateImpl : public X509Certificate { + X509Ptr _cert; +public: + explicit X509CertificateImpl(X509Ptr cert) : _cert(std::move(cert)) {} + ~X509CertificateImpl() = default; + + ::X509* native_cert() noexcept { return _cert.get(); } + const ::X509* native_cert() const noexcept { return _cert.get(); } + + vespalib::string to_pem() const override; + + // Generates an X509 certificate using a SHA-256 digest + static std::shared_ptr<X509CertificateImpl> generate_openssl_x509_from(Params params); +}; + +} diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_typedefs.h b/vespalib/src/vespa/vespalib/crypto/openssl_typedefs.h index afafe556338..2986a4515f7 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_typedefs.h +++ b/vespalib/src/vespa/vespalib/crypto/openssl_typedefs.h @@ -6,7 +6,7 @@ #include <openssl/crypto.h> #include <openssl/x509.h> -namespace vespalib::net::tls::impl { +namespace vespalib::crypto { struct BioDeleter { void operator()(::BIO* bio) const noexcept { diff --git a/vespalib/src/vespa/vespalib/crypto/private_key.cpp b/vespalib/src/vespa/vespalib/crypto/private_key.cpp new file mode 100644 index 00000000000..7ece9418bef --- /dev/null +++ b/vespalib/src/vespa/vespalib/crypto/private_key.cpp @@ -0,0 +1,11 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "private_key.h" +#include "openssl_crypto_impl.h" + +namespace vespalib::crypto { + +std::shared_ptr<PrivateKey> PrivateKey::generate_p256_ec_key() { + return openssl_impl::PrivateKeyImpl::generate_openssl_p256_ec_key(); +} + +} diff --git a/vespalib/src/vespa/vespalib/crypto/private_key.h b/vespalib/src/vespa/vespalib/crypto/private_key.h new file mode 100644 index 00000000000..7ac5c31502c --- /dev/null +++ b/vespalib/src/vespa/vespalib/crypto/private_key.h @@ -0,0 +1,34 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <memory> + +namespace vespalib::crypto { + +/* + * Represents an asymmetric cryptographic private key. + * + * Can only be used for private/public key crypto, not for secret key (e.g. AES) crypto. + * Currently only supports generating EC keys on the standard P-256 curve. + */ +class PrivateKey { +public: + enum class Type { + EC, + RSA // TODO implement support..! + }; + + virtual ~PrivateKey() = default; + + virtual Type type() const noexcept = 0; + // TODO should have a wrapper for this that takes care to securely erase + // string memory on destruction. + virtual vespalib::string private_to_pem() const = 0; + + static std::shared_ptr<PrivateKey> generate_p256_ec_key(); +protected: + PrivateKey() = default; +}; + +} diff --git a/vespalib/src/vespa/vespalib/crypto/x509_certificate.cpp b/vespalib/src/vespa/vespalib/crypto/x509_certificate.cpp new file mode 100644 index 00000000000..ecd061e573a --- /dev/null +++ b/vespalib/src/vespa/vespalib/crypto/x509_certificate.cpp @@ -0,0 +1,62 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "x509_certificate.h" +#include "openssl_crypto_impl.h" + +namespace vespalib::crypto { + +X509Certificate::DistinguishedName::DistinguishedName() = default; +X509Certificate::DistinguishedName::DistinguishedName(const DistinguishedName&) = default; +X509Certificate::DistinguishedName& X509Certificate::DistinguishedName::operator=(const DistinguishedName&) = default; +X509Certificate::DistinguishedName::DistinguishedName(DistinguishedName&&) noexcept = default; +X509Certificate::DistinguishedName& X509Certificate::DistinguishedName::operator=(DistinguishedName&&) noexcept = default; +X509Certificate::DistinguishedName::~DistinguishedName() = default; + +X509Certificate::Params::Params() = default; +X509Certificate::Params::~Params() = default; + +X509Certificate::Params::Params(const Params&) = default; +X509Certificate::Params& X509Certificate::Params::operator=(const Params&) = default; +X509Certificate::Params::Params(Params&&) noexcept = default; +X509Certificate::Params& X509Certificate::Params::operator=(Params&&) noexcept = default; + +X509Certificate::Params +X509Certificate::Params::self_signed(SubjectInfo subject, + std::shared_ptr<PrivateKey> key) +{ + Params params; + params.subject_info = std::move(subject); + params.subject_key = key; + params.issuer_key = std::move(key); // self-signed, subject == issuer + params.is_ca = true; + return params; +} + +X509Certificate::Params +X509Certificate::Params::issued_by(SubjectInfo subject, + std::shared_ptr<PrivateKey> subject_key, + std::shared_ptr<X509Certificate> issuer, + std::shared_ptr<PrivateKey> issuer_key) +{ + Params params; + params.subject_info = std::move(subject); + params.issuer = std::move(issuer); + params.subject_key = std::move(subject_key); + params.issuer_key = std::move(issuer_key); + params.is_ca = false; // By default, caller can change for intermediate CAs + return params; +} + +std::shared_ptr<X509Certificate> X509Certificate::generate_from(Params params) { + return openssl_impl::X509CertificateImpl::generate_openssl_x509_from(std::move(params)); +} + +CertKeyWrapper::CertKeyWrapper(std::shared_ptr<X509Certificate> cert_, + std::shared_ptr<PrivateKey> key_) + : cert(std::move(cert_)), + key(std::move(key_)) +{} + +CertKeyWrapper::~CertKeyWrapper() = default; + +} diff --git a/vespalib/src/tests/net/tls/openssl_impl/crypto_utils.h b/vespalib/src/vespa/vespalib/crypto/x509_certificate.h index 017dfbdbc12..9eea423c5a0 100644 --- a/vespalib/src/tests/net/tls/openssl_impl/crypto_utils.h +++ b/vespalib/src/vespa/vespalib/crypto/x509_certificate.h @@ -1,52 +1,32 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "private_key.h" #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/net/tls/impl/openssl_typedefs.h> #include <chrono> #include <memory> #include <vector> -// TODOs -// - add unit testing -// - extend interfaces (separate PublicKey etc) -// - hide all OpenSSL details from header -// - move to appropriate new namespace/directory somewhere under vespalib - -namespace vespalib::net::tls::impl { - -class PrivateKey { -public: - enum class Type { - EC, - RSA // TODO implement support..! - }; -private: - EvpPkeyPtr _pkey; - Type _type; -public: - PrivateKey(EvpPkeyPtr pkey, Type type) - : _pkey(std::move(pkey)), - _type(type) - {} - - ::EVP_PKEY* native_key() noexcept { return _pkey.get(); } - const ::EVP_PKEY* native_key() const noexcept { return _pkey.get(); } - - Type type() const noexcept { return _type; } - vespalib::string private_to_pem() const; - - static std::shared_ptr<PrivateKey> generate_p256_ec_key(); -}; - - +namespace vespalib::crypto { + +/** + * Represents an X509 certificate instance and provides utility methods + * for generating new certificates on the fly. Certificates can be created + * for both Certificate Authorities and regular hosts (leaves). + * + * This implementation aims to ensure that best cryptographic practices are + * followed automatically. In particular: + * - The certificate digest is always SHA-256, never SHA-1 or MD5 + * - The certificate serial number is a 160-bit secure random sequence + * (technically 159 bits since the MSB is always zero) rather than a + * collision-prone or predictable sequence number. + * + */ class X509Certificate { - X509Ptr _cert; public: - explicit X509Certificate(X509Ptr cert) : _cert(std::move(cert)) {} + virtual ~X509Certificate() = default; - ::X509* native_cert() noexcept { return _cert.get(); } - const ::X509* native_cert() const noexcept { return _cert.get(); } + virtual vespalib::string to_pem() const = 0; struct DistinguishedName { vespalib::string _country; // "C" @@ -61,9 +41,8 @@ public: DistinguishedName(); DistinguishedName(const DistinguishedName&); DistinguishedName& operator=(const DistinguishedName&); - // TODO make these noexcept once vespalib::string has noexcept move.. or move at all! - DistinguishedName(DistinguishedName&&); - DistinguishedName& operator=(DistinguishedName&&); + DistinguishedName(DistinguishedName&&) noexcept; + DistinguishedName& operator=(DistinguishedName&&) noexcept; ~DistinguishedName(); // TODO could add rvalue overloads as well... @@ -101,8 +80,13 @@ public: Params(); ~Params(); + Params(const Params&); + Params& operator=(const Params&); + Params(Params&&) noexcept; + Params& operator=(Params&&) noexcept; + SubjectInfo subject_info; - // TODO make public key, but private key has both and this is currently just for testing. + // TODO make public key, but private key has both. std::shared_ptr<PrivateKey> subject_key; std::shared_ptr<X509Certificate> issuer; // May be nullptr for self-signed certs std::shared_ptr<PrivateKey> issuer_key; @@ -119,9 +103,14 @@ public: // Generates an X509 certificate using a SHA-256 digest static std::shared_ptr<X509Certificate> generate_from(Params params); - vespalib::string to_pem() const; +protected: + X509Certificate() = default; }; +/* + * Simple wrapper for storing both a X509 certificate and the private key + * that signed it. Useful for testing. + */ struct CertKeyWrapper { std::shared_ptr<X509Certificate> cert; std::shared_ptr<PrivateKey> key; diff --git a/vespalib/src/vespa/vespalib/data/slime/object_value.h b/vespalib/src/vespa/vespalib/data/slime/object_value.h index 377bf5bd37f..651f3a156d2 100644 --- a/vespalib/src/vespa/vespalib/data/slime/object_value.h +++ b/vespalib/src/vespa/vespalib/data/slime/object_value.h @@ -12,8 +12,7 @@ #include <vespa/vespalib/stllike/vector_map.h> #include <vespa/vespalib/util/stash.h> -namespace vespalib { -namespace slime { +namespace vespalib::slime { /** * Class representing a collection of unordered values that can be @@ -32,7 +31,7 @@ private: Cursor &setIfUnset(SymbolInserter &symbol, const ValueFactory &input) { Value *&pos = _fields[symbol.insert()]; - if (pos != 0) { + if (pos != nullptr) { return *NixValue::invalid(); } pos = input.create(_stash); @@ -40,7 +39,7 @@ private: } Value *lookup(const SymbolLookup &symbol) const { - SymbolValueMap::const_iterator pos = _fields.find(symbol.lookup()); + auto pos = _fields.find(symbol.lookup()); if (pos == _fields.end()) { return NixValue::invalid(); } @@ -81,9 +80,7 @@ public: Cursor &setObject(Memory name) override; Symbol resolve(Memory symbol_name) override; - ~ObjectValue() { } + ~ObjectValue() override = default; }; -} // namespace vespalib::slime -} // namespace vespalib - +} diff --git a/vespalib/src/vespa/vespalib/geo/zcurve.cpp b/vespalib/src/vespa/vespalib/geo/zcurve.cpp index b2dc02759dc..00ce7ee18c6 100644 --- a/vespalib/src/vespa/vespalib/geo/zcurve.cpp +++ b/vespalib/src/vespa/vespalib/geo/zcurve.cpp @@ -4,8 +4,7 @@ #include <vespa/vespalib/util/priority_queue.h> #include <vespa/vespalib/util/fiddle.h> -namespace vespalib { -namespace geo { +namespace vespalib::geo { namespace { @@ -182,4 +181,3 @@ ZCurve::decodeSlow(int64_t enc, int32_t *xp, int32_t *yp) } } -} diff --git a/vespalib/src/vespa/vespalib/geo/zcurve.h b/vespalib/src/vespa/vespalib/geo/zcurve.h index 2a05c4c7744..bd76b78ea23 100644 --- a/vespalib/src/vespa/vespalib/geo/zcurve.h +++ b/vespalib/src/vespa/vespalib/geo/zcurve.h @@ -6,8 +6,7 @@ #include <cassert> #include <vector> -namespace vespalib { -namespace geo { +namespace vespalib::geo { /** * @brief Utility methods for a Z-curve (Morton-order) encoder and decoder. @@ -31,7 +30,7 @@ public: public: BoundingBox(int32_t minx, int32_t maxx, int32_t miny, int32_t maxy); - ~BoundingBox() { } + ~BoundingBox() = default; int64_t getzMinx() const { return _zMinx; } int64_t getzMaxx() const { return _zMaxx; } @@ -221,5 +220,3 @@ public: }; } -} - diff --git a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp index a92b0e06bbe..7bbc4b7523c 100644 --- a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp @@ -2,9 +2,9 @@ #include "crypto_engine.h" #include <vespa/vespalib/data/smart_buffer.h> +#include <vespa/vespalib/crypto/crypto_exception.h> #include <vespa/vespalib/net/tls/authorization_mode.h> #include <vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h> -#include <vespa/vespalib/net/tls/crypto_exception.h> #include <vespa/vespalib/net/tls/maybe_tls_crypto_engine.h> #include <vespa/vespalib/net/tls/statistics.h> #include <vespa/vespalib/net/tls/tls_crypto_engine.h> @@ -232,7 +232,7 @@ CryptoEngine::SP create_default_crypto_engine() { CryptoEngine::SP try_create_default_crypto_engine() { try { return create_default_crypto_engine(); - } catch (net::tls::CryptoException &e) { + } catch (crypto::CryptoException &e) { LOG(error, "failed to create default crypto engine: %s", e.what()); std::_Exit(78); } diff --git a/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt b/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt index 6dc48db68e4..b7801f40959 100644 --- a/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt @@ -5,7 +5,6 @@ vespa_add_library(vespalib_vespalib_net_tls OBJECT auto_reloading_tls_crypto_engine.cpp crypto_codec.cpp crypto_codec_adapter.cpp - crypto_exception.cpp maybe_tls_crypto_engine.cpp maybe_tls_crypto_socket.cpp peer_credentials.cpp diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.cpp b/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.cpp index 614722a9769..d7d02534242 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.cpp @@ -1,11 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "direct_buffer_bio.h" -#include <vespa/vespalib/net/tls/crypto_exception.h> +#include <vespa/vespalib/crypto/crypto_exception.h> +#include <vespa/vespalib/util/backtrace.h> #include <utility> #include <cassert> -#include <vespa/vespalib/util/backtrace.h> - #include <vespa/log/log.h> LOG_SETUP(".vespalib.net.tls.impl.direct_buffer_bio"); @@ -19,6 +18,8 @@ LOG_SETUP(".vespalib.net.tls.impl.direct_buffer_bio"); * - https://github.com/indutny/uv_ssl_t/blob/master/src/bio.c */ +using namespace vespalib::crypto; + namespace vespalib::net::tls::impl { namespace { diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.h b/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.h index 581d43d6f29..8492bf2c436 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.h +++ b/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.h @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "openssl_typedefs.h" +#include <vespa/vespalib/crypto/openssl_typedefs.h> #include <openssl/bio.h> /* @@ -22,8 +22,8 @@ namespace vespalib::net::tls::impl { -BioPtr new_mutable_direct_buffer_bio(); -BioPtr new_const_direct_buffer_bio(); +crypto::BioPtr new_mutable_direct_buffer_bio(); +crypto::BioPtr new_const_direct_buffer_bio(); struct MutableBufferView { // Could use a pointer pair instead (or just modify the ptr), but being explicit is good for readability. diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp index 6a79caa8264..1d87a50190e 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp @@ -3,8 +3,8 @@ #include "openssl_tls_context_impl.h" #include "direct_buffer_bio.h" +#include <vespa/vespalib/crypto/crypto_exception.h> #include <vespa/vespalib/net/tls/crypto_codec.h> -#include <vespa/vespalib/net/tls/crypto_exception.h> #include <vespa/vespalib/net/tls/statistics.h> #include <mutex> @@ -36,6 +36,8 @@ LOG_SETUP(".vespalib.net.tls.openssl_crypto_codec_impl"); * light fades and turns to all-enveloping darkness. */ +using namespace vespalib::crypto; + namespace vespalib::net::tls::impl { namespace { diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h index ec8df853c16..80f3e12786a 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h +++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "openssl_typedefs.h" +#include <vespa/vespalib/crypto/openssl_typedefs.h> #include <vespa/vespalib/net/socket_address.h> #include <vespa/vespalib/net/socket_spec.h> #include <vespa/vespalib/net/tls/transport_security_options.h> @@ -47,12 +47,12 @@ class OpenSslCryptoCodecImpl : public CryptoCodec { // The context maintains shared verification callback state, so it must be // kept alive explictly for at least as long as any codecs. std::shared_ptr<OpenSslTlsContextImpl> _ctx; - SocketSpec _peer_spec; - SocketAddress _peer_address; - SslPtr _ssl; - ::BIO* _input_bio; // Owned by _ssl - ::BIO* _output_bio; // Owned by _ssl - Mode _mode; + SocketSpec _peer_spec; + SocketAddress _peer_address; + crypto::SslPtr _ssl; + ::BIO* _input_bio; // Owned by _ssl + ::BIO* _output_bio; // Owned by _ssl + Mode _mode; std::optional<DeferredHandshakeParams> _deferred_handshake_params; std::optional<HandshakeResult> _deferred_handshake_result; public: diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp index 98675ec6b0b..e66baf87999 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp @@ -1,9 +1,9 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "iana_cipher_map.h" -#include "openssl_typedefs.h" #include "openssl_tls_context_impl.h" #include "openssl_crypto_codec_impl.h" -#include <vespa/vespalib/net/tls/crypto_exception.h> +#include <vespa/vespalib/crypto/crypto_exception.h> +#include <vespa/vespalib/crypto/openssl_typedefs.h> #include <vespa/vespalib/net/tls/statistics.h> #include <vespa/vespalib/net/tls/transport_security_options.h> #include <vespa/vespalib/util/stringfmt.h> @@ -26,6 +26,8 @@ LOG_SETUP(".vespalib.net.tls.openssl_tls_context_impl"); # error "Provided OpenSSL version is too darn old, need at least 1.0.0" #endif +using namespace vespalib::crypto; + namespace vespalib::net::tls::impl { namespace { diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h index c519b1ae874..badfe8306d1 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h +++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "openssl_typedefs.h" +#include <vespa/vespalib/crypto/openssl_typedefs.h> #include <vespa/vespalib/net/socket_address.h> #include <vespa/vespalib/net/tls/tls_context.h> #include <vespa/vespalib/net/tls/transport_security_options.h> @@ -13,7 +13,7 @@ namespace vespalib::net::tls::impl { class OpenSslTlsContextImpl : public TlsContext { - SslCtxPtr _ctx; + crypto::SslCtxPtr _ctx; AuthorizationMode _authorization_mode; std::shared_ptr<CertificateVerificationCallback> _cert_verify_callback; TransportSecurityOptions _redacted_transport_options; diff --git a/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp b/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp index b31f2830976..ad46e7272cb 100644 --- a/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp +++ b/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp @@ -1,82 +1,77 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "make_tls_options_for_testing.h" +#include <vespa/vespalib/crypto/private_key.h> +#include <vespa/vespalib/crypto/x509_certificate.h> -/* - * Generated with the following commands: - * - * openssl ecparam -name prime256v1 -genkey -noout -out ca.key - * - * openssl req -new -x509 -nodes -key ca.key \ - * -sha256 -out ca.pem \ - * -subj '/C=US/L=LooneyVille/O=ACME/OU=ACME test CA/CN=acme.example.com' \ - * -days 10000 - * - * openssl ecparam -name prime256v1 -genkey -noout -out host.key - * - * openssl req -new -key host.key -out host.csr \ - * -subj '/C=US/L=LooneyVille/O=Wile. E. Coyote, Ltd./CN=wile.example.com' \ - * -sha256 - * - * openssl x509 -req -in host.csr \ - * -CA ca.pem \ - * -CAkey ca.key \ - * -CAcreateserial \ - * -out host.pem \ - * -days 10000 \ - * -sha256 - * - * TODO generate keypairs and certs at test-time to avoid any hard-coding - * There certs are valid until 2046, so that buys us some time..! - */ - -// ca.pem -constexpr const char* ca_pem = R"(-----BEGIN CERTIFICATE----- -MIIBuDCCAV4CCQDpVjQIixTxvDAKBggqhkjOPQQDAjBkMQswCQYDVQQGEwJVUzEU -MBIGA1UEBwwLTG9vbmV5VmlsbGUxDTALBgNVBAoMBEFDTUUxFTATBgNVBAsMDEFD -TUUgdGVzdCBDQTEZMBcGA1UEAwwQYWNtZS5leGFtcGxlLmNvbTAeFw0xODA4MzEx -MDU3NDVaFw00NjAxMTYxMDU3NDVaMGQxCzAJBgNVBAYTAlVTMRQwEgYDVQQHDAtM -b29uZXlWaWxsZTENMAsGA1UECgwEQUNNRTEVMBMGA1UECwwMQUNNRSB0ZXN0IENB -MRkwFwYDVQQDDBBhY21lLmV4YW1wbGUuY29tMFkwEwYHKoZIzj0CAQYIKoZIzj0D -AQcDQgAE1L7IzCN5pbyVnBATIHieuxq+hf9kWyn5yfjkXMhD52T5ITz1huq4nbiN -YtRoRP7XmipI60R/uiCHzERcsVz4rDAKBggqhkjOPQQDAgNIADBFAiEA6wmZDBca -y0aJ6ABtjbjx/vlmVDxdkaSZSgO8h2CkvIECIFktCkbZhDFfSvbqUScPOGuwkdGQ -L/EW2Bxp+1BPcYoZ ------END CERTIFICATE-----)"; - -// host.pem -constexpr const char* cert_pem = R"(-----BEGIN CERTIFICATE----- -MIIBsTCCAVgCCQD6GfDh0ltpsjAKBggqhkjOPQQDAjBkMQswCQYDVQQGEwJVUzEU -MBIGA1UEBwwLTG9vbmV5VmlsbGUxDTALBgNVBAoMBEFDTUUxFTATBgNVBAsMDEFD -TUUgdGVzdCBDQTEZMBcGA1UEAwwQYWNtZS5leGFtcGxlLmNvbTAeFw0xODA4MzEx -MDU3NDVaFw00NjAxMTYxMDU3NDVaMF4xCzAJBgNVBAYTAlVTMRQwEgYDVQQHDAtM -b29uZXlWaWxsZTEeMBwGA1UECgwVV2lsZS4gRS4gQ295b3RlLCBMdGQuMRkwFwYD -VQQDDBB3aWxlLmV4YW1wbGUuY29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE -e+Y4hxt66em0STviGUj6ZDbxzoLoubXWRml8JDFrEc2S2433KWw2npxYSKVCyo3a -/Vo33V8/H0WgOXioKEZJxDAKBggqhkjOPQQDAgNHADBEAiAN+87hQuGv3z0Ja2BV -b8PHq2vp3BJHjeMuxWu4BFPn0QIgYlvIHikspgGatXRNMZ1gPC0oCccsJFcie+Cw -zL06UPI= ------END CERTIFICATE-----)"; - -// host.key -constexpr const char* key_pem = R"(-----BEGIN EC PRIVATE KEY----- -MHcCAQEEID6di2PFYn8hPrxPbkFDGkSqF+K8L520In7nx3g0jwzOoAoGCCqGSM49 -AwEHoUQDQgAEe+Y4hxt66em0STviGUj6ZDbxzoLoubXWRml8JDFrEc2S2433KWw2 -npxYSKVCyo3a/Vo33V8/H0WgOXioKEZJxA== ------END EC PRIVATE KEY-----)"; +namespace { + +using namespace vespalib::crypto; + +struct TransientCryptoCredentials { + CertKeyWrapper root_ca; + CertKeyWrapper host_creds; + vespalib::net::tls::TransportSecurityOptions cached_transport_options; + + TransientCryptoCredentials(); + ~TransientCryptoCredentials(); + + static CertKeyWrapper make_root_ca() { + auto dn = X509Certificate::DistinguishedName() + .country("US").state("CA").locality("Sunnyvale") + .organization("ACME, Inc.") + .organizational_unit("ACME Root CA") + .add_common_name("acme.example.com"); + auto subject = X509Certificate::SubjectInfo(std::move(dn)); + auto key = PrivateKey::generate_p256_ec_key(); + auto params = X509Certificate::Params::self_signed(std::move(subject), key); + auto cert = X509Certificate::generate_from(std::move(params)); + return {std::move(cert), std::move(key)}; + } + + static CertKeyWrapper make_host_creds(const CertKeyWrapper& root_ca_creds) { + auto dn = X509Certificate::DistinguishedName() + .country("US").state("CA").locality("Sunnyvale") + .organization("Wile E. Coyote, Ltd.") + .organizational_unit("Unit Testing and Anvil Dropping Division") + .add_common_name("localhost"); // Should technically not be needed, but including it anyway. + auto subject = X509Certificate::SubjectInfo(std::move(dn)); + subject.add_subject_alt_name("DNS:localhost"); + auto key = PrivateKey::generate_p256_ec_key(); + auto params = X509Certificate::Params::issued_by(std::move(subject), key, root_ca_creds.cert, root_ca_creds.key); + params.valid_for = std::chrono::hours(1); + auto cert = X509Certificate::generate_from(std::move(params)); + return {std::move(cert), std::move(key)}; + } + + static const TransientCryptoCredentials& instance(); +}; + +TransientCryptoCredentials::TransientCryptoCredentials() + : root_ca(make_root_ca()), + host_creds(make_host_creds(root_ca)), + cached_transport_options(vespalib::net::tls::TransportSecurityOptions::Params(). + ca_certs_pem(root_ca.cert->to_pem()). + cert_chain_pem(host_creds.cert->to_pem()). + private_key_pem(host_creds.key->private_to_pem()). + authorized_peers(vespalib::net::tls::AuthorizedPeers::allow_all_authenticated())) +{} + +TransientCryptoCredentials::~TransientCryptoCredentials() = default; + +const TransientCryptoCredentials& TransientCryptoCredentials::instance() { + static TransientCryptoCredentials test_creds; + return test_creds; +} + +} namespace vespalib::test { SocketSpec local_spec("tcp/localhost:123"); vespalib::net::tls::TransportSecurityOptions make_tls_options_for_testing() { - auto ts_builder = vespalib::net::tls::TransportSecurityOptions::Params(). - ca_certs_pem(ca_pem). - cert_chain_pem(cert_pem). - private_key_pem(key_pem). - authorized_peers(vespalib::net::tls::AuthorizedPeers::allow_all_authenticated()). - disable_hostname_validation(true); // FIXME this is to avoid mass breakage of TLS'd networking tests. - return vespalib::net::tls::TransportSecurityOptions(std::move(ts_builder)); + return TransientCryptoCredentials::instance().cached_transport_options; } } // namespace vespalib::test diff --git a/vespalib/src/vespa/vespalib/util/stash.cpp b/vespalib/src/vespa/vespalib/util/stash.cpp index 18ed0e56e27..836654c2fb2 100644 --- a/vespalib/src/vespa/vespalib/util/stash.cpp +++ b/vespalib/src/vespa/vespalib/util/stash.cpp @@ -59,14 +59,14 @@ Stash::do_alloc(size_t size) } } -Stash::Stash(size_t chunk_size) +Stash::Stash(size_t chunk_size) noexcept : _chunks(nullptr), _cleanup(nullptr), _chunk_size(std::max(size_t(4096), chunk_size)) { } -Stash::Stash(Stash &&rhs) +Stash::Stash(Stash &&rhs) noexcept : _chunks(rhs._chunks), _cleanup(rhs._cleanup), _chunk_size(rhs._chunk_size) @@ -76,7 +76,7 @@ Stash::Stash(Stash &&rhs) } Stash & -Stash::operator=(Stash &&rhs) +Stash::operator=(Stash &&rhs) noexcept { stash::run_cleanup(_cleanup); stash::free_chunks(_chunks); diff --git a/vespalib/src/vespa/vespalib/util/stash.h b/vespalib/src/vespa/vespalib/util/stash.h index aa1441aa0bb..c5e8631ca9e 100644 --- a/vespalib/src/vespa/vespalib/util/stash.h +++ b/vespalib/src/vespa/vespalib/util/stash.h @@ -14,19 +14,19 @@ struct Cleanup { explicit Cleanup(Cleanup *next_in) noexcept : next(next_in) {} virtual void cleanup() = 0; protected: - virtual ~Cleanup() {} + virtual ~Cleanup() = default; }; // used as header for memory allocated outside the stash struct DeleteMemory : public Cleanup { explicit DeleteMemory(Cleanup *next_in) noexcept : Cleanup(next_in) {} - virtual void cleanup() override { free((void*)this); } + void cleanup() override { free((void*)this); } }; // used as prefix for objects to be destructed template<typename T> struct DestructObject : public Cleanup { explicit DestructObject(Cleanup *next_in) noexcept : Cleanup(next_in) {} - virtual void cleanup() override { reinterpret_cast<T*>(this + 1)->~T(); } + void cleanup() override { reinterpret_cast<T*>(this + 1)->~T(); } }; // used as prefix for arrays to be destructed @@ -34,7 +34,7 @@ template<typename T> struct DestructArray : public Cleanup { size_t size; explicit DestructArray(Cleanup *next_in, size_t size_in) noexcept : Cleanup(next_in), size(size_in) {} - virtual void cleanup() override { + void cleanup() override { T *array = reinterpret_cast<T*>(this + 1); for (size_t i = size; i-- > 0;) { array[i].~T(); @@ -46,7 +46,7 @@ struct Chunk { Chunk *next; size_t used; Chunk(const Chunk &) = delete; - Chunk(Chunk *next_in) : next(next_in), used(sizeof(Chunk)) {} + explicit Chunk(Chunk *next_in) : next(next_in), used(sizeof(Chunk)) {} void clear() { used = sizeof(Chunk); } char *alloc(size_t size, size_t chunk_size) { size_t aligned_size = ((size + (sizeof(char *) - 1)) @@ -124,14 +124,14 @@ public: }; typedef std::unique_ptr<Stash> UP; - explicit Stash(size_t chunk_size); - Stash() : Stash(4096) {} - Stash(Stash &&rhs); + explicit Stash(size_t chunk_size) noexcept ; + Stash() noexcept : Stash(4096) {} + Stash(Stash &&rhs) noexcept; Stash(const Stash &) = delete; Stash & operator = (const Stash &) = delete; ~Stash(); - Stash &operator=(Stash &&rhs); + Stash &operator=(Stash &&rhs) noexcept; void clear(); |