diff options
265 files changed, 2539 insertions, 1535 deletions
diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java index 2208b470ed8..fdf2bbfccff 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java @@ -201,7 +201,8 @@ public class InstanceValidatorTest { ApplicationInfo::getApplicationId, Function.identity() ) - )); + ), + true); SuperModelProvider superModelProvider = mock(SuperModelProvider.class); when(superModelProvider.getSuperModel()).thenReturn(superModel); diff --git a/client/src/main/java/ai/vespa/client/dsl/EndQuery.java b/client/src/main/java/ai/vespa/client/dsl/EndQuery.java index 4facf6ce0fb..2af1e0bb49d 100644 --- a/client/src/main/java/ai/vespa/client/dsl/EndQuery.java +++ b/client/src/main/java/ai/vespa/client/dsl/EndQuery.java @@ -177,7 +177,7 @@ public class EndQuery { } else if (others.isEmpty()) { sb.append("order by ").append(orderStr); } else { - sb.append("order by ").append(orderStr).append(", ").append(others); + sb.append("order by ").append(orderStr).append(" ").append(others); } if (groupQueryStr != null) { diff --git a/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy b/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy index e363a9bcc13..19c87d6aecd 100644 --- a/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy +++ b/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy @@ -59,7 +59,7 @@ class QTest extends Specification { .build() expect: - q == """yql=select * from sd1 where f1 contains "v1" and f2 contains "v2" or f3 contains "v3" and !(f4 contains "v4") order by f1 desc, f2 asc, limit 2 offset 1 timeout 3;¶mk1=paramv1""" + q == """yql=select * from sd1 where f1 contains "v1" and f2 contains "v2" or f3 contains "v3" and !(f4 contains "v4") order by f1 desc, f2 asc limit 2 offset 1 timeout 3;¶mk1=paramv1""" } def "matches"() { diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 43527335802..90132d6924a 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -990,13 +990,16 @@ ], "methods": [ "public void <init>()", - "public void <init>(java.util.Map)", + "public void <init>(java.util.Map, boolean)", "public java.util.Map getModelsPerTenant()", "public java.util.Map getModels()", + "public boolean isComplete()", "public java.util.List getAllApplicationInfos()", "public java.util.Optional getApplicationInfo(com.yahoo.config.provision.ApplicationId)", - "public com.yahoo.config.model.api.SuperModel cloneAndSetApplication(com.yahoo.config.model.api.ApplicationInfo)", - "public com.yahoo.config.model.api.SuperModel cloneAndRemoveApplication(com.yahoo.config.provision.ApplicationId)" + "public com.yahoo.config.model.api.SuperModel cloneAndSetApplication(com.yahoo.config.model.api.ApplicationInfo, boolean)", + "public com.yahoo.config.model.api.SuperModel cloneAndRemoveApplication(com.yahoo.config.provision.ApplicationId)", + "public com.yahoo.config.model.api.SuperModel cloneAsComplete()", + "public java.util.Set getApplicationIds()" ], "fields": [] }, @@ -1010,7 +1013,8 @@ ], "methods": [ "public abstract void applicationActivated(com.yahoo.config.model.api.SuperModel, com.yahoo.config.model.api.ApplicationInfo)", - "public abstract void applicationRemoved(com.yahoo.config.model.api.SuperModel, com.yahoo.config.provision.ApplicationId)" + "public abstract void applicationRemoved(com.yahoo.config.model.api.SuperModel, com.yahoo.config.provision.ApplicationId)", + "public abstract void notifyOfCompleteness(com.yahoo.config.model.api.SuperModel)" ], "fields": [] }, diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java index 1735e08c930..15502dac1f1 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java @@ -20,13 +20,15 @@ import java.util.Set; public class SuperModel { private final Map<ApplicationId, ApplicationInfo> models; + private final boolean complete; public SuperModel() { - this.models = Collections.emptyMap(); + this(Collections.emptyMap(), false); } - public SuperModel(Map<ApplicationId, ApplicationInfo> models) { + public SuperModel(Map<ApplicationId, ApplicationInfo> models, boolean complete) { this.models = models; + this.complete = complete; } public Map<TenantName, Set<ApplicationInfo>> getModelsPerTenant() { @@ -45,6 +47,8 @@ public class SuperModel { return ImmutableMap.copyOf(models); } + public boolean isComplete() { return complete; } + public List<ApplicationInfo> getAllApplicationInfos() { return new ArrayList<>(models.values()); } @@ -54,20 +58,22 @@ public class SuperModel { return applicationInfo == null ? Optional.empty() : Optional.of(applicationInfo); } - public SuperModel cloneAndSetApplication(ApplicationInfo application) { + public SuperModel cloneAndSetApplication(ApplicationInfo application, boolean complete) { Map<ApplicationId, ApplicationInfo> newModels = cloneModels(models); newModels.put(application.getApplicationId(), application); - - return new SuperModel(newModels); + return new SuperModel(newModels, complete); } public SuperModel cloneAndRemoveApplication(ApplicationId applicationId) { Map<ApplicationId, ApplicationInfo> newModels = cloneModels(models); newModels.remove(applicationId); - - return new SuperModel(newModels); + return new SuperModel(newModels, complete); } + public SuperModel cloneAsComplete() { return new SuperModel(models, true); } + + public Set<ApplicationId> getApplicationIds() { return models.keySet(); } + private static Map<ApplicationId, ApplicationInfo> cloneModels(Map<ApplicationId, ApplicationInfo> models) { return new LinkedHashMap<>(models); } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java index 497c38af908..e66a7e1ef7e 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java @@ -17,4 +17,12 @@ public interface SuperModelListener { * Application has been removed. */ void applicationRemoved(SuperModel superModel, ApplicationId id); + + /** + * Invoked once all applications that were supposed to be deployed on bootstrap + * have been activated (and the respective {@link #applicationActivated(SuperModel, ApplicationInfo) + * applicationActivated} have been invoked). The SuperModel is then said to be "complete". + * @param superModel + */ + void notifyOfCompleteness(SuperModel superModel); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index f81757ac568..b30cf9248a1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -20,7 +20,9 @@ import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; import ai.vespa.metricsproxy.rpc.RpcServer; import ai.vespa.metricsproxy.service.ConfigSentinelClient; import ai.vespa.metricsproxy.service.SystemPollerProvider; +import ai.vespa.metricsproxy.telegraf.Telegraf; import ai.vespa.metricsproxy.telegraf.TelegrafConfig; +import ai.vespa.metricsproxy.telegraf.TelegrafRegistry; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; @@ -118,6 +120,8 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC addHttpHandler(ApplicationMetricsHandler.class, ApplicationMetricsHandler.V1_PATH); addMetricsProxyComponent(ApplicationMetricsRetriever.class); + + addTelegrafComponents(); } private void addHttpHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) { @@ -133,6 +137,15 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC return metricsHandler; } + private void addTelegrafComponents() { + getAdmin().ifPresent(admin -> { + if (admin.getUserMetrics().usesExternalMetricSystems()) { + addMetricsProxyComponent(Telegraf.class); + addMetricsProxyComponent(TelegrafRegistry.class); + } + }); + } + @Override protected void doPrepare(DeployState deployState) { } @@ -224,7 +237,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC Optional.of(monitoring.getInterval()) : Optional.empty(); } - private void addMetricsProxyComponent(Class<?> componentClass) { + private void addMetricsProxyComponent(Class<?> componentClass) { addSimpleComponent(componentClass.getName(), null, METRICS_PROXY_BUNDLE_NAME); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java index 9b0d9dbfadc..1f81f16a80b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java @@ -28,4 +28,13 @@ public class Metrics { return consumers.keySet().stream() .anyMatch(existing -> existing.equalsIgnoreCase(id)); } + + /** + * Returns true if any of the consumers have specified external metric systems. + */ + public boolean usesExternalMetricSystems() { + return consumers.values().stream() + .anyMatch(consumer -> ! consumer.cloudWatches().isEmpty()); + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java index 6b1a94e16ae..67c7b67ad9e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.http; -import com.google.common.collect.ImmutableList; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.config.application.api.DeployLogger; @@ -30,7 +29,7 @@ public final class AccessControl { public static final ComponentId ACCESS_CONTROL_CHAIN_ID = ComponentId.fromString("access-control-chain"); - private static final List<String> UNPROTECTED_HANDLERS = ImmutableList.of( + public static final List<String> UNPROTECTED_HANDLERS = List.of( FileStatusHandlerComponent.CLASS, ContainerCluster.APPLICATION_STATUS_HANDLER_CLASS, ContainerCluster.BINDINGS_OVERVIEW_HANDLER_CLASS, diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc index 055f57dd7c0..e3ba7dc500d 100644 --- a/config-model/src/main/resources/schema/admin.rnc +++ b/config-model/src/main/resources/schema/admin.rnc @@ -89,7 +89,7 @@ Metrics = element metrics { Cloudwatch = element cloudwatch { attribute region { xsd:Name } & - attribute namespace { xsd:Name } & + attribute namespace { xsd:string { pattern = "[\w_\-/#:\.]+" } } & ( ( element access-key-name { xsd:Name } & diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java index 144c45a7dd2..dbcbad7bf49 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java @@ -1,13 +1,19 @@ package com.yahoo.vespa.model.admin.metricsproxy; +import ai.vespa.metricsproxy.telegraf.Telegraf; import ai.vespa.metricsproxy.telegraf.TelegrafConfig; +import ai.vespa.metricsproxy.telegraf.TelegrafRegistry; +import com.yahoo.component.ComponentId; import com.yahoo.vespa.model.VespaModel; import org.junit.Test; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CLUSTER_CONFIG_ID; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; +import static org.hamcrest.CoreMatchers.hasItem; +import static org.hamcrest.CoreMatchers.not; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; /** * @author gjoranv @@ -15,25 +21,39 @@ import static org.junit.Assert.assertEquals; public class TelegrafTest { @Test - public void telegraf_config_is_generated_for_cloudwatch_in_services() { + public void telegraf_components_are_set_up_when_cloudwatch_is_configured() { + String services = servicesWithCloudwatch(); + VespaModel hostedModel = getModel(services, hosted); + + var clusterComponents = hostedModel.getAdmin().getMetricsProxyCluster().getComponentsMap(); + assertThat(clusterComponents.keySet(), hasItem(ComponentId.fromString(Telegraf.class.getName()))); + assertThat(clusterComponents.keySet(), hasItem(ComponentId.fromString(TelegrafRegistry.class.getName()))); + } + + @Test + public void telegraf_components_are_not_set_up_when_no_external_systems_are_added_in_services() { String services = String.join("\n", "<services>", " <admin version='2.0'>", " <adminserver hostalias='node1'/>", " <metrics>", - " <consumer id='cloudwatch-consumer'>", - " <metric id='my-metric'/>", - " <cloudwatch region='us-east-1' namespace='my-namespace' >", - " <access-key-name>my-access-key</access-key-name>", - " <secret-key-name>my-secret-key</secret-key-name>", - " </cloudwatch>", - " </consumer>", + " <consumer id='foo' />", " </metrics>", " </admin>", - "</services>" - ); + "</services>"); + VespaModel hostedModel = getModel(services, hosted); + + var clusterComponents = hostedModel.getAdmin().getMetricsProxyCluster().getComponentsMap(); + assertThat(clusterComponents.keySet(), not(hasItem(ComponentId.fromString(Telegraf.class.getName())))); + assertThat(clusterComponents.keySet(), not(hasItem(ComponentId.fromString(TelegrafRegistry.class.getName())))); + } + + @Test + public void telegraf_config_is_generated_for_cloudwatch_in_services() { + String services = servicesWithCloudwatch(); VespaModel hostedModel = getModel(services, hosted); TelegrafConfig config = hostedModel.getConfig(TelegrafConfig.class, CLUSTER_CONFIG_ID); + var cloudWatch0 = config.cloudWatch(0); assertEquals("cloudwatch-consumer", cloudWatch0.consumer()); assertEquals("us-east-1", cloudWatch0.region()); @@ -43,6 +63,25 @@ public class TelegrafTest { assertEquals("", cloudWatch0.profile()); } + private String servicesWithCloudwatch() { + return String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='cloudwatch-consumer'>", + " <metric id='my-metric'/>", + " <cloudwatch region='us-east-1' namespace='my-namespace' >", + " <access-key-name>my-access-key</access-key-name>", + " <secret-key-name>my-secret-key</secret-key-name>", + " </cloudwatch>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + } + @Test public void multiple_cloudwatches_are_allowed_for_the_same_consumer() { String services = String.join("\n", diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index b06c93d6406..604e2abbbae 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -27,7 +27,7 @@ </consumer> <consumer id="cloudwatch-self-hosted-with-default-auth"> <metric-set id="public" /> - <cloudwatch region="us-east1" namespace="my-namespace" /> + <cloudwatch region="us-east1" namespace="namespace_legal.chars:/#1" /> </consumer> <consumer id="cloudwatch-self-hosted-with-profile"> <metric id="my-custom-metric" /> diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index 45f8171436d..f2ae997a164 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -506,7 +506,7 @@ ], "methods": [ "public abstract java.util.Optional getDeployment(com.yahoo.config.provision.ApplicationId)", - "public abstract java.util.Map getSupportedInfraDeployments()" + "public abstract void activateAllSupportedInfraApplications(boolean)" ], "fields": [] }, diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java index 6fbabfd0c95..363732ee8a7 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java @@ -1,7 +1,6 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import java.util.Map; import java.util.Optional; /** @@ -17,6 +16,6 @@ public interface InfraDeployer { */ Optional<Deployment> getDeployment(ApplicationId application); - /** Returns deployments by application id for the supported infrastructure applications in this zone */ - Map<ApplicationId, Deployment> getSupportedInfraDeployments(); + /** Deploys all supported infrastructure applications in this zone. */ + void activateAllSupportedInfraApplications(boolean propagateException); } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java index 9d33824e0be..b73cf89d1b4 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java @@ -162,9 +162,9 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer dispatchRpcRequest(req, () -> { StringBuilder sb = new StringBuilder(); sb.append("\nDelayed responses queue size: "); - sb.append(proxyServer.delayedResponses.size()); + sb.append(proxyServer.delayedResponses().size()); sb.append("\nContents: "); - for (DelayedResponse delayed : proxyServer.delayedResponses.responses()) { + for (DelayedResponse delayed : proxyServer.delayedResponses().responses()) { sb.append(delayed.getRequest().toString()).append("\n"); } @@ -357,7 +357,7 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer @Override public void notifyTargetInvalid(Target target) { log.log(LogLevel.DEBUG, () -> "Target invalid " + target); - for (Iterator<DelayedResponse> it = proxyServer.delayedResponses.responses().iterator(); it.hasNext(); ) { + for (Iterator<DelayedResponse> it = proxyServer.delayedResponses().responses().iterator(); it.hasNext(); ) { DelayedResponse delayed = it.next(); JRTServerConfigRequest request = delayed.getRequest(); if (request.getRequest().target().equals(target)) { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java index 2f8a1b463e3..9fb78e7e812 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java @@ -26,4 +26,6 @@ interface ConfigSourceClient { void updateSubscribers(RawConfig config); + DelayedResponses delayedResponses(); + } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java index f9f5c475723..51446882025 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java @@ -18,6 +18,7 @@ class MemoryCacheConfigClient implements ConfigSourceClient { private final static Logger log = Logger.getLogger(MemoryCacheConfigClient.class.getName()); private final MemoryCache cache; + private final DelayedResponses delayedResponses = new DelayedResponses(); MemoryCacheConfigClient(MemoryCache cache) { this.cache = cache; @@ -61,4 +62,9 @@ class MemoryCacheConfigClient implements ConfigSourceClient { @Override public void updateSubscribers(RawConfig config) {} + @Override + public DelayedResponses delayedResponses() { + return delayedResponses; + } + } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java index 0966de940f1..545b962f6ff 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.proxy; -import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Spec; import com.yahoo.jrt.Supervisor; @@ -10,20 +9,15 @@ import com.yahoo.log.LogLevel; import com.yahoo.log.LogSetup; import com.yahoo.log.event.Event; import com.yahoo.vespa.config.RawConfig; -import com.yahoo.vespa.config.TimingValues; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import com.yahoo.vespa.config.proxy.filedistribution.FileDistributionAndUrlDownload; import com.yahoo.yolean.system.CatchSignals; import java.util.List; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Logger; import static com.yahoo.vespa.config.proxy.Mode.ModeName.DEFAULT; -import static java.util.concurrent.TimeUnit.SECONDS; /** * A proxy server that handles RPC config requests. The proxy can run in two modes: @@ -40,58 +34,34 @@ public class ProxyServer implements Runnable { private final static Logger log = Logger.getLogger(ProxyServer.class.getName()); private final AtomicBoolean signalCaught = new AtomicBoolean(false); - - // Scheduled executor that periodically checks for requests that have timed out and response should be returned to clients - private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1, new DaemonThreadFactory()); private final Supervisor supervisor = new Supervisor(new Transport(JRT_TRANSPORT_THREADS)); - private ScheduledFuture<?> delayedResponseScheduler; private final ConfigProxyRpcServer rpcServer; - final DelayedResponses delayedResponses; private ConfigSourceSet configSource; private volatile ConfigSourceClient configClient; - private final TimingValues timingValues; private final MemoryCache memoryCache; - private static final double timingValuesRatio = 0.8; - private final static TimingValues defaultTimingValues; private final FileDistributionAndUrlDownload fileDistributionAndUrlDownload; private volatile Mode mode = new Mode(DEFAULT); - static { - // Proxy should time out before clients upon subscription. - TimingValues tv = new TimingValues(); - tv.setUnconfiguredDelay((long)(tv.getUnconfiguredDelay()* timingValuesRatio)). - setConfiguredErrorDelay((long)(tv.getConfiguredErrorDelay()* timingValuesRatio)). - setSubscribeTimeout((long)(tv.getSubscribeTimeout()* timingValuesRatio)). - setConfiguredErrorTimeout(-1); // Never cache errors - defaultTimingValues = tv; - } - ProxyServer(Spec spec, ConfigSourceSet source, MemoryCache memoryCache, ConfigSourceClient configClient) { - this.delayedResponses = new DelayedResponses(); this.configSource = source; log.log(LogLevel.DEBUG, "Using config source '" + source); - this.timingValues = defaultTimingValues; this.memoryCache = memoryCache; this.rpcServer = createRpcServer(spec); - this.configClient = createClient(rpcServer, delayedResponses, source, timingValues, memoryCache, configClient); + this.configClient = (configClient == null) ? createRpcClient(rpcServer, source, memoryCache) : configClient; this.fileDistributionAndUrlDownload = new FileDistributionAndUrlDownload(supervisor, source); } + @Override public void run() { if (rpcServer != null) { Thread t = new Thread(rpcServer); t.setName("RpcServer"); t.start(); } - // Wait for 5 seconds initially, then run every second - delayedResponseScheduler = scheduler.scheduleAtFixedRate(new DelayedResponseHandler(delayedResponses, - memoryCache, - rpcServer), - 5, 1, SECONDS); } RawConfig resolveConfig(JRTServerConfigRequest req) { @@ -123,7 +93,7 @@ public class ProxyServer implements Runnable { break; case DEFAULT: flush(); - configClient = createRpcClient(); + configClient = createRpcClient(rpcServer, configSource, memoryCache); this.mode = new Mode(modeName); break; default: @@ -132,20 +102,12 @@ public class ProxyServer implements Runnable { log.log(LogLevel.INFO, "Switched from '" + oldMode.name().toLowerCase() + "' mode to '" + getMode().name().toLowerCase() + "' mode"); } - private ConfigSourceClient createClient(RpcServer rpcServer, DelayedResponses delayedResponses, - ConfigSourceSet source, TimingValues timingValues, - MemoryCache memoryCache, ConfigSourceClient client) { - return (client == null) - ? new RpcConfigSourceClient(rpcServer, source, memoryCache, timingValues, delayedResponses) - : client; - } - private ConfigProxyRpcServer createRpcServer(Spec spec) { return (spec == null) ? null : new ConfigProxyRpcServer(this, supervisor, spec); // TODO: Try to avoid first argument being 'this' } - private RpcConfigSourceClient createRpcClient() { - return new RpcConfigSourceClient(rpcServer, configSource, memoryCache, timingValues, delayedResponses); + private static RpcConfigSourceClient createRpcClient(RpcServer rpcServer, ConfigSourceSet source, MemoryCache memoryCache) { + return new RpcConfigSourceClient(rpcServer, source, memoryCache); } private void setupSignalHandler() { @@ -202,14 +164,6 @@ public class ProxyServer implements Runnable { } } - static TimingValues defaultTimingValues() { - return defaultTimingValues; - } - - TimingValues getTimingValues() { - return timingValues; - } - // Cancels all config instances and flushes the cache. When this method returns, // the cache will not be updated again before someone calls getConfig(). private synchronized void flush() { @@ -220,7 +174,7 @@ public class ProxyServer implements Runnable { void stop() { Event.stopping("configproxy", "shutdown"); if (rpcServer != null) rpcServer.shutdown(); - if (delayedResponseScheduler != null) delayedResponseScheduler.cancel(true); + if (configClient != null) configClient.cancel(); flush(); fileDistributionAndUrlDownload.close(); } @@ -240,7 +194,11 @@ public class ProxyServer implements Runnable { void updateSourceConnections(List<String> sources) { configSource = new ConfigSourceSet(sources); flush(); - configClient = createRpcClient(); + configClient = createRpcClient(rpcServer, configSource, memoryCache); + } + + DelayedResponses delayedResponses() { + return configClient.delayedResponses(); } } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java index 47afbe83bb6..2a33e8c6928 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java @@ -22,8 +22,12 @@ import java.util.List; import java.util.concurrent.DelayQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.logging.Logger; +import static java.util.concurrent.TimeUnit.SECONDS; + /** * An Rpc client to a config source * @@ -32,6 +36,8 @@ import java.util.logging.Logger; class RpcConfigSourceClient implements ConfigSourceClient { private final static Logger log = Logger.getLogger(RpcConfigSourceClient.class.getName()); + private static final double timingValuesRatio = 0.8; + private final Supervisor supervisor = new Supervisor(new Transport()); private final RpcServer rpcServer; @@ -40,25 +46,37 @@ class RpcConfigSourceClient implements ConfigSourceClient { private final Object activeSubscribersLock = new Object(); private final MemoryCache memoryCache; private final DelayedResponses delayedResponses; - private final TimingValues timingValues; - + private final static TimingValues timingValues; private final ExecutorService exec; private final JRTConfigRequester requester; + // Scheduled executor that periodically checks for requests that have timed out and response should be returned to clients + private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1, new DaemonThreadFactory()); + private ScheduledFuture<?> delayedResponseScheduler; + + static { + // Proxy should time out before clients upon subscription. + TimingValues tv = new TimingValues(); + tv.setUnconfiguredDelay((long)(tv.getUnconfiguredDelay()* timingValuesRatio)). + setConfiguredErrorDelay((long)(tv.getConfiguredErrorDelay()* timingValuesRatio)). + setSubscribeTimeout((long)(tv.getSubscribeTimeout()* timingValuesRatio)). + setConfiguredErrorTimeout(-1); // Never cache errors + timingValues = tv; + } - - RpcConfigSourceClient(RpcServer rpcServer, - ConfigSourceSet configSourceSet, - MemoryCache memoryCache, - TimingValues timingValues, - DelayedResponses delayedResponses) { + RpcConfigSourceClient(RpcServer rpcServer, ConfigSourceSet configSourceSet, MemoryCache memoryCache) { this.rpcServer = rpcServer; this.configSourceSet = configSourceSet; this.memoryCache = memoryCache; - this.delayedResponses = delayedResponses; - this.timingValues = timingValues; + this.delayedResponses = new DelayedResponses(); checkConfigSources(); exec = Executors.newCachedThreadPool(new DaemonThreadFactory("subscriber-")); requester = JRTConfigRequester.create(configSourceSet, timingValues); + // Wait for 5 seconds initially, then run every second + delayedResponseScheduler = scheduler.scheduleAtFixedRate( + new DelayedResponseHandler(delayedResponses, memoryCache, rpcServer), + 5, + 1, + SECONDS); } /** @@ -140,8 +158,8 @@ class RpcConfigSourceClient implements ConfigSourceClient { log.log(LogLevel.DEBUG, () -> "Already a subscriber running for: " + configCacheKey); } else { log.log(LogLevel.DEBUG, () -> "Could not find good config in cache, creating subscriber for: " + configCacheKey); - UpstreamConfigSubscriber subscriber = new UpstreamConfigSubscriber(input, this, configSourceSet, - timingValues, requester, memoryCache); + UpstreamConfigSubscriber subscriber = + new UpstreamConfigSubscriber(input, this, configSourceSet, timingValues, requester, memoryCache); try { subscriber.subscribe(); activeSubscribers.put(configCacheKey, subscriber); @@ -157,6 +175,8 @@ class RpcConfigSourceClient implements ConfigSourceClient { @Override public void cancel() { shutdownSourceConnections(); + delayedResponseScheduler.cancel(true); + scheduler.shutdown(); } /** @@ -225,4 +245,9 @@ class RpcConfigSourceClient implements ConfigSourceClient { log.log(LogLevel.DEBUG, () -> "Finished updating config for " + config.getKey() + "," + config.getGeneration()); } + @Override + public DelayedResponses delayedResponses() { + return delayedResponses; + } + } diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java index 963c922d5b5..06e55eef4fa 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java @@ -16,6 +16,7 @@ import java.util.List; public class MockConfigSourceClient implements ConfigSourceClient{ private final MockConfigSource configSource; private final MemoryCache memoryCache; + private final DelayedResponses delayedResponses = new DelayedResponses(); MockConfigSourceClient(MockConfigSource configSource, MemoryCache memoryCache) { this.configSource = configSource; @@ -53,7 +54,9 @@ public class MockConfigSourceClient implements ConfigSourceClient{ } @Override - public void updateSubscribers(RawConfig config) { + public void updateSubscribers(RawConfig config) { } + + @Override + public DelayedResponses delayedResponses() { return delayedResponses; } - } } diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java index bc35a8670a3..f52598b3ee5 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java @@ -27,7 +27,7 @@ public class ProxyServerTest { private final MemoryCache memoryCache = new MemoryCache(); private final MockConfigSource source = new MockConfigSource(); - private MockConfigSourceClient client = new MockConfigSourceClient(source, memoryCache); + private final MockConfigSourceClient client = new MockConfigSourceClient(source, memoryCache); private ProxyServer proxy; static final RawConfig fooConfig = ConfigTester.fooConfig; @@ -58,7 +58,6 @@ public class ProxyServerTest { public void basic() { assertTrue(proxy.getMode().isDefault()); assertThat(proxy.getMemoryCache().size(), is(0)); - assertThat(proxy.getTimingValues(), is(ProxyServer.defaultTimingValues())); ConfigTester tester = new ConfigTester(); final MemoryCache memoryCache = proxy.getMemoryCache(); diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClientTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClientTest.java index 35f1dd8fcd8..8510b23bbd2 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClientTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClientTest.java @@ -18,7 +18,6 @@ import static org.junit.Assert.assertEquals; public class RpcConfigSourceClientTest { private MockRpcServer rpcServer; - private DelayedResponses delayedResponses; private RpcConfigSourceClient rpcConfigSourceClient; @Rule @@ -28,10 +27,7 @@ public class RpcConfigSourceClientTest { @Before public void setup() { rpcServer = new MockRpcServer(); - delayedResponses = new DelayedResponses(); - rpcConfigSourceClient = - new RpcConfigSourceClient(rpcServer, new MockConfigSource(), - new MemoryCache(), ProxyServer.defaultTimingValues(), delayedResponses); + rpcConfigSourceClient = new RpcConfigSourceClient(rpcServer, new MockConfigSource(), new MemoryCache()); } @Test @@ -98,7 +94,7 @@ public class RpcConfigSourceClientTest { } private void simulateClientRequestingConfig(RawConfig config) { - delayedResponses.add(new DelayedResponse(JRTServerConfigRequestV3.createFromRequest(JRTConfigRequestFactory.createFromRaw(config, -10L).getRequest()))); + rpcConfigSourceClient.delayedResponses().add(new DelayedResponse(JRTServerConfigRequestV3.createFromRequest(JRTConfigRequestFactory.createFromRaw(config, -10L).getRequest()))); } private void configUpdatedSendResponse(RawConfig config) { diff --git a/config/src/tests/trace/trace.cpp b/config/src/tests/trace/trace.cpp index 41c874eb1d4..9a355f39ecc 100644 --- a/config/src/tests/trace/trace.cpp +++ b/config/src/tests/trace/trace.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/common/trace.h> +#include <vespa/vespalib/data/slime/slime.h> using namespace config; diff --git a/config/src/vespa/config/common/configdefinition.cpp b/config/src/vespa/config/common/configdefinition.cpp index 861fc224867..92af068cff5 100644 --- a/config/src/vespa/config/common/configdefinition.cpp +++ b/config/src/vespa/config/common/configdefinition.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "configdefinition.h" #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/data/slime/slime.h> using namespace vespalib; using namespace vespalib::slime; diff --git a/config/src/vespa/config/common/configdefinition.h b/config/src/vespa/config/common/configdefinition.h index 3d86e5d2567..9154f3ad88a 100644 --- a/config/src/vespa/config/common/configdefinition.h +++ b/config/src/vespa/config/common/configdefinition.h @@ -3,8 +3,11 @@ #include <vespa/vespalib/stllike/string.h> #include <vector> -#include <vespa/vespalib/data/slime/slime.h> +namespace vespalib::slime { + class Cursor; + class Inspector; +} namespace config { /** diff --git a/config/src/vespa/config/common/trace.cpp b/config/src/vespa/config/common/trace.cpp index d1bb154eda9..76310d08c7d 100644 --- a/config/src/vespa/config/common/trace.cpp +++ b/config/src/vespa/config/common/trace.cpp @@ -2,6 +2,7 @@ #include "trace.h" #include <vespa/vespalib/trace/slime_trace_serializer.h> #include <vespa/vespalib/trace/slime_trace_deserializer.h> +#include <vespa/vespalib/data/slime/slime.h> using namespace vespalib; using namespace vespalib::slime; diff --git a/config/src/vespa/config/common/trace.h b/config/src/vespa/config/common/trace.h index 772cdb6f31e..c120fe30d12 100644 --- a/config/src/vespa/config/common/trace.h +++ b/config/src/vespa/config/common/trace.h @@ -1,11 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/trace/tracenode.h> #include <vespa/vespalib/stllike/string.h> -#include <memory> +#include <vespa/vespalib/data/memory.h> +namespace vespalib::slime { + class Cursor; + class Inspector; +} namespace config { /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index b2a10f4bb21..4e1006213c6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -62,6 +62,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable private final StateMonitor stateMonitor; private final VipStatus vipStatus; private final ConfigserverConfig configserverConfig; + private final SuperModelManager superModelManager; private final Duration maxDurationOfRedeployment; private final Duration sleepTimeWhenRedeployingFails; private final RedeployingApplicationsFails exitIfRedeployingApplicationsFails; @@ -70,29 +71,32 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable @SuppressWarnings("unused") @Inject public ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, - VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus) { + VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus, + SuperModelManager superModelManager) { this(applicationRepository, server, versionState, stateMonitor, vipStatus, BOOTSTRAP_IN_CONSTRUCTOR, EXIT_JVM, applicationRepository.configserverConfig().hostedVespa() ? VipStatusMode.VIP_STATUS_FILE - : VipStatusMode.VIP_STATUS_PROGRAMMATICALLY); + : VipStatusMode.VIP_STATUS_PROGRAMMATICALLY, + superModelManager); } // For testing only ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus, Mode mode, VipStatusMode vipStatusMode) { - this(applicationRepository, server, versionState, stateMonitor, vipStatus, mode, CONTINUE, vipStatusMode); + this(applicationRepository, server, versionState, stateMonitor, vipStatus, mode, CONTINUE, vipStatusMode, null); } private ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus, Mode mode, RedeployingApplicationsFails exitIfRedeployingApplicationsFails, - VipStatusMode vipStatusMode) { + VipStatusMode vipStatusMode, SuperModelManager superModelManager) { this.applicationRepository = applicationRepository; this.server = server; this.versionState = versionState; this.stateMonitor = stateMonitor; this.vipStatus = vipStatus; this.configserverConfig = applicationRepository.configserverConfig(); + this.superModelManager = superModelManager; this.maxDurationOfRedeployment = Duration.ofSeconds(configserverConfig.maxDurationOfBootstrap()); this.sleepTimeWhenRedeployingFails = Duration.ofSeconds(configserverConfig.sleepTimeWhenRedeployingFails()); this.exitIfRedeployingApplicationsFails = exitIfRedeployingApplicationsFails; @@ -208,6 +212,9 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable private boolean redeployAllApplications() throws InterruptedException { Instant end = Instant.now().plus(maxDurationOfRedeployment); Set<ApplicationId> applicationsNotRedeployed = applicationRepository.listApplications(); + if (superModelManager != null) { + superModelManager.setBootstrapApplicationSet(applicationsNotRedeployed); + } do { applicationsNotRedeployed = redeployApplications(applicationsNotRedeployed); if ( ! applicationsNotRedeployed.isEmpty()) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java index ea835206b7c..ccd92d4a195 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java @@ -20,6 +20,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Set; /** * Provides a SuperModel - a model of all application instances, and makes it stays @@ -38,6 +39,9 @@ public class SuperModelManager implements SuperModelProvider { private final long masterGeneration; // ConfigserverConfig's generation private final GenerationCounter generationCounter; + // The initial set of applications to be deployed on bootstrap. + private Optional<Set<ApplicationId>> bootstrapApplicationSet = Optional.empty(); + @Inject public SuperModelManager(ConfigserverConfig configserverConfig, NodeFlavors nodeFlavors, @@ -75,6 +79,10 @@ public class SuperModelManager implements SuperModelProvider { listeners.add(listener); SuperModel superModel = superModelConfigProvider.getSuperModel(); superModel.getAllApplicationInfos().forEach(application -> listener.applicationActivated(superModel, application)); + + if (superModel.isComplete()) { + listener.notifyOfCompleteness(superModel); + } } } @@ -89,13 +97,19 @@ public class SuperModelManager implements SuperModelProvider { .getForVersionOrLatest(Optional.empty(), Instant.now()) .toApplicationInfo(); - SuperModel newSuperModel = this.superModelConfigProvider - .getSuperModel() - .cloneAndSetApplication(applicationInfo); + SuperModel oldSuperModel = superModelConfigProvider.getSuperModel(); + SuperModel newSuperModel = oldSuperModel + .cloneAndSetApplication(applicationInfo, isComplete(oldSuperModel)); + generationCounter.increment(); makeNewSuperModelConfigProvider(newSuperModel); - listeners.stream().forEach(listener -> - listener.applicationActivated(newSuperModel, applicationInfo)); + listeners.forEach(listener -> listener.applicationActivated(newSuperModel, applicationInfo)); + + if (!oldSuperModel.isComplete() && newSuperModel.isComplete()) { + for (var listener : listeners) { + listener.notifyOfCompleteness(newSuperModel); + } + } } } @@ -106,11 +120,36 @@ public class SuperModelManager implements SuperModelProvider { .cloneAndRemoveApplication(applicationId); generationCounter.increment(); makeNewSuperModelConfigProvider(newSuperModel); - listeners.stream().forEach(listener -> - listener.applicationRemoved(newSuperModel, applicationId)); + listeners.forEach(listener -> listener.applicationRemoved(newSuperModel, applicationId)); } } + public void setBootstrapApplicationSet(Set<ApplicationId> bootstrapApplicationSet) { + synchronized (monitor) { + this.bootstrapApplicationSet = Optional.of(bootstrapApplicationSet); + + SuperModel superModel = superModelConfigProvider.getSuperModel(); + if (!superModel.isComplete() && isComplete(superModel)) { + // We do NOT increment the generation since completeness is not part of the config: + // generationCounter.increment() + + SuperModel newSuperModel = superModel.cloneAsComplete(); + makeNewSuperModelConfigProvider(newSuperModel); + listeners.forEach(listener -> listener.notifyOfCompleteness(newSuperModel)); + } + } + } + + /** Returns freshly calculated value of isComplete. */ + private boolean isComplete(SuperModel currentSuperModel) { + if (currentSuperModel.isComplete()) return true; + if (bootstrapApplicationSet.isEmpty()) return false; + + Set<ApplicationId> currentApplicationIds = superModelConfigProvider.getSuperModel().getApplicationIds(); + if (currentApplicationIds.size() < bootstrapApplicationSet.get().size()) return false; + return currentApplicationIds.containsAll(bootstrapApplicationSet.get()); + } + private void makeNewSuperModelConfigProvider(SuperModel newSuperModel) { generation = masterGeneration + generationCounter.get(); superModelConfigProvider = new SuperModelConfigProvider(newSuperModel, zone, flagSource); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java index b7486dc7951..561422c1cf8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java @@ -55,7 +55,7 @@ public class SuperModelControllerTest { ApplicationId app = ApplicationId.from(TenantName.from("a"), ApplicationName.from("foo"), InstanceName.defaultName()); models.put(app, new ApplicationInfo(app, 4L, new VespaModel(FilesApplicationPackage.fromFile(testApp)))); - SuperModel superModel = new SuperModel(models); + SuperModel superModel = new SuperModel(models, true); handler = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); } @@ -98,7 +98,7 @@ public class SuperModelControllerTest { models.put(advanced, createApplicationInfo(testApp2, advanced, 4L)); models.put(tooAdvanced, createApplicationInfo(testApp3, tooAdvanced, 4L)); - SuperModel superModel = new SuperModel(models); + SuperModel superModel = new SuperModel(models, true); SuperModelController han = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); LbServicesConfig.Builder lb = new LbServicesConfig.Builder(); han.getSuperModel().getConfig(lb); @@ -126,7 +126,7 @@ public class SuperModelControllerTest { models.put(advanced, createApplicationInfo(testApp2, advanced, 4L)); models.put(tooAdvanced, createApplicationInfo(testApp3, tooAdvanced, 4L)); - SuperModel superModel = new SuperModel(models); + SuperModel superModel = new SuperModel(models, true); SuperModelController han = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); LbServicesConfig.Builder lb = new LbServicesConfig.Builder(); han.getSuperModel().getConfig(lb); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java index c38ea158507..23799f48f91 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java @@ -1,4 +1,4 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.certificates; import com.yahoo.config.provision.ApplicationId; @@ -8,7 +8,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.UUID; /** * @author tokle @@ -24,9 +23,8 @@ public class EndpointCertificateMock implements EndpointCertificateProvider { @Override public EndpointCertificateMetadata requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames, Optional<EndpointCertificateMetadata> currentMetadata) { this.dnsNames.put(applicationId, dnsNames); - String endpointCertificatePrefix = String.format("vespa.tls.%s.%s@%s", applicationId.tenant(), - applicationId.application(), - UUID.randomUUID().toString()); + String endpointCertificatePrefix = String.format("vespa.tls.%s.%s.%s", applicationId.tenant(), + applicationId.application(), applicationId.instance()); return new EndpointCertificateMetadata(endpointCertificatePrefix + "-key", endpointCertificatePrefix + "-cert", 0); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index e3d41873a8e..854f72b27fb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -130,13 +130,18 @@ public class ApplicationController { int count = 0; for (TenantAndApplicationId id: curator.readApplicationIds()) { lockApplicationIfPresent(id, application -> { - if (id.tenant().value().startsWith("by-")) - application = application.with(DeploymentSpec.empty); - else + if (id.tenant().value().startsWith("by-")) { // TODO jonmv: Remove after run once. + for (Instance instance : application.get().instances().values()) + for (ZoneId zone : instance.deployments().keySet()) + deactivate(instance.id(), zone); + curator.removeApplication(id); + } + else { for (InstanceName instance : application.get().deploymentSpec().instanceNames()) - if ( ! application.get().instances().containsKey(instance)) + if (!application.get().instances().containsKey(instance)) application = withNewInstance(application, id.instance(instance)); - store(application); + store(application); + } }); count++; } @@ -229,7 +234,7 @@ public class ApplicationController { * * @throws IllegalArgumentException if the application already exists */ - public Application createApplication(TenantAndApplicationId id, Optional<Credentials> credentials) { + public Application createApplication(TenantAndApplicationId id, Credentials credentials) { try (Lock lock = lock(id)) { if (getApplication(id).isPresent()) throw new IllegalArgumentException("Could not create '" + id + "': Application already exists"); @@ -238,14 +243,9 @@ public class ApplicationController { com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId.validate(id.application().value()); - Optional<Tenant> tenant = controller.tenants().get(id.tenant()); - if (tenant.isEmpty()) + if (controller.tenants().get(id.tenant()).isEmpty()) throw new IllegalArgumentException("Could not create '" + id + "': This tenant does not exist"); - if (tenant.get().type() != Tenant.Type.user) { - if (credentials.isEmpty()) - throw new IllegalArgumentException("Could not create '" + id + "': No credentials provided"); - accessControl.createApplication(id, credentials.get()); - } + accessControl.createApplication(id, credentials); LockedApplication locked = new LockedApplication(new Application(id, clock.instant()), lock); store(locked); @@ -296,10 +296,6 @@ public class ApplicationController { throw new IllegalArgumentException("'" + instanceId + "' is a tester application!"); TenantAndApplicationId applicationId = TenantAndApplicationId.from(instanceId); - if ( getApplication(applicationId).isEmpty() - && controller.tenants().require(instanceId.tenant()).type() == Tenant.Type.user) - createApplication(applicationId, Optional.empty()); - if (getInstance(instanceId).isEmpty()) createInstance(instanceId); @@ -508,11 +504,7 @@ public class ApplicationController { * * @throws IllegalArgumentException if the application has deployments or the caller is not authorized */ - public void deleteApplication(TenantAndApplicationId id, Optional<Credentials> credentials) { - Tenant tenant = controller.tenants().require(id.tenant()); - if (tenant.type() != Tenant.Type.user && credentials.isEmpty()) - throw new IllegalArgumentException("Could not delete application '" + id + "': No credentials provided"); - + public void deleteApplication(TenantAndApplicationId id, Credentials credentials) { lockApplicationOrThrow(id, application -> { var deployments = application.get().instances().values().stream() .filter(instance -> ! instance.deployments().isEmpty()) @@ -531,8 +523,7 @@ public class ApplicationController { applicationStore.removeAll(id.tenant(), id.application()); applicationStore.removeAllTesters(id.tenant(), id.application()); - if (tenant.type() != Tenant.Type.user) - accessControl.deleteApplication(id, credentials.get()); + accessControl.deleteApplication(id, credentials); curator.removeApplication(id); controller.jobController().collectGarbage(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java index 6caf716aed4..12b985d1812 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import java.security.Principal; import java.security.PublicKey; @@ -40,7 +39,6 @@ public abstract class LockedTenant { static LockedTenant of(Tenant tenant, Lock lock) { switch (tenant.type()) { case athenz: return new Athenz((AthenzTenant) tenant); - case user: return new User((UserTenant) tenant); case cloud: return new Cloud((CloudTenant) tenant); default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.getClass().getName() + "'."); } @@ -99,32 +97,6 @@ public abstract class LockedTenant { } - /** A locked UserTenant. */ - public static class User extends LockedTenant { - - private final Optional<Contact> contact; - - private User(TenantName name, Optional<Contact> contact) { - super(name); - this.contact = contact; - } - - private User(UserTenant tenant) { - this(tenant.name(), tenant.contact()); - } - - @Override - public UserTenant get() { - return new UserTenant(name, contact); - } - - public User with(Contact contact) { - return new User(name, Optional.of(contact)); - } - - } - - /** A locked CloudTenant. */ public static class Cloud extends LockedTenant { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index e794334232f..ae905d2b209 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.security.TenantSpec; import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import java.time.Duration; import java.time.Instant; @@ -47,8 +46,13 @@ public class TenantController { Instant start = controller.clock().instant(); int count = 0; for (TenantName name : curator.readTenantNames()) { - lockIfPresent(name, LockedTenant.class, this::store); - count++; + if (name.value().startsWith(Tenant.userPrefix)) // TODO jonmv: Remove after run once. + + curator.removeTenant(name); + else { + lockIfPresent(name, LockedTenant.class, this::store); + count++; + } } log.log(Level.INFO, String.format("Wrote %d tenants in %s", count, Duration.between(start, controller.clock().instant()))); @@ -94,14 +98,6 @@ public class TenantController { curator.writeTenant(tenant.get()); } - /** Create an user tenant with given username */ - public void createUser(UserTenant tenant) { - try (Lock lock = lock(tenant.name())) { - requireNonExistent(tenant.name()); - curator.writeTenant(tenant); - } - } - /** Create a tenant, provided the given credentials are valid. */ public void create(TenantSpec tenantSpec, Credentials credentials) { try (Lock lock = lock(tenantSpec.tenant())) { @@ -141,13 +137,6 @@ public class TenantController { } } - /** Deletes the given user tenant. */ - public void deleteUser(UserTenant tenant) { - try (Lock lock = lock(tenant.name())) { - curator.removeTenant(tenant.name()); - } - } - private void requireNonExistent(TenantName name) { if ( "hosted-vespa".equals(name.value()) || get(name).isPresent() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index 628d7f48c85..301dad94b7e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -29,7 +29,6 @@ import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.security.TenantSpec; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import javax.ws.rs.ForbiddenException; import java.util.Arrays; @@ -186,8 +185,8 @@ public class AthenzFacade implements AccessControl { AthenzIdentity identity = ((AthenzPrincipal) credentials.user()).getIdentity(); List<AthenzDomain> userDomains = ztsClient.getTenantDomains(service, identity, "admin"); return tenants.stream() - .filter(tenant -> tenant.type() == Tenant.Type.user && ((UserTenant) tenant).is(identity.getName()) - || tenant.type() == Tenant.Type.athenz && userDomains.contains(((AthenzTenant) tenant).domain())) + .filter(tenant -> tenant.type() == Tenant.Type.athenz + && userDomains.contains(((AthenzTenant) tenant).domain())) .collect(Collectors.toUnmodifiableList()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateException.java new file mode 100644 index 00000000000..6099ecbb253 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateException.java @@ -0,0 +1,26 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.certificate; + +public class EndpointCertificateException extends RuntimeException { + + private Type type; + + public EndpointCertificateException(Type type, String message) { + super(message); + this.type = type; + } + + public EndpointCertificateException(Type type, String message, Throwable cause) { + super(message, cause); + this.type = type; + } + + public Type type() { + return type; + } + + public enum Type { + CERT_NOT_AVAILABLE, + VERIFICATION_FAILURE + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java index e22d381e615..ca359f5953a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java @@ -48,7 +48,7 @@ import java.util.stream.Stream; /** * Looks up stored endpoint certificate metadata, provisions new certificates if none is found, - * and refreshes certificates if a newer version is available. + * re-provisions if zone is not covered, and uses refreshed certificates if a newer version is available. * * @author andreer */ @@ -62,6 +62,7 @@ public class EndpointCertificateManager { private final EndpointCertificateProvider endpointCertificateProvider; private final Clock clock; private final BooleanFlag useRefreshedEndpointCertificate; + private final BooleanFlag validateEndpointCertificates; private final StringFlag endpointCertificateBackfill; private final BooleanFlag endpointCertInSharedRouting; @@ -76,6 +77,7 @@ public class EndpointCertificateManager { this.endpointCertificateProvider = endpointCertificateProvider; this.clock = clock; this.useRefreshedEndpointCertificate = Flags.USE_REFRESHED_ENDPOINT_CERTIFICATE.bindTo(flagSource); + this.validateEndpointCertificates = Flags.VALIDATE_ENDPOINT_CERTIFICATES.bindTo(flagSource); this.endpointCertificateBackfill = Flags.ENDPOINT_CERTIFICATE_BACKFILL.bindTo(flagSource); this.endpointCertInSharedRouting = Flags.ENDPOINT_CERT_IN_SHARED_ROUTING.bindTo(flagSource); Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> { @@ -90,33 +92,50 @@ public class EndpointCertificateManager { public Optional<EndpointCertificateMetadata> getEndpointCertificateMetadata(Instance instance, ZoneId zone) { boolean endpointCertInSharedRouting = this.endpointCertInSharedRouting.with(FetchVector.Dimension.APPLICATION_ID, instance.id().serializedForm()).value(); - if (!zoneRegistry.zones().directlyRouted().ids().contains(zone) && !endpointCertInSharedRouting) return Optional.empty(); + if (!zoneRegistry.zones().directlyRouted().ids().contains(zone) && !endpointCertInSharedRouting) + return Optional.empty(); + + final var currentCertificateMetadata = curator.readEndpointCertificateMetadata(instance.id()); + + if (currentCertificateMetadata.isEmpty()) { + var provisionedCertificateMetadata = provisionEndpointCertificate(instance, Optional.empty()); + // We do not verify the certificate if one has never existed before - because we do not want to + // wait for it to be available before we deploy. This allows the config server to start + // provisioning nodes ASAP, and the risk is small for a new deployment. + curator.writeEndpointCertificateMetadata(instance.id(), provisionedCertificateMetadata); + return Optional.of(provisionedCertificateMetadata); + } - // Re-use existing certificate if already provisioned - var endpointCertificateMetadata = - curator.readEndpointCertificateMetadata(instance.id()) - .orElseGet(() -> provisionEndpointCertificate(instance)); + // Reprovision certificate if it is missing SANs for the zone we are deploying to + var sansInCertificate = currentCertificateMetadata.get().requestedDnsSans(); + var requiredSansForZone = dnsNamesOf(instance.id(), List.of(zone)); + if (sansInCertificate.isPresent() && !sansInCertificate.get().containsAll(requiredSansForZone)) { + var reprovisionedCertificateMetadata = provisionEndpointCertificate(instance, currentCertificateMetadata); + curator.writeEndpointCertificateMetadata(instance.id(), reprovisionedCertificateMetadata); + // Verification is unlikely to succeed in this case, as certificate must be available first - controller will retry + validateEndpointCertificate(reprovisionedCertificateMetadata, instance, zone); + return Optional.of(reprovisionedCertificateMetadata); + } // If feature flag set for application, look for and use refreshed certificate if (useRefreshedEndpointCertificate.with(FetchVector.Dimension.APPLICATION_ID, instance.id().serializedForm()).value()) { - var latestAvailableVersion = latestVersionInSecretStore(endpointCertificateMetadata); + var latestAvailableVersion = latestVersionInSecretStore(currentCertificateMetadata.get()); - if (latestAvailableVersion.isPresent() && latestAvailableVersion.getAsInt() > endpointCertificateMetadata.version()) { + if (latestAvailableVersion.isPresent() && latestAvailableVersion.getAsInt() > currentCertificateMetadata.get().version()) { var refreshedCertificateMetadata = new EndpointCertificateMetadata( - endpointCertificateMetadata.keyName(), - endpointCertificateMetadata.certName(), + currentCertificateMetadata.get().keyName(), + currentCertificateMetadata.get().certName(), latestAvailableVersion.getAsInt() ); - if (verifyEndpointCertificate(refreshedCertificateMetadata, instance, zone, "Did not refresh, problems with refreshed certificate: ")) - return Optional.of(refreshedCertificateMetadata); + validateEndpointCertificate(refreshedCertificateMetadata, instance, zone); + curator.writeEndpointCertificateMetadata(instance.id(), refreshedCertificateMetadata); + return Optional.of(refreshedCertificateMetadata); } } - // Only log warnings - verifyEndpointCertificate(endpointCertificateMetadata, instance, zone, "Problems while verifying certificate: "); - - return Optional.of(endpointCertificateMetadata); + validateEndpointCertificate(currentCertificateMetadata.get(), instance, zone); + return currentCertificateMetadata; } enum BackfillMode { @@ -150,7 +169,7 @@ public class EndpointCertificateManager { var hashedCn = commonNameHashOf(applicationId, zoneRegistry.system()); // use as join key EndpointCertificateMetadata providerMetadata = sanToEndpointCertificate.get(hashedCn); - if(providerMetadata == null) { + if (providerMetadata == null) { log.log(LogLevel.INFO, "No matching certificate provider metadata found for application " + applicationId.serializedForm()); return; } @@ -179,58 +198,57 @@ public class EndpointCertificateManager { return Sets.intersection(certVersions, keyVersions).stream().mapToInt(Integer::intValue).max(); } - private EndpointCertificateMetadata provisionEndpointCertificate(Instance instance) { + private EndpointCertificateMetadata provisionEndpointCertificate(Instance instance, Optional<EndpointCertificateMetadata> currentMetadata) { List<ZoneId> zones = zoneRegistry.zones().controllerUpgraded().zones().stream().map(ZoneApi::getId).collect(Collectors.toUnmodifiableList()); - EndpointCertificateMetadata provisionedCertificateMetadata = endpointCertificateProvider - .requestCaSignedCertificate(instance.id(), dnsNamesOf(instance.id(), zones), Optional.empty()); - curator.writeEndpointCertificateMetadata(instance.id(), provisionedCertificateMetadata); - return provisionedCertificateMetadata; + return endpointCertificateProvider.requestCaSignedCertificate(instance.id(), dnsNamesOf(instance.id(), zones), currentMetadata); } - private boolean verifyEndpointCertificate(EndpointCertificateMetadata endpointCertificateMetadata, Instance instance, ZoneId zone, String warningPrefix) { - try { - var pemEncodedEndpointCertificate = secretStore.getSecret(endpointCertificateMetadata.certName(), endpointCertificateMetadata.version()); - - if (pemEncodedEndpointCertificate == null) - return logWarning(warningPrefix, "Secret store returned null for certificate"); - - List<X509Certificate> x509CertificateList = X509CertificateUtils.certificateListFromPem(pemEncodedEndpointCertificate); - - if (x509CertificateList.isEmpty()) return logWarning(warningPrefix, "Empty certificate list"); - if (x509CertificateList.size() < 2) - return logWarning(warningPrefix, "Only a single certificate found in chain - intermediate certificates likely missing"); - - Instant now = clock.instant(); - Instant firstExpiry = Instant.MAX; - for (X509Certificate x509Certificate : x509CertificateList) { - Instant notBefore = x509Certificate.getNotBefore().toInstant(); - Instant notAfter = x509Certificate.getNotAfter().toInstant(); - if (now.isBefore(notBefore)) return logWarning(warningPrefix, "Certificate is not yet valid"); - if (now.isAfter(notAfter)) return logWarning(warningPrefix, "Certificate has expired"); - if (notAfter.isBefore(firstExpiry)) firstExpiry = notAfter; - } - - X509Certificate endEntityCertificate = x509CertificateList.get(0); - Set<String> subjectAlternativeNames = X509CertificateUtils.getSubjectAlternativeNames(endEntityCertificate).stream() - .filter(san -> san.getType().equals(SubjectAlternativeName.Type.DNS_NAME)) - .map(SubjectAlternativeName::getValue).collect(Collectors.toSet()); - - if (Sets.intersection(subjectAlternativeNames, Set.copyOf(dnsNamesOf(instance.id(), List.of(zone)))).isEmpty()) { - return logWarning(warningPrefix, "No overlap between SANs in certificate and expected SANs"); + private void validateEndpointCertificate(EndpointCertificateMetadata endpointCertificateMetadata, Instance instance, ZoneId zone) { + if (validateEndpointCertificates.value()) + try { + var pemEncodedEndpointCertificate = secretStore.getSecret(endpointCertificateMetadata.certName(), endpointCertificateMetadata.version()); + + if (pemEncodedEndpointCertificate == null) + throw new EndpointCertificateException(EndpointCertificateException.Type.VERIFICATION_FAILURE, "Secret store returned null for certificate"); + + List<X509Certificate> x509CertificateList = X509CertificateUtils.certificateListFromPem(pemEncodedEndpointCertificate); + + if (x509CertificateList.isEmpty()) + throw new EndpointCertificateException(EndpointCertificateException.Type.VERIFICATION_FAILURE, "Empty certificate list"); + if (x509CertificateList.size() < 2) + throw new EndpointCertificateException(EndpointCertificateException.Type.VERIFICATION_FAILURE, "Only a single certificate found in chain - intermediate certificates likely missing"); + + Instant now = clock.instant(); + Instant firstExpiry = Instant.MAX; + for (X509Certificate x509Certificate : x509CertificateList) { + Instant notBefore = x509Certificate.getNotBefore().toInstant(); + Instant notAfter = x509Certificate.getNotAfter().toInstant(); + if (now.isBefore(notBefore)) + throw new EndpointCertificateException(EndpointCertificateException.Type.VERIFICATION_FAILURE, "Certificate is not yet valid"); + if (now.isAfter(notAfter)) + throw new EndpointCertificateException(EndpointCertificateException.Type.VERIFICATION_FAILURE, "Certificate has expired"); + if (notAfter.isBefore(firstExpiry)) firstExpiry = notAfter; + } + + X509Certificate endEntityCertificate = x509CertificateList.get(0); + Set<String> subjectAlternativeNames = X509CertificateUtils.getSubjectAlternativeNames(endEntityCertificate).stream() + .filter(san -> san.getType().equals(SubjectAlternativeName.Type.DNS_NAME)) + .map(SubjectAlternativeName::getValue).collect(Collectors.toSet()); + + var dnsNamesOfZone = dnsNamesOf(instance.id(), List.of(zone)); + if (!subjectAlternativeNames.containsAll(dnsNamesOfZone)) + throw new EndpointCertificateException(EndpointCertificateException.Type.VERIFICATION_FAILURE, "Certificate is missing required SANs for zone " + zone.value()); + + } catch (SecretNotFoundException s) { + // Normally because the cert is in the process of being provisioned - this will cause a retry in InternalStepRunner + throw new EndpointCertificateException(EndpointCertificateException.Type.CERT_NOT_AVAILABLE, "Certificate not found in secret store"); + } catch (EndpointCertificateException e) { + log.log(LogLevel.WARNING, "Certificate validation failure for " + instance.id().serializedForm(), e); + throw e; + } catch (Exception e) { + log.log(LogLevel.WARNING, "Certificate validation failure for " + instance.id().serializedForm(), e); + throw new EndpointCertificateException(EndpointCertificateException.Type.VERIFICATION_FAILURE, "Certificate validation failure for app " + instance.id().serializedForm(), e); } - - return true; // All good then, hopefully - } catch (SecretNotFoundException s) { - return logWarning(warningPrefix, "Certificate not found in secret store"); - } catch (Exception e) { - log.log(LogLevel.WARNING, "Exception thrown when verifying endpoint certificate", e); - return false; - } - } - - private static boolean logWarning(String warningPrefix, String message) { - log.log(LogLevel.WARNING, warningPrefix + message); - return false; } private List<String> dnsNamesOf(ApplicationId applicationId, List<ZoneId> zones) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 4d09765f78b..927ebca67ec 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -41,6 +41,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.certificate.EndpointCertificateException; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; import com.yahoo.yolean.Exceptions; @@ -299,6 +300,18 @@ public class InternalStepRunner implements StepRunner { } throw e; + } catch (EndpointCertificateException e) { + switch (e.type()) { + case CERT_NOT_AVAILABLE: + // Same as CERTIFICATE_NOT_READY above, only from the controller + if (startTime.plus(endpointCertificateTimeout).isBefore(controller.clock().instant())) { + logger.log("Deployment failed to find provisioned endpoint certificate after " + endpointCertificateTimeout); + return Optional.of(RunStatus.endpointCertificateTimeout); + } + return Optional.empty(); + default: + throw e; // Should be surfaced / fail deployment + } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index be189004f6d..6904bff8548 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -435,10 +435,6 @@ public class JobController { if ( ! type.environment().isManuallyDeployed()) throw new IllegalArgumentException("Direct deployments are only allowed to manually deployed environments."); - if ( controller.tenants().require(id.tenant()).type() == Tenant.Type.user - && controller.applications().getApplication(TenantAndApplicationId.from(id)).isEmpty()) - controller.applications().createApplication(TenantAndApplicationId.from(id), Optional.empty()); - controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { if ( ! application.get().instances().containsKey(id.instance())) application = controller.applications().withNewInstance(application, id); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java index c854c5b45bf..002ec1f4a05 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java @@ -14,7 +14,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import com.yahoo.yolean.Exceptions; import java.time.Duration; @@ -57,11 +56,10 @@ public class ApplicationOwnershipConfirmer extends Maintainer { .filter(application -> application.createdAt().isBefore(controller().clock().instant().minus(Duration.ofDays(90)))) .forEach(application -> { try { - Tenant tenant = tenantOf(application.id()); - tenant.contact().ifPresent(contact -> { // TODO jvenstad: Makes sense to require, and run this only in main? + tenantOf(application.id()).contact().ifPresent(contact -> { // TODO jvenstad: Makes sense to require, and run this only in main? ownershipIssues.confirmOwnership(application.ownershipIssueId(), summaryOf(application.id()), - determineAssignee(tenant, application), + determineAssignee(application), contact) .ifPresent(newIssueId -> store(newIssueId, application.id())); }); @@ -94,7 +92,7 @@ public class ApplicationOwnershipConfirmer extends Maintainer { application.ownershipIssueId().ifPresent(issueId -> { try { Tenant tenant = tenantOf(application.id()); - ownershipIssues.ensureResponse(issueId, tenant.type() == Tenant.Type.athenz ? tenant.contact() : Optional.empty()); + ownershipIssues.ensureResponse(issueId, tenant.contact()); } catch (RuntimeException e) { log.log(Level.INFO, "Exception caught when attempting to escalate issue with id '" + issueId + "': " + Exceptions.toMessageString(e)); @@ -118,8 +116,8 @@ public class ApplicationOwnershipConfirmer extends Maintainer { }); } - private User determineAssignee(Tenant tenant, Application application) { - return application.owner().orElse(tenant instanceof UserTenant ? userFor(tenant) : null); + private User determineAssignee(Application application) { + return application.owner().orElse(null); } private Tenant tenantOf(TenantAndApplicationId applicationId) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java index cbe9d8c70c1..5825285f9b0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java @@ -40,9 +40,6 @@ public class ContactInformationMaintainer extends Maintainer { case athenz: tenants.lockIfPresent(tenant.name(), LockedTenant.Athenz.class, lockedTenant -> tenants.store(lockedTenant.with(contactRetriever.getContact(lockedTenant.get().propertyId())))); return; - case user: tenants.lockIfPresent(tenant.name(), LockedTenant.User.class, lockedTenant -> - tenants.store(lockedTenant.with(contactRetriever.getContact(Optional.empty())))); - return; case cloud: return; default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'."); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java index 3be15b67252..e7c0094b7ab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java @@ -114,7 +114,7 @@ public class DeploymentIssueReporter extends Maintainer { try { Tenant tenant = ownerOf(application.id()); tenant.contact().ifPresent(contact -> { - User assignee = tenant.type() == Tenant.Type.user ? userFor(tenant) : application.owner().orElse(null); + User assignee = application.owner().orElse(null); Optional<IssueId> ourIssueId = application.deploymentIssueId(); IssueId issueId = deploymentIssues.fileUnlessOpen(ourIssueId, application.id().defaultInstance(), assignee, contact); store(application.id(), issueId); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index 7f938885cac..9df87ab4c12 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -19,7 +19,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import java.net.URI; import java.security.Principal; @@ -68,7 +67,6 @@ public class TenantSerializer { switch (tenant.type()) { case athenz: toSlime((AthenzTenant) tenant, tenantObject); break; - case user: toSlime((UserTenant) tenant, tenantObject); break; case cloud: toSlime((CloudTenant) tenant, tenantObject); break; default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'."); } @@ -85,13 +83,6 @@ public class TenantSerializer { }); } - private void toSlime(UserTenant tenant, Cursor tenantObject) { - tenant.contact().ifPresent(contact -> { - Cursor contactCursor = tenantObject.setObject(contactField); - writeContact(contact, contactCursor); - }); - } - private void toSlime(CloudTenant tenant, Cursor root) { developerKeysToSlime(tenant.developerKeys(), root.setArray(pemDeveloperKeysField)); toSlime(tenant.billingInfo(), root.setObject(billingInfoField)); @@ -117,7 +108,7 @@ public class TenantSerializer { switch (type) { case athenz: return athenzTenantFrom(tenantObject); - case user: return userTenantFrom(tenantObject); + case user: return null; // TODO jonmv: Remove when run once. case cloud: return cloudTenantFrom(tenantObject); default: throw new IllegalArgumentException("Unexpected tenant type '" + type + "'."); } @@ -132,12 +123,6 @@ public class TenantSerializer { return new AthenzTenant(name, domain, property, propertyId, contact); } - private UserTenant userTenantFrom(Inspector tenantObject) { - TenantName name = TenantName.from(tenantObject.field(nameField).asString()); - Optional<Contact> contact = contactFrom(tenantObject.field(contactField)); - return new UserTenant(name, contact); - } - private CloudTenant cloudTenantFrom(Inspector tenantObject) { TenantName name = TenantName.from(tenantObject.field(nameField).asString()); BillingInfo billingInfo = billingInfoFrom(tenantObject.field(billingInfoField)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 94d7b120406..5e234d31322 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -90,7 +90,6 @@ import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.yolean.Exceptions; @@ -252,7 +251,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private HttpResponse handlePUT(Path path, HttpRequest request) { - if (path.matches("/application/v4/user")) return createUser(request); + if (path.matches("/application/v4/user")) return new EmptyResponse(); if (path.matches("/application/v4/tenant/{tenant}")) return updateTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); @@ -335,11 +334,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // TODO jonmv: Move to Athenz API. private HttpResponse authenticatedUser(HttpRequest request) { Principal user = requireUserPrincipal(request); - if (user == null) - throw new NotAuthorizedException("You must be authenticated."); String userName = user instanceof AthenzPrincipal ? ((AthenzPrincipal) user).getIdentity().getName() : user.getName(); - TenantName tenantName = TenantName.from(UserTenant.normalizeUser(userName)); List<Tenant> tenants = controller.tenants().asList(new Credentials(user)); Slime slime = new Slime(); @@ -348,7 +344,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Cursor tenantsArray = response.setArray("tenants"); for (Tenant tenant : tenants) tenantInTenantsListToSlime(tenant, request.getUri(), tenantsArray.addObject()); - response.setBool("tenantExists", tenants.stream().anyMatch(tenant -> tenant.name().equals(tenantName))); + response.setBool("tenantExists", true); return new SlimeJsonResponse(slime); } @@ -1423,26 +1419,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return response; } - private HttpResponse createUser(HttpRequest request) { - String user = Optional.of(requireUserPrincipal(request)) - .filter(AthenzPrincipal.class::isInstance) - .map(AthenzPrincipal.class::cast) - .map(AthenzPrincipal::getIdentity) - .filter(AthenzUser.class::isInstance) - .map(AthenzIdentity::getName) - .map(UserTenant::normalizeUser) - .orElseThrow(() -> new ForbiddenException("Not authenticated or not a user.")); - - UserTenant tenant = UserTenant.create(user); - try { - controller.tenants().createUser(tenant); - return new MessageResponse("Created user '" + user + "'"); - } catch (AlreadyExistsException e) { - // Ok - return new MessageResponse("User '" + user + "' already exists"); - } - } - private HttpResponse updateTenant(String tenantName, HttpRequest request) { getTenantOrThrow(tenantName); TenantName tenant = TenantName.from(tenantName); @@ -1463,11 +1439,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse createApplication(String tenantName, String applicationName, HttpRequest request) { Inspector requestObject = toSlime(request.getData()).get(); TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName); - Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user - ? Optional.empty() - : Optional.of(accessControlRequests.credentials(id.tenant(), requestObject, request.getJDiscRequest())); + Credentials credentials = accessControlRequests.credentials(id.tenant(), requestObject, request.getJDiscRequest()); Application application = controller.applications().createApplication(id, credentials); - Slime slime = new Slime(); toSlime(id, slime.setObject(), request); return new SlimeJsonResponse(slime); @@ -1693,16 +1666,13 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse deleteTenant(String tenantName, HttpRequest request) { Optional<Tenant> tenant = controller.tenants().get(tenantName); - if ( ! tenant.isPresent()) + if (tenant.isEmpty()) return ErrorResponse.notFoundError("Could not delete tenant '" + tenantName + "': Tenant not found"); - if (tenant.get().type() == Tenant.Type.user) - controller.tenants().deleteUser((UserTenant) tenant.get()); - else - controller.tenants().delete(tenant.get().name(), - accessControlRequests.credentials(tenant.get().name(), - toSlime(request.getData()).get(), - request.getJDiscRequest())); + controller.tenants().delete(tenant.get().name(), + accessControlRequests.credentials(tenant.get().name(), + toSlime(request.getData()).get(), + request.getJDiscRequest())); // TODO: Change to a message response saying the tenant was deleted return tenant(tenant.get(), request); @@ -1710,9 +1680,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse deleteApplication(String tenantName, String applicationName, HttpRequest request) { TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName); - Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user - ? Optional.empty() - : Optional.of(accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest())); + Credentials credentials = accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest()); controller.applications().deleteApplication(id, credentials); return new MessageResponse("Deleted application " + id); } @@ -1721,9 +1689,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName); controller.applications().deleteInstance(id.instance(instanceName)); if (controller.applications().requireApplication(id).instances().isEmpty()) { - Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user - ? Optional.empty() - : Optional.of(accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest())); + Credentials credentials = accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest()); controller.applications().deleteApplication(id, credentials); } return new MessageResponse("Deleted instance " + id.instance(instanceName).toFullString()); @@ -1797,7 +1763,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { }); }); break; - case user: break; case cloud: { CloudTenant cloudTenant = (CloudTenant) tenant; @@ -1836,7 +1801,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { metaData.setString("athensDomain", athenzTenant.domain().getName()); metaData.setString("property", athenzTenant.property().id()); break; - case user: break; case cloud: break; default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'."); } @@ -2055,7 +2019,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private static String tenantType(Tenant tenant) { switch (tenant.type()) { - case user: return "USER"; case athenz: return "ATHENS"; case cloud: return "CLOUD"; default: throw new IllegalArgumentException("Unknown tenant type: " + tenant.getClass().getSimpleName()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java index e0c750dec80..55133aa06c8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java @@ -65,7 +65,7 @@ public abstract class Tenant { athenz, /** Tenant authenticated through Okta, as a user. */ - user, + user, // TODO jonmv: Remove. /** Tenant authenticated through some cloud identity provider. */ cloud; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/UserTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/UserTenant.java deleted file mode 100644 index a46d847f6f3..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/UserTenant.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.tenant; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; - -import java.util.Optional; - -/** - * Represents an user tenant in hosted Vespa. - * - * @author mpolden - */ -public class UserTenant extends Tenant { - - /** - * This should only be used by serialization. - * Use {@link #create(String)}. - * */ - public UserTenant(TenantName name, Optional<Contact> contact) { - super(name, contact); - } - - @Override - public Type type() { - return Type.user; - } - - public UserTenant(TenantName name) { - super(name, Optional.empty()); - } - - /** Returns true if this is the tenant for the given user name */ - public boolean is(String username) { - return name().value().equals(normalizeUser(username)); - } - - @Override - public String toString() { - return "user tenant '" + name() + "'"; - } - - /** Create a new user tenant */ - public static UserTenant create(String username) { - TenantName name = TenantName.from(username); - return new UserTenant(requireName(requireUser(name))); - } - - public static UserTenant create(String username, Optional<Contact> contact) { - TenantName name = TenantName.from(username); - return new UserTenant(requireName(requireUser(name)), contact); - } - - /** Normalize given username. E.g. foo_bar becomes by-foo-bar */ - public static String normalizeUser(String username) { - int offset = 0; - if (username.startsWith(Tenant.userPrefix)) { - offset = Tenant.userPrefix.length(); - } - return Tenant.userPrefix + username.substring(offset).replace('_', '-'); - } - - private static TenantName requireUser(TenantName name) { - if (!name.value().startsWith(Tenant.userPrefix)) { - throw new IllegalArgumentException("User tenant must have prefix '" + Tenant.userPrefix + "'"); - } - if (name.value().substring(Tenant.userPrefix.length()).contains("_")) { - throw new IllegalArgumentException("User tenant cannot contain '_'"); - } - return name; - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index a816f82c044..b5796bb4ecc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -327,20 +327,20 @@ public final class ControllerTester { return tenant; } - public Optional<Credentials> credentialsFor(TenantName tenantName) { + public Credentials credentialsFor(TenantName tenantName) { Tenant tenant = controller().tenants().require(tenantName); switch (tenant.type()) { case athenz: - return Optional.of(new AthenzCredentials(new AthenzPrincipal(new AthenzUser("user")), + return new AthenzCredentials(new AthenzPrincipal(new AthenzUser("user")), ((AthenzTenant) tenant).domain(), new OktaIdentityToken("okta-identity-token"), - new OktaAccessToken("okta-access-token"))); + new OktaAccessToken("okta-access-token")); case cloud: - return Optional.of(new Credentials(new SimplePrincipal("dev"))); + return new Credentials(new SimplePrincipal("dev")); default: - return Optional.empty(); + throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'"); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java index 93f081be63e..f9fed820a5b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java @@ -25,6 +25,7 @@ import java.security.cert.X509Certificate; import java.time.Clock; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.List; import java.util.Optional; import static org.junit.Assert.assertEquals; @@ -41,7 +42,8 @@ public class EndpointCertificateManagerTest { private final EndpointCertificateMock endpointCertificateMock = new EndpointCertificateMock(); private final InMemoryFlagSource inMemoryFlagSource = new InMemoryFlagSource(); private final Clock clock = Clock.systemUTC(); - private final EndpointCertificateManager endpointCertificateManager = new EndpointCertificateManager(zoneRegistryMock, mockCuratorDb, secretStore, endpointCertificateMock, clock, inMemoryFlagSource); + private final EndpointCertificateManager endpointCertificateManager = + new EndpointCertificateManager(zoneRegistryMock, mockCuratorDb, secretStore, endpointCertificateMock, clock, inMemoryFlagSource); private static final KeyPair testKeyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 192); private static final X509Certificate testCertificate = X509CertificateBuilder @@ -66,7 +68,8 @@ public class EndpointCertificateManagerTest { @Before public void setUp() { zoneRegistryMock.exclusiveRoutingIn(zoneRegistryMock.zones().all().zones()); - testZone = zoneRegistryMock.zones().directlyRouted().zones().stream().findFirst().get().getId(); + testZone = zoneRegistryMock.zones().directlyRouted().zones().stream().findFirst().orElseThrow().getId(); + inMemoryFlagSource.withBooleanFlag(Flags.VALIDATE_ENDPOINT_CERTIFICATES.id(), true); } @Test @@ -81,6 +84,8 @@ public class EndpointCertificateManagerTest { @Test public void reuses_stored_certificate_metadata() { mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7)); + secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 7); + secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate)+X509CertificateUtils.toPem(testCertificate), 7); Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone); assertTrue(endpointCertificateMetadata.isPresent()); assertEquals(testKeyName, endpointCertificateMetadata.get().keyName()); @@ -105,4 +110,15 @@ public class EndpointCertificateManagerTest { assertEquals(8, endpointCertificateMetadata.get().version()); } + @Test + public void reprovisions_certificate_when_necessary() { + mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, "uuid", List.of())); + secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), 0); + secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate)+X509CertificateUtils.toPem(testCertificate), 0); + Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone); + assertTrue(endpointCertificateMetadata.isPresent()); + assertEquals(0, endpointCertificateMetadata.get().version()); + assertEquals(endpointCertificateMetadata, mockCuratorDb.readEndpointCertificateMetadata(testInstance.id())); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java index 8997f34fb98..e7e508f1789 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipI import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; -import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import org.junit.Before; import org.junit.Test; @@ -42,70 +41,70 @@ public class ApplicationOwnershipConfirmerTest { @Test public void testConfirmation() { Optional<Contact> contact = Optional.of(tester.controllerTester().serviceRegistry().contactRetrieverMock().contact()); - var propertyApp = tester.newDeploymentContext(); + var app = tester.newDeploymentContext(); tester.controller().tenants().lockOrThrow(appId.tenant(), LockedTenant.Athenz.class, tenant -> tester.controller().tenants().store(tenant.with(contact.get()))); - propertyApp.submit().deploy(); + app.submit().deploy(); - UserTenant user = UserTenant.create("by-user", contact); - tester.controller().tenants().createUser(user); - var userApp = tester.newDeploymentContext("by-user", "application", "default"); - userApp.submit().deploy(); + var appWithoutContact = tester.newDeploymentContext("other", "application", "default"); + appWithoutContact.submit().deploy(); - assertFalse("No issue is initially stored for a new application.", propertyApp.application().ownershipIssueId().isPresent()); - assertFalse("No issue is initially stored for a new application.", userApp.application().ownershipIssueId().isPresent()); - assertFalse("No escalation has been attempted for a new application", issues.escalatedToContact || issues.escalatedToTerminator); + assertFalse("No issue is initially stored for a new application.", app.application().ownershipIssueId().isPresent()); + assertFalse("No issue is initially stored for a new application.", appWithoutContact.application().ownershipIssueId().isPresent()); + assertFalse("No escalation has been attempted for a new application", issues.escalated); // Set response from the issue mock, which will be obtained by the maintainer on issue filing. Optional<IssueId> issueId = Optional.of(IssueId.from("1")); issues.response = issueId; confirmer.maintain(); - assertFalse("No issue is stored for an application newer than 3 months.", propertyApp.application().ownershipIssueId().isPresent()); - assertFalse("No issue is stored for an application newer than 3 months.", userApp.application().ownershipIssueId().isPresent()); + assertFalse("No issue is stored for an application newer than 3 months.", app.application().ownershipIssueId().isPresent()); + assertFalse("No issue is stored for an application newer than 3 months.", appWithoutContact.application().ownershipIssueId().isPresent()); tester.clock().advance(Duration.ofDays(91)); confirmer.maintain(); - assertEquals("Confirmation issue has been filed for property owned application.", issueId, propertyApp.application().ownershipIssueId()); - assertEquals("Confirmation issue has been filed for user owned application.", issueId, userApp.application().ownershipIssueId()); - assertTrue(issues.escalatedToTerminator); - assertTrue("Both applications have had their responses ensured.", issues.escalatedToContact); + assertEquals("Confirmation issue has been filed for application with contact.", issueId, app.application().ownershipIssueId()); + assertTrue("The confirmation issue response has been ensured.", issues.escalated); + assertEquals("No confirmation issue has been filed for application without contact.", Optional.empty(), appWithoutContact.application().ownershipIssueId()); // No new issue is created, so return empty now. issues.response = Optional.empty(); confirmer.maintain(); - assertEquals("Confirmation issue reference is not updated when no issue id is returned.", issueId, propertyApp.application().ownershipIssueId()); - assertEquals("Confirmation issue reference is not updated when no issue id is returned.", issueId, userApp.application().ownershipIssueId()); - - // The user deletes all production deployments — see that the issue is forgotten. - assertEquals("Confirmation issue for user is still open.", issueId, userApp.application().ownershipIssueId()); - userApp.application().productionDeployments().values().stream().flatMap(List::stream) - .forEach(deployment -> tester.controller().applications().deactivate(userApp.instanceId(), deployment.zone())); - assertTrue("No production deployments are listed for user.", userApp.application().require(InstanceName.defaultName()).productionDeployments().isEmpty()); - confirmer.maintain(); + assertEquals("Confirmation issue reference is not updated when no issue id is returned.", issueId, app.application().ownershipIssueId()); // Time has passed, and a new confirmation issue is in order for the property which is still in production. Optional<IssueId> issueId2 = Optional.of(IssueId.from("2")); issues.response = issueId2; confirmer.maintain(); - assertEquals("A new confirmation issue id is stored when something is returned to the maintainer.", issueId2, propertyApp.application().ownershipIssueId()); - assertEquals("Confirmation issue for application without production deployments has not been filed.", issueId, userApp.application().ownershipIssueId()); + assertEquals("A new confirmation issue id is stored when something is returned to the maintainer.", issueId2, app.application().ownershipIssueId()); - assertFalse("No owner is stored for application", propertyApp.application().owner().isPresent()); + assertFalse("No owner is stored for application", app.application().owner().isPresent()); issues.owner = Optional.of(User.from("username")); confirmer.maintain(); - assertEquals("Owner has been added to application", propertyApp.application().owner().get().username(), "username"); + assertEquals("Owner has been added to application", app.application().owner().get().username(), "username"); + + // The app deletes all production deployments — see that the issue is forgotten. + assertEquals("Confirmation issue for application is still open.", issueId2, app.application().ownershipIssueId()); + app.application().productionDeployments().values().stream().flatMap(List::stream) + .forEach(deployment -> tester.controller().applications().deactivate(app.instanceId(), deployment.zone())); + assertTrue("No production deployments are listed for user.", app.application().require(InstanceName.defaultName()).productionDeployments().isEmpty()); + confirmer.maintain(); + + // Time has passed, and a new confirmation issue is in order for the property which is still in production. + Optional<IssueId> issueId3 = Optional.of(IssueId.from("3")); + issues.response = issueId3; + confirmer.maintain(); + assertEquals("Confirmation issue for application without production deployments has not been filed.", issueId2, app.application().ownershipIssueId()); } private class MockOwnershipIssues implements OwnershipIssues { private Optional<IssueId> response; - private boolean escalatedToContact = false; - private boolean escalatedToTerminator = false; + private boolean escalated = false; private Optional<User> owner = Optional.empty(); @Override @@ -115,8 +114,7 @@ public class ApplicationOwnershipConfirmerTest { @Override public void ensureResponse(IssueId issueId, Optional<Contact> contact) { - if (contact.isPresent()) escalatedToContact = true; - else escalatedToTerminator = true; + escalated = true; } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java index ff1c952c2a5..9c085bb72f7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; -import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import org.junit.Test; import java.net.URI; @@ -77,14 +76,6 @@ public class TenantSerializerTest { } @Test - public void user_tenant() { - UserTenant tenant = UserTenant.create("by-foo", Optional.of(contact())); - UserTenant serialized = (UserTenant) serializer.tenantFrom(serializer.toSlime(tenant)); - assertEquals(tenant.name(), serialized.name()); - assertEquals(contact(), serialized.contact().get()); - } - - @Test public void cloud_tenant() { CloudTenant tenant = new CloudTenant(TenantName.from("elderly-lady"), new BillingInfo("old cat lady", "vespa"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 442770ba23e..cfde999973f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -182,15 +182,16 @@ public class ApplicationApiTest extends ControllerContainerTest { // GET the authenticated user (with associated tenants) tester.assertResponse(request("/application/v4/user", GET).userIdentity(USER_ID), new File("user.json")); - // PUT a user tenant + // TODO jonmv: Remove when dashboard is gone. + // PUT a user tenant — does nothing tester.assertResponse(request("/application/v4/user", PUT).userIdentity(USER_ID), - "{\"message\":\"Created user 'by-myuser'\"}"); + ""); // GET the authenticated user which now exists (with associated tenants) tester.assertResponse(request("/application/v4/user", GET).userIdentity(USER_ID), - new File("user-which-exists.json")); - // DELETE the user + new File("user.json")); + // DELETE the user — it doesn't exist, so access control fails tester.assertResponse(request("/application/v4/tenant/by-myuser", DELETE).userIdentity(USER_ID), - "{\"tenant\":\"by-myuser\",\"type\":\"USER\",\"applications\":[]}"); + "{\n \"code\" : 403,\n \"message\" : \"Access denied\"\n}", 403); // GET all tenants tester.assertResponse(request("/application/v4/tenant/", GET).userIdentity(USER_ID), new File("tenant-list.json")); @@ -753,17 +754,10 @@ public class ApplicationApiTest extends ControllerContainerTest { .userIdentity(USER_ID), "{\"message\":\"Aborting run 2 of staging-test for tenant1.application1.instance1\"}"); - // PUT (create) the authenticated user - byte[] data = new byte[0]; - tester.assertResponse(request("/application/v4/user?user=new_user&domain=by", PUT) - .data(data) - .userIdentity(new UserId("new_user")), // Normalized to by-new-user by API - new File("create-user-response.json")); - // GET user lists only tenants for the authenticated user tester.assertResponse(request("/application/v4/user", GET) .userIdentity(new UserId("other_user")), - "{\"user\":\"other_user\",\"tenants\":[],\"tenantExists\":false}"); + "{\"user\":\"other_user\",\"tenants\":[],\"tenantExists\":true}"); // OPTIONS return 200 OK tester.assertResponse(request("/application/v4/", Request.Method.OPTIONS) @@ -1374,22 +1368,22 @@ public class ApplicationApiTest extends ControllerContainerTest { // PUT (create) the authenticated user tester.assertResponse(request("/application/v4/user?user=new_user&domain=by", PUT) .userIdentity(userId), // Normalized to by-new-user by API - new File("create-user-response.json")); + ""); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from("domain1"), com.yahoo.config.provision.AthenzService.from("service")) .build(); - // POST (deploy) an application to a dev zone + // POST (deploy) an application to a dev zone fails because user tenant is used — these do not exist. MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); tester.assertResponse(request("/application/v4/tenant/by-new-user/application/application1/environment/dev/region/us-west-1/instance/default", POST) .data(entity) .userIdentity(userId), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"User user.new-user is not allowed to launch service domain1.service. Please reach out to the domain admin.\"}", - 400); + "{\n \"code\" : 403,\n \"message\" : \"Access denied\"\n}", + 403); createTenantAndApplication(); - // POST (deploy) an application to dev through a deployment job + // POST (deploy) an application to dev through a deployment job, with user instance and a proper tenant tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/new-user/deploy/dev-us-east-1", POST) .data(entity) .userIdentity(userId), @@ -1401,11 +1395,12 @@ public class ApplicationApiTest extends ControllerContainerTest { .domains.get(ATHENZ_TENANT_DOMAIN) .admin(HostedAthenzIdentities.from(userId)); - // POST (deploy) an application to a dev zone - tester.assertResponse(request("/application/v4/tenant/by-new-user/application/application1/environment/dev/region/us-east-1/instance/default", POST) + // POST (deploy) an application to a dev zone fails because user tenant is used — these do not exist. + tester.assertResponse(request("/application/v4/tenant/by-new-user/application/application1/environment/dev/region/us-west-1/instance/default", POST) .data(entity) .userIdentity(userId), - new File("deploy-result.json")); + "{\n \"code\" : 403,\n \"message\" : \"Access denied\"\n}", + 403); // POST (deploy) an application to dev through a deployment job tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/new-user/deploy/dev-us-east-1", POST) @@ -1434,7 +1429,7 @@ public class ApplicationApiTest extends ControllerContainerTest { AthenzCredentials credentials = new AthenzCredentials( new AthenzPrincipal(new AthenzUser(developer.id())), sandboxDomain, OKTA_IT, OKTA_AT); tester.controller().tenants().create(tenantSpec, credentials); - tester.controller().applications().createApplication(TenantAndApplicationId.from("sandbox", "myapp"), Optional.of(credentials)); + tester.controller().applications().createApplication(TenantAndApplicationId.from("sandbox", "myapp"), credentials); // Create an application package referencing the service from the other domain ApplicationPackage applicationPackage = new ApplicationPackageBuilder() diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json index 79b9a785801..9902267dbb5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json @@ -1,5 +1,5 @@ { "user": "myuser", "tenants": @include(tenant-list.json), - "tenantExists": false + "tenantExists": true }
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index d70a09414bb..93d88ff8abd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -71,10 +71,10 @@ public class UserApiTest extends ControllerContainerCloudTest { .data("{\"token\":\"hello\"}"), new File("tenant-without-applications.json")); - // PUT a tenant is not available to anyone. + // PUT a tenant is ignored. tester.assertResponse(request("/application/v4/user/", PUT) .roles(operator), - "{\"error-code\":\"FORBIDDEN\",\"message\":\"Not authenticated or not a user.\"}", 403); + "", 200); // GET at user/v1 root fails as no access control is defined there. tester.assertResponse(request("/user/v1/"), diff --git a/dist/vespa.spec b/dist/vespa.spec index 8ce494fd7cb..1c1f5071684 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -6,6 +6,12 @@ # Force special prefix for Vespa %define _prefix /opt/vespa %define _vespa_deps_prefix /opt/vespa-deps +%define _vespa_user vespa +%define _vespa_group vespa +%define _create_vespa_group 1 +%define _create_vespa_user 1 +%define _create_vespa_service 1 +%define _defattr_is_vespa_vespa 0 Name: vespa Version: _VESPA_VERSION_ @@ -211,7 +217,7 @@ Vespa - The open big data serving engine %prep %if 0%{?installdir:1} -%setup -D -T +%setup -c -D -T %else %setup -q %endif @@ -238,6 +244,7 @@ cmake3 -DCMAKE_INSTALL_PREFIX=%{_prefix} \ -DEXTRA_INCLUDE_DIRECTORY="%{_extra_include_directory}" \ -DCMAKE_INSTALL_RPATH="%{_prefix}/lib64%{?_extra_link_directory:;%{_extra_link_directory}};/usr/lib/jvm/jre-11-openjdk/lib" \ %{?_vespa_llvm_version:-DVESPA_LLVM_VERSION="%{_vespa_llvm_version}"} \ + -DVESPA_USER=%{_vespa_user} \ -DVESPA_UNPRIVILEGED=no \ . @@ -253,49 +260,97 @@ cp -r %{installdir} %{buildroot} make install DESTDIR=%{buildroot} %endif +%if %{_create_vespa_service} mkdir -p %{buildroot}/usr/lib/systemd/system cp %{buildroot}/%{_prefix}/etc/systemd/system/vespa.service %{buildroot}/usr/lib/systemd/system cp %{buildroot}/%{_prefix}/etc/systemd/system/vespa-configserver.service %{buildroot}/usr/lib/systemd/system +%endif %clean rm -rf $RPM_BUILD_ROOT %pre -getent group vespa >/dev/null || groupadd -r vespa -getent passwd vespa >/dev/null || \ - useradd -r -g vespa --home-dir %{_prefix} --create-home -s /sbin/nologin \ - -c "Create owner of all Vespa data files" vespa -# Home dir created with rwx on user only. -chmod a+rx %{_prefix} +%if %{_create_vespa_group} +getent group %{_vespa_group} >/dev/null || groupadd -r %{_vespa_group} +%endif +%if %{_create_vespa_user} +getent passwd %{_vespa_user} >/dev/null || \ + useradd -r -g %{_vespa_group} --home-dir %{_prefix} -s /sbin/nologin \ + -c "Create owner of all Vespa data files" %{_vespa_user} +%endif echo "pathmunge %{_prefix}/bin" > /etc/profile.d/vespa.sh echo "export VESPA_HOME=%{_prefix}" >> /etc/profile.d/vespa.sh exit 0 +%if %{_create_vespa_service} %post %systemd_post vespa-configserver.service %systemd_post vespa.service +%endif +%if %{_create_vespa_service} %preun %systemd_preun vespa.service %systemd_preun vespa-configserver.service +%endif %postun +%if %{_create_vespa_service} %systemd_postun_with_restart vespa.service %systemd_postun_with_restart vespa-configserver.service +%endif if [ $1 -eq 0 ]; then # this is an uninstallation rm -f /etc/profile.d/vespa.sh - ! getent passwd vespa >/dev/null || userdel vespa - ! getent group vespa >/dev/null || groupdel vespa +%if %{_create_vespa_user} + ! getent passwd %{_vespa_user} >/dev/null || userdel %{_vespa_user} +%endif +%if %{_create_vespa_group} + ! getent group %{_vespa_group} >/dev/null || groupdel %{_vespa_group} +%endif fi %files -%defattr(-,vespa,vespa,-) +%if %{_defattr_is_vespa_vespa} +%defattr(-,%{_vespa_user},%{_vespa_group},-) +%endif %doc -%{_prefix}/* +%dir %{_prefix} +%{_prefix}/bin +%dir %{_prefix}/conf +%{_prefix}/conf/configserver +%{_prefix}/conf/configserver-app +%dir %{_prefix}/conf/logd +%{_prefix}/conf/node-admin-app +%dir %{_prefix}/conf/vespa +%dir %attr(-,%{_vespa_user},-) %{_prefix}/conf/zookeeper +%dir %{_prefix}/etc +%{_prefix}/etc/systemd +%{_prefix}/etc/vespa +%{_prefix}/include +%{_prefix}/lib +%{_prefix}/lib64 +%{_prefix}/libexec +%dir %attr(1777,-,-) %{_prefix}/logs +%dir %attr(1777,%{_vespa_user},-) %{_prefix}/logs/vespa +%dir %attr(-,%{_vespa_user},-) %{_prefix}/logs/vespa/configserver +%dir %attr(-,%{_vespa_user},-) %{_prefix}/logs/vespa/node-admin +%dir %attr(-,%{_vespa_user},-) %{_prefix}/logs/vespa/search +%{_prefix}/man +%{_prefix}/sbin +%{_prefix}/share +%dir %attr(1777,-,-) %{_prefix}/tmp +%dir %attr(1777,%{_vespa_user},-) %{_prefix}/tmp/vespa +%dir %{_prefix}/var +%dir %{_prefix}/var/db +%dir %attr(-,%{_vespa_user},-) %{_prefix}/var/db/vespa +%dir %attr(-,%{_vespa_user},-) %{_prefix}/var/db/vespa/logcontrol +%dir %attr(-,%{_vespa_user},-) %{_prefix}/var/zookeeper %config(noreplace) %{_prefix}/conf/logd/logd.cfg %config(noreplace) %{_prefix}/conf/vespa/default-env.txt %config(noreplace) %{_prefix}/etc/vespamalloc.conf +%if %{_create_vespa_service} %attr(644,root,root) /usr/lib/systemd/system/vespa.service %attr(644,root,root) /usr/lib/systemd/system/vespa-configserver.service +%endif %changelog diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java index 32302a98757..ecfe9b2468a 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java @@ -34,6 +34,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { private final Map<String, String> labels = new HashMap<>(); private final List<String> environmentAssignments = new ArrayList<>(); private final List<String> volumeBindSpecs = new ArrayList<>(); + private final List<String> dnsOptions = new ArrayList<>(); private final List<Ulimit> ulimits = new ArrayList<>(); private final Set<Capability> addCapabilities = new HashSet<>(); private final Set<Capability> dropCapabilities = new HashSet<>(); @@ -96,6 +97,12 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { } @Override + public Docker.CreateContainerCommand withDnsOption(String dnsOption) { + dnsOptions.add(dnsOption); + return this; + } + + @Override public Docker.CreateContainerCommand withPrivileged(boolean privileged) { this.privileged = privileged; return this; @@ -171,6 +178,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { .withPidsLimit(-1L) .withCapAdd(addCapabilities.toArray(new Capability[0])) .withCapDrop(dropCapabilities.toArray(new Capability[0])) + .withDnsOptions(dnsOptions) .withPrivileged(privileged); containerResources.ifPresent(cr -> hostConfig @@ -241,6 +249,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { toRepeatedOption("--cap-add", addCapabilitiesList), toRepeatedOption("--cap-drop", dropCapabilitiesList), toRepeatedOption("--security-opt", securityOpts), + toRepeatedOption("--dns-option", dnsOptions), toOptionalOption("--net", networkMode), toOptionalOption("--ip", ipv4Address), toOptionalOption("--ip6", ipv6Address), diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java index 4e7ef5a1ff6..648c94d71ab 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java @@ -53,6 +53,7 @@ public interface Docker { CreateContainerCommand withAddCapability(String capabilityName); CreateContainerCommand withDropCapability(String capabilityName); CreateContainerCommand withSecurityOpts(String securityOpt); + CreateContainerCommand withDnsOption(String dnsOption); CreateContainerCommand withPrivileged(boolean privileged); void create(); diff --git a/eval/src/vespa/eval/eval/interpreted_function.cpp b/eval/src/vespa/eval/eval/interpreted_function.cpp index 0a630a3e20a..ec28604fd87 100644 --- a/eval/src/vespa/eval/eval/interpreted_function.cpp +++ b/eval/src/vespa/eval/eval/interpreted_function.cpp @@ -3,21 +3,16 @@ #include "interpreted_function.h" #include "node_visitor.h" #include "node_traverser.h" -#include "check_type.h" -#include "tensor_spec.h" -#include "operation.h" #include "tensor_nodes.h" #include "tensor_engine.h" +#include "make_tensor_function.h" +#include "compile_tensor_function.h" #include <vespa/vespalib/util/classname.h> #include <vespa/eval/eval/llvm/compile_cache.h> #include <vespa/vespalib/util/benchmark_timer.h> #include <set> -#include "make_tensor_function.h" -#include "compile_tensor_function.h" - -namespace vespalib { -namespace eval { +namespace vespalib::eval { namespace { @@ -42,11 +37,12 @@ InterpretedFunction::State::State(const TensorEngine &engine_in) params(nullptr), stash(), stack(), - program_offset(0) + program_offset(0), + if_cnt(0) { } -InterpretedFunction::State::~State() {} +InterpretedFunction::State::~State() = default; void InterpretedFunction::State::init(const LazyParams ¶ms_in) { @@ -82,7 +78,7 @@ InterpretedFunction::InterpretedFunction(const TensorEngine &engine, const nodes _program = compile_tensor_function(optimized, _stash); } -InterpretedFunction::~InterpretedFunction() {} +InterpretedFunction::~InterpretedFunction() = default; const Value & InterpretedFunction::eval(Context &ctx, const LazyParams ¶ms) const @@ -123,5 +119,4 @@ InterpretedFunction::detect_issues(const Function &function) return Function::Issues(std::move(checker.issues)); } -} // namespace vespalib::eval -} // namespace vespalib +} diff --git a/eval/src/vespa/eval/eval/interpreted_function.h b/eval/src/vespa/eval/eval/interpreted_function.h index e3e8d18b44f..e638ccffcea 100644 --- a/eval/src/vespa/eval/eval/interpreted_function.h +++ b/eval/src/vespa/eval/eval/interpreted_function.h @@ -7,8 +7,7 @@ #include "lazy_params.h" #include <vespa/vespalib/util/stash.h> -namespace vespalib { -namespace eval { +namespace vespalib::eval { namespace nodes { struct Node; } struct TensorEngine; @@ -107,5 +106,4 @@ public: static Function::Issues detect_issues(const Function &function); }; -} // namespace vespalib::eval -} // namespace vespalib +} diff --git a/eval/src/vespa/eval/eval/lazy_params.cpp b/eval/src/vespa/eval/eval/lazy_params.cpp index aec8cf78059..2c00c4c312b 100644 --- a/eval/src/vespa/eval/eval/lazy_params.cpp +++ b/eval/src/vespa/eval/eval/lazy_params.cpp @@ -1,19 +1,16 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "lazy_params.h" -#include <assert.h> +#include <vespa/vespalib/util/stash.h> +#include <cassert> namespace vespalib::eval { -LazyParams::~LazyParams() -{ -} +LazyParams::~LazyParams() = default; //----------------------------------------------------------------------------- -SimpleObjectParams::~SimpleObjectParams() -{ -} +SimpleObjectParams::~SimpleObjectParams() = default; const Value & SimpleObjectParams::resolve(size_t idx, Stash &) const @@ -24,9 +21,7 @@ SimpleObjectParams::resolve(size_t idx, Stash &) const //----------------------------------------------------------------------------- -SimpleParams::~SimpleParams() -{ -} +SimpleParams::~SimpleParams() = default; const Value & SimpleParams::resolve(size_t idx, Stash &stash) const diff --git a/eval/src/vespa/eval/eval/tensor_function.h b/eval/src/vespa/eval/eval/tensor_function.h index 4b862e9ec6a..55d27fb74ea 100644 --- a/eval/src/vespa/eval/eval/tensor_function.h +++ b/eval/src/vespa/eval/eval/tensor_function.h @@ -2,20 +2,17 @@ #pragma once -#include <memory> -#include <vector> -#include <variant> -#include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/util/arrayref.h> -#include <vespa/vespalib/util/overload.h> #include "tensor_spec.h" #include "lazy_params.h" #include "value_type.h" #include "value.h" #include "aggr.h" - #include "interpreted_function.h" +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/arrayref.h> +#include <vespa/vespalib/util/overload.h> +#include <variant> namespace vespalib { diff --git a/eval/src/vespa/eval/eval/value.h b/eval/src/vespa/eval/eval/value.h index 6701173bcd3..cad76c93c5c 100644 --- a/eval/src/vespa/eval/eval/value.h +++ b/eval/src/vespa/eval/eval/value.h @@ -2,13 +2,11 @@ #pragma once -#include <vespa/vespalib/stllike/string.h> -#include <memory> -#include <vespa/vespalib/util/stash.h> #include "value_type.h" +#include <vespa/vespalib/util/traits.h> +#include <memory> -namespace vespalib { -namespace eval { +namespace vespalib::eval { class Tensor; @@ -40,7 +38,6 @@ public: static const ValueType &double_type() { return _type; } }; -} // namespace vespalib::eval -} // namespace vespalib +} VESPA_CAN_SKIP_DESTRUCTION(::vespalib::eval::DoubleValue); diff --git a/fbench/CMakeLists.txt b/fbench/CMakeLists.txt index 5cc56786227..3da632d98a6 100644 --- a/fbench/CMakeLists.txt +++ b/fbench/CMakeLists.txt @@ -15,6 +15,7 @@ vespa_define_module( TESTS src/test + src/test/authority ) vespa_install_script(util/resultfilter.pl vespa-fbench-result-filter.pl bin) diff --git a/fbench/src/fbench/fbench.cpp b/fbench/src/fbench/fbench.cpp index 593ae30a0e5..c2996eebb5c 100644 --- a/fbench/src/fbench/fbench.cpp +++ b/fbench/src/fbench/fbench.cpp @@ -3,10 +3,10 @@ #include <httpclient/httpclient.h> #include <util/filereader.h> #include <util/clientstatus.h> +#include <vespa/vespalib/crypto/crypto_exception.h> #include <vespa/vespalib/net/crypto_engine.h> #include <vespa/vespalib/net/tls/transport_security_options.h> #include <vespa/vespalib/net/tls/tls_crypto_engine.h> -#include <vespa/vespalib/net/tls/crypto_exception.h> #include <vespa/vespalib/io/mapped_file_input.h> #include "client.h" #include "fbench.h" @@ -99,7 +99,7 @@ FBench::init_crypto_engine(const std::string &ca_certs_file_name, } try { _crypto_engine = std::make_shared<vespalib::TlsCryptoEngine>(tls_opts); - } catch (vespalib::net::tls::CryptoException &e) { + } catch (vespalib::crypto::CryptoException &e) { fprintf(stderr, "%s\n", e.what()); return false; } diff --git a/fbench/src/httpclient/CMakeLists.txt b/fbench/src/httpclient/CMakeLists.txt index 5f3333128b3..a28f3666383 100644 --- a/fbench/src/httpclient/CMakeLists.txt +++ b/fbench/src/httpclient/CMakeLists.txt @@ -3,5 +3,6 @@ vespa_add_library(fbench_httpclient STATIC SOURCES httpclient.cpp DEPENDS + fbench_util fastos ) diff --git a/fbench/src/httpclient/httpclient.cpp b/fbench/src/httpclient/httpclient.cpp index 99134a6e297..9615a6e6df7 100644 --- a/fbench/src/httpclient/httpclient.cpp +++ b/fbench/src/httpclient/httpclient.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "httpclient.h" #include <vespa/vespalib/net/socket_spec.h> +#include <util/authority.h> #include <cassert> #include <cstring> @@ -29,7 +30,8 @@ HTTPClient::HTTPClient(vespalib::CryptoEngine::SP engine, const char *hostname, _keepAlive(keepAlive), _headerBenchmarkdataCoverage(headerBenchmarkdataCoverage), _extraHeaders(extraHeaders), - _authority(authority), + _sni_spec(make_sni_spec(authority, hostname, port, _engine->use_tls_when_client())), + _host_header_value(make_host_header_value(_sni_spec, _engine->use_tls_when_client())), _reuseCount(0), _bufsize(10240), _buf(new char[_bufsize]), @@ -51,11 +53,6 @@ HTTPClient::HTTPClient(vespalib::CryptoEngine::SP engine, const char *hostname, _dataDone(false), _reader(NULL) { - if (_authority == "") { - char tmp[1024]; - snprintf(tmp, 1024, "%s:%d", hostname, port); - _authority = tmp; - } } bool @@ -70,8 +67,7 @@ HTTPClient::connect_socket() if (!handle.valid()) { return false; } - _socket = vespalib::SyncCryptoSocket::create_client(*_engine, std::move(handle), - vespalib::SocketSpec::from_host_port(_hostname, _port)); + _socket = vespalib::SyncCryptoSocket::create_client(*_engine, std::move(handle), _sni_spec); return bool(_socket); } @@ -153,14 +149,14 @@ HTTPClient::Connect(const char *url, bool usePost, const char *content, int cLen "Content-Length: %d\r\n" "%s" "\r\n", - url, _authority.c_str(), cLen, headers.c_str()); + url, _host_header_value.c_str(), cLen, headers.c_str()); } else { snprintf(req, req_max, "GET %s HTTP/1.1\r\n" "Host: %s\r\n" "%s" "\r\n", - url, _authority.c_str(), headers.c_str()); + url, _host_header_value.c_str(), headers.c_str()); } // try to reuse connection if keep-alive is enabled diff --git a/fbench/src/httpclient/httpclient.h b/fbench/src/httpclient/httpclient.h index 9c3ccd437d1..cad01826db7 100644 --- a/fbench/src/httpclient/httpclient.h +++ b/fbench/src/httpclient/httpclient.h @@ -6,6 +6,7 @@ #include <vespa/vespalib/net/sync_crypto_socket.h> #include <vespa/vespalib/net/crypto_engine.h> #include <vespa/vespalib/net/socket_address.h> +#include <vespa/vespalib/net/socket_spec.h> /** * This class implements a HTTP client that may be used to fetch @@ -99,7 +100,8 @@ protected: bool _keepAlive; bool _headerBenchmarkdataCoverage; std::string _extraHeaders; - std::string _authority; + vespalib::SocketSpec _sni_spec; + std::string _host_header_value; uint64_t _reuseCount; size_t _bufsize; diff --git a/fbench/src/test/authority/CMakeLists.txt b/fbench/src/test/authority/CMakeLists.txt new file mode 100644 index 00000000000..00f804f43f6 --- /dev/null +++ b/fbench/src/test/authority/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(fbench_authority_test_app TEST + SOURCES + authority_test.cpp + DEPENDS + fbench_util + vespalib + gtest +) +vespa_add_test(NAME fbench_authority_test_app COMMAND fbench_authority_test_app) diff --git a/fbench/src/test/authority/authority_test.cpp b/fbench/src/test/authority/authority_test.cpp new file mode 100644 index 00000000000..de723a8730f --- /dev/null +++ b/fbench/src/test/authority/authority_test.cpp @@ -0,0 +1,87 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <util/authority.h> +#include <vespa/vespalib/gtest/gtest.h> + +using vespalib::SocketSpec; + +//----------------------------------------------------------------------------- + +TEST(MakeSNISpecTest, host_port_is_parsed_as_expected) { + EXPECT_EQ(make_sni_spec("my_host:123", "fallback", 456, false).host(), "my_host"); + EXPECT_EQ(make_sni_spec("my_host:123", "fallback", 456, true).host(), "my_host"); + EXPECT_EQ(make_sni_spec("my_host:123", "fallback", 456, false).port(), 123); + EXPECT_EQ(make_sni_spec("my_host:123", "fallback", 456, true).port(), 123); +} + +TEST(MakeSNISpecTest, user_info_is_stripped) { + EXPECT_EQ(make_sni_spec("myuser:deprecated@my_host:123", "fallback", 456, false).host(), "my_host"); + EXPECT_EQ(make_sni_spec("myuser:deprecated@my_host:123", "fallback", 456, true).host(), "my_host"); + EXPECT_EQ(make_sni_spec("myuser:deprecated@my_host:123", "fallback", 456, false).port(), 123); + EXPECT_EQ(make_sni_spec("myuser:deprecated@my_host:123", "fallback", 456, true).port(), 123); +} + +TEST(MakeSNISpecTest, port_can_be_skipped) { + EXPECT_EQ(make_sni_spec("my_host", "fallback", 456, false).host(), "my_host"); + EXPECT_EQ(make_sni_spec("my_host", "fallback", 456, true).host(), "my_host"); + EXPECT_EQ(make_sni_spec("my_host", "fallback", 456, false).port(), 80); + EXPECT_EQ(make_sni_spec("my_host", "fallback", 456, true).port(), 443); +} + +TEST(MakeSNISpecTest, quoted_ip_addresses_work_as_expected) { + EXPECT_EQ(make_sni_spec("[::1]:123", "fallback", 456, false).host(), "::1"); + EXPECT_EQ(make_sni_spec("[::1]:123", "fallback", 456, true).host(), "::1"); + EXPECT_EQ(make_sni_spec("[::1]:123", "fallback", 456, false).port(), 123); + EXPECT_EQ(make_sni_spec("[::1]:123", "fallback", 456, true).port(), 123); + EXPECT_EQ(make_sni_spec("[::1]", "fallback", 456, false).host(), "::1"); + EXPECT_EQ(make_sni_spec("[::1]", "fallback", 456, true).host(), "::1"); + EXPECT_EQ(make_sni_spec("[::1]", "fallback", 456, false).port(), 80); + EXPECT_EQ(make_sni_spec("[::1]", "fallback", 456, true).port(), 443); +} + +TEST(MakeSNISpecTest, supplied_host_port_is_used_as_fallback) { + EXPECT_EQ(make_sni_spec("", "fallback", 456, false).host(), "fallback"); + EXPECT_EQ(make_sni_spec("", "fallback", 456, true).host(), "fallback"); + EXPECT_EQ(make_sni_spec("", "fallback", 456, false).port(), 456); + EXPECT_EQ(make_sni_spec("", "fallback", 456, true).port(), 456); +} + +//----------------------------------------------------------------------------- + +TEST(MakeHostHeaderValueTest, host_port_is_formatted_as_expected) { + auto my_spec = SocketSpec::from_host_port("myhost", 123); + EXPECT_EQ(make_host_header_value(my_spec, false), "myhost:123"); + EXPECT_EQ(make_host_header_value(my_spec, true), "myhost:123"); +} + +TEST(MakeHostHeaderValueTest, inappropriate_spec_gives_empty_host_value) { + std::vector<SocketSpec> bad_specs = { + SocketSpec::invalid, + SocketSpec::from_port(123), + SocketSpec::from_name("foo"), + SocketSpec::from_path("bar") + }; + for (const auto &spec: bad_specs) { + EXPECT_EQ(make_host_header_value(spec, false), ""); + EXPECT_EQ(make_host_header_value(spec, true), ""); + } +} + +TEST(MakeHostHeaderValueTest, default_port_is_omitted) { + auto spec1 = SocketSpec::from_host_port("myhost", 80); + auto spec2 = SocketSpec::from_host_port("myhost", 443); + EXPECT_EQ(make_host_header_value(spec1, false), "myhost"); + EXPECT_EQ(make_host_header_value(spec1, true), "myhost:80"); + EXPECT_EQ(make_host_header_value(spec2, false), "myhost:443"); + EXPECT_EQ(make_host_header_value(spec2, true), "myhost"); +} + +TEST(MakeHostHeaderValueTest, ipv6_addresses_are_quoted) { + auto my_spec = SocketSpec::from_host_port("::1", 123); + EXPECT_EQ(make_host_header_value(my_spec, false), "[::1]:123"); + EXPECT_EQ(make_host_header_value(my_spec, true), "[::1]:123"); +} + +//----------------------------------------------------------------------------- + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/fbench/src/util/CMakeLists.txt b/fbench/src/util/CMakeLists.txt index 47cc46ffc8f..3cdff26ce16 100644 --- a/fbench/src/util/CMakeLists.txt +++ b/fbench/src/util/CMakeLists.txt @@ -1,8 +1,9 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(fbench_util STATIC SOURCES + authority.cpp + clientstatus.cpp filereader.cpp timer.cpp - clientstatus.cpp DEPENDS ) diff --git a/fbench/src/util/authority.cpp b/fbench/src/util/authority.cpp new file mode 100644 index 00000000000..6247c72d9b0 --- /dev/null +++ b/fbench/src/util/authority.cpp @@ -0,0 +1,42 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "authority.h" +#include <vespa/vespalib/util/stringfmt.h> +#include <cassert> + +namespace { + +int default_port(bool use_https) { return use_https ? 443 : 80; } + +} + +vespalib::SocketSpec make_sni_spec(const std::string &authority, const char *hostname, int port, bool use_https) { + if (authority.empty()) { + return vespalib::SocketSpec::from_host_port(hostname, port); + } + auto split = authority.rfind('@'); + std::string spec_str = (split == std::string::npos) ? authority : authority.substr(split + 1); + auto a = spec_str.rfind(':'); + auto b = spec_str.rfind(']'); + bool has_port = (a != std::string::npos) && ((b == std::string::npos) || (a > b)); + if (has_port) { + spec_str = "tcp/" + spec_str; + } else { + spec_str = vespalib::make_string("tcp/%s:%d", spec_str.c_str(), default_port(use_https)); + } + // use SocketSpec parser to ensure ipv6 addresses are dequoted + return vespalib::SocketSpec(spec_str); +} + +std::string make_host_header_value(const vespalib::SocketSpec &sni_spec, bool use_https) { + if (sni_spec.host().empty()) { + return ""; + } + if (sni_spec.port() == default_port(use_https)) { + return sni_spec.host(); + } + // use SocketSpec formatter to ensure ipv6 addresses are quoted + std::string spec_str = sni_spec.spec(); + assert(spec_str.find("tcp/") == 0); + return spec_str.substr(4); +} diff --git a/fbench/src/util/authority.h b/fbench/src/util/authority.h new file mode 100644 index 00000000000..49dab4a29fd --- /dev/null +++ b/fbench/src/util/authority.h @@ -0,0 +1,30 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/net/socket_spec.h> + +/** + * Assemble an SNI (Server Name Indication) spec that will be used + * when handshaking over TLS. The authority will be used if + * non-empty. Hostname/port will be used as fall-back. Note that the + * SNI spec will also be used to generate the Host header used in + * subsequent HTTP requests. + * + * @return sni spec + * @param authority user-provided authority + * @param hostname name of the host we are connecting to + * @param port which port we are connecting to + * @param use_https are we using https? (TLS) + **/ +vespalib::SocketSpec make_sni_spec(const std::string &authority, const char *hostname, int port, bool use_https); + +/** + * Use an SNI spec to generate a matching Host header to be used in + * HTTP requests. Note that default port numbers will be omitted. + * + * @return host header value + * @param sni_spec SNI spec + * @param use_https are we using https? (TLS) + **/ +std::string make_host_header_value(const vespalib::SocketSpec &sni_spec, bool use_https); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index bb4e281a85f..815fcda6ee7 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -67,6 +67,12 @@ public class Flags { "Takes effect on next host admin tick.", HOSTNAME); + public static final UnboundBooleanFlag SERVICE_MODEL_CACHE = defineFeatureFlag( + "service-model-cache", true, + "Whether the service model is cached.", + "Takes effect on restart of config server.", + HOSTNAME); + public static final UnboundListFlag<String> DISABLED_HOST_ADMIN_TASKS = defineListFlag( "disabled-host-admin-tasks", List.of(), String.class, "List of host-admin task names (as they appear in the log, e.g. root>main>UpgradeTask) that should be skipped", @@ -204,6 +210,11 @@ public class Flags { "Takes effect on the next deployment of the application", APPLICATION_ID); + public static final UnboundBooleanFlag VALIDATE_ENDPOINT_CERTIFICATES = defineFeatureFlag( + "validate-endpoint-certificates", false, + "Whether endpoint certificates should be validated before use", + "Takes effect on the next deployment of the application"); + public static final UnboundStringFlag ENDPOINT_CERTIFICATE_BACKFILL = defineStringFlag( "endpoint-certificate-backfill", "disable", "Whether the endpoint certificate maintainer should backfill missing certificate data from cameo", diff --git a/fnet/src/tests/connect/connect_test.cpp b/fnet/src/tests/connect/connect_test.cpp index d94b6759077..62000efb682 100644 --- a/fnet/src/tests/connect/connect_test.cpp +++ b/fnet/src/tests/connect/connect_test.cpp @@ -65,6 +65,8 @@ struct BlockingCryptoEngine : public CryptoEngine { Gate handshake_work_enter; Gate handshake_work_exit; Gate handshake_socket_deleted; + bool use_tls_when_client() const override { return false; } + bool always_use_tls_when_server() const override { return false; } CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &) override { return std::make_unique<BlockingCryptoSocket>(std::move(socket), handshake_work_enter, handshake_work_exit, handshake_socket_deleted); diff --git a/fnet/src/tests/frt/values/values_test.cpp b/fnet/src/tests/frt/values/values_test.cpp index 5bc6b0e2dce..3b36e8989c1 100644 --- a/fnet/src/tests/frt/values/values_test.cpp +++ b/fnet/src/tests/frt/values/values_test.cpp @@ -3,6 +3,7 @@ #include <vespa/fnet/frt/values.h> #include <vespa/fnet/databuffer.h> #include <vespa/fnet/info.h> +#include <vespa/vespalib/util/stash.h> using vespalib::Stash; diff --git a/fnet/src/vespa/fnet/frt/rpcrequest.h b/fnet/src/vespa/fnet/frt/rpcrequest.h index cc871e7ac0c..eaa34a46b7a 100644 --- a/fnet/src/vespa/fnet/frt/rpcrequest.h +++ b/fnet/src/vespa/fnet/frt/rpcrequest.h @@ -5,6 +5,7 @@ #include "values.h" #include "error.h" #include <vespa/fnet/context.h> +#include <vespa/vespalib/util/stash.h> #include <atomic> class FNETConnection; diff --git a/fnet/src/vespa/fnet/frt/values.cpp b/fnet/src/vespa/fnet/frt/values.cpp index a5f59df19b2..3b37aa9a1bc 100644 --- a/fnet/src/vespa/fnet/frt/values.cpp +++ b/fnet/src/vespa/fnet/frt/values.cpp @@ -3,6 +3,7 @@ #include "values.h" #include <vespa/fnet/databuffer.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> #include <cassert> static_assert(sizeof(uint8_t) == 1, "uint8_t must be 1 byte."); @@ -81,7 +82,7 @@ FRT_Values::FRT_Values(Stash &stash) _stash(stash) { } -FRT_Values::~FRT_Values() { } +FRT_Values::~FRT_Values() = default; LocalBlob::LocalBlob(const char *data, uint32_t len) : _data(Alloc::alloc(len)), @@ -294,7 +295,7 @@ FRT_Values::AddSharedData(FRT_ISharedBlob *blob) { } void -FRT_Values::AddData(vespalib::alloc::Alloc buf, uint32_t len) { +FRT_Values::AddData(vespalib::alloc::Alloc && buf, uint32_t len) { AddSharedData(&_stash.create<LocalBlob>(std::move(buf), len)); } diff --git a/fnet/src/vespa/fnet/frt/values.h b/fnet/src/vespa/fnet/frt/values.h index e00aec8423c..2aa7551c423 100644 --- a/fnet/src/vespa/fnet/frt/values.h +++ b/fnet/src/vespa/fnet/frt/values.h @@ -3,9 +3,10 @@ #pragma once #include "isharedblob.h" -#include <vespa/vespalib/util/stash.h> #include <cstring> +namespace vespalib { class Stash; } +namespace vespalib::alloc { class Alloc; } namespace fnet { char * copyString(char *dst, const char *src, size_t len); char * copyData(char *dst, const void *src, size_t len); @@ -216,7 +217,7 @@ public: char *AddString(uint32_t len); FRT_StringValue *AddStringArray(uint32_t len); void AddSharedData(FRT_ISharedBlob *blob); - void AddData(Alloc buf, uint32_t len); + void AddData(Alloc && buf, uint32_t len); void AddData(const char *buf, uint32_t len); char *AddData(uint32_t len); FRT_DataValue *AddDataArray(uint32_t len); diff --git a/jrt/src/com/yahoo/jrt/Connector.java b/jrt/src/com/yahoo/jrt/Connector.java index 57fad5a163d..10f2b3742f2 100644 --- a/jrt/src/com/yahoo/jrt/Connector.java +++ b/jrt/src/com/yahoo/jrt/Connector.java @@ -1,44 +1,67 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jrt; -import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.concurrent.CachedThreadPoolWithFallback; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.Executor; import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; -class Connector { +class Connector implements AutoCloseable { - private final ExecutorService executor = new ThreadPoolExecutor(1, 64, 1L, TimeUnit.SECONDS, - new LinkedBlockingQueue<>(), - ThreadFactoryFactory.getDaemonThreadFactory("jrt.connector")); + private static final Object globalLock = new Object(); + private static CachedThreadPoolWithFallback globalExecutor = null; + private static long usages = 0; - private void connect(Connection conn) { - conn.transportThread().addConnection(conn.connect()); + private static CachedThreadPoolWithFallback acquire() { + synchronized (globalLock) { + if (globalExecutor == null) { + globalExecutor = new CachedThreadPoolWithFallback("jrt.connector", 1, 64, 1L, TimeUnit.SECONDS); + } + usages++; + return globalExecutor; + } } - public void connectLater(Connection conn) { - try { - executor.execute(() -> connect(conn)); - } catch (RejectedExecutionException e) { - conn.transportThread().addConnection(conn); + private static void release(CachedThreadPoolWithFallback executor) { + synchronized (globalLock) { + assert executor == globalExecutor; + usages--; + if (usages == 0) { + globalExecutor.close(); + globalExecutor = null; + } } } - public Connector shutdown() { - executor.shutdown(); - return this; + private final AtomicReference<CachedThreadPoolWithFallback> executor; + + Connector() { + executor = new AtomicReference<>(acquire()); } - public void join() { - while (true) { + private void connect(Connection conn) { + conn.transportThread().addConnection(conn.connect()); + } + + public void connectLater(Connection conn) { + Executor executor = this.executor.get(); + if (executor != null) { try { - if (executor.awaitTermination(60, TimeUnit.SECONDS)) { - return; - } - } catch (InterruptedException e) {} + executor.execute(() -> connect(conn)); + return; + } catch (RejectedExecutionException ignored) { + } + } + conn.transportThread().addConnection(conn); + } + + @Override + public void close() { + CachedThreadPoolWithFallback toShutdown = executor.getAndSet(null); + if (toShutdown != null) { + release(toShutdown); } } } diff --git a/jrt/src/com/yahoo/jrt/Supervisor.java b/jrt/src/com/yahoo/jrt/Supervisor.java index 09360c2da7b..d4168e97743 100644 --- a/jrt/src/com/yahoo/jrt/Supervisor.java +++ b/jrt/src/com/yahoo/jrt/Supervisor.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jrt; - import java.util.HashMap; import java.util.concurrent.atomic.AtomicReference; @@ -103,19 +102,6 @@ public class Supervisor { } /** - * Remove a method from the set of methods held by this Supervisor - * - * @param methodName name of the method to remove - **/ - public void removeMethod(String methodName) { - synchronized (methodMapLock) { - HashMap<String, Method> newMap = new HashMap<>(methodMap()); - newMap.remove(methodName); - methodMap.setRelease(newMap); - } - } - - /** * Remove a method from the set of methods held by this * Supervisor. Use this if you know exactly which method to remove * and not only the name. diff --git a/jrt/src/com/yahoo/jrt/Transport.java b/jrt/src/com/yahoo/jrt/Transport.java index 6f5a381fd6b..8abd3942a39 100644 --- a/jrt/src/com/yahoo/jrt/Transport.java +++ b/jrt/src/com/yahoo/jrt/Transport.java @@ -4,7 +4,6 @@ package com.yahoo.jrt; import java.nio.channels.SocketChannel; import java.util.ArrayList; -import java.util.Iterator; import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; @@ -19,7 +18,7 @@ import java.util.logging.Logger; **/ public class Transport { - private static Logger log = Logger.getLogger(Transport.class.getName()); + private static final Logger log = Logger.getLogger(Transport.class.getName()); private final FatalErrorHandler fatalHandler; // NB: this must be set first private final CryptoEngine cryptoEngine; @@ -28,7 +27,7 @@ public class Transport { private final AtomicInteger runCnt; private final TransportMetrics metrics = TransportMetrics.getInstance(); - private final ArrayList<TransportThread> threads = new ArrayList<TransportThread>(); + private final ArrayList<TransportThread> threads = new ArrayList<>(); private final Random rnd = new Random(); /** @@ -174,7 +173,7 @@ public class Transport { * @return this object, to enable chaining with join **/ public Transport shutdown() { - connector.shutdown().join(); + connector.close(); for (TransportThread thread: threads) { thread.shutdown(); } diff --git a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp index bf98bbd75ef..e37a6fb2dcb 100644 --- a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp +++ b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp @@ -80,7 +80,9 @@ CfHandler::doConfigure() if (getenv("VESPA_HOSTNAME") != NULL && getenv("VESPA_TENANT") != NULL && getenv("VESPA_APPLICATION")!= NULL && - getenv("VESPA_INSTANCE") != NULL ) + getenv("VESPA_INSTANCE") != NULL && + getenv("VESPA_ENVIRONMENT") != NULL && + getenv("VESPA_REGION") != NULL) { path = cfFilePath(config.splunkHome, "inputs.conf"); tmpPath = path + ".new"; @@ -88,7 +90,7 @@ CfHandler::doConfigure() if (fp != NULL) { fprintf(fp, "[default]\n"); fprintf(fp, "host = %s\n", getenv("VESPA_HOSTNAME")); - fprintf(fp, "_meta = vespa_tenant::%s vespa_app::%s.%s\n", getenv("VESPA_TENANT"), getenv("VESPA_APPLICATION"), getenv("VESPA_INSTANCE")); + fprintf(fp, "_meta = vespa_tenant::%s vespa_app::%s.%s vespa_zone::%s.%s\n", getenv("VESPA_TENANT"), getenv("VESPA_APPLICATION"), getenv("VESPA_INSTANCE"), getenv("VESPA_ENVIRONMENT"), getenv("VESPA_REGION")); fclose(fp); rename(tmpPath.c_str(), path.c_str()); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java index 42c8a13e626..2afc0267434 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java @@ -6,6 +6,7 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.log.LogLevel; import com.yahoo.system.execution.ProcessExecutor; import com.yahoo.system.execution.ProcessResult; +import com.yahoo.vespa.defaults.Defaults; import org.apache.velocity.VelocityContext; import org.apache.velocity.app.VelocityEngine; @@ -24,6 +25,7 @@ public class Telegraf extends AbstractComponent { private static final String TELEGRAF_CONFIG_PATH = "/etc/telegraf/telegraf.conf"; private static final String TELEGRAF_CONFIG_TEMPLATE_PATH = "templates/telegraf.conf.vm"; + private static final String TELEGRAF_LOG_FILE_PATH = Defaults.getDefaults().underVespaHome("logs/telegraf/telegraf.log"); private final TelegrafRegistry telegrafRegistry; private static final Logger logger = Logger.getLogger(Telegraf.class.getName()); @@ -38,6 +40,7 @@ public class Telegraf extends AbstractComponent { protected static void writeConfig(TelegrafConfig telegrafConfig, Writer writer) { VelocityContext context = new VelocityContext(); + context.put("logFilePath", TELEGRAF_LOG_FILE_PATH); context.put("intervalSeconds", telegrafConfig.intervalSeconds()); context.put("cloudwatchPlugins", telegrafConfig.cloudWatch()); // TODO: Add node cert if hosted diff --git a/metrics-proxy/src/main/resources/templates/telegraf.conf.vm b/metrics-proxy/src/main/resources/templates/telegraf.conf.vm index c427ee1ce4b..e99bab8b02d 100644 --- a/metrics-proxy/src/main/resources/templates/telegraf.conf.vm +++ b/metrics-proxy/src/main/resources/templates/telegraf.conf.vm @@ -9,7 +9,7 @@ flush_jitter = "0s" precision = "" logtarget = "file" - logfile = "/var/log/telegraf/telegraf.log" + logfile = "$logFilePath" logfile_rotation_interval = "1d" logfile_rotation_max_size = "20MB" logfile_rotation_max_archives = 5 diff --git a/metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt b/metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt index 85656465901..accd2cc87eb 100644 --- a/metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt +++ b/metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt @@ -9,7 +9,7 @@ flush_jitter = "0s" precision = "" logtarget = "file" - logfile = "/var/log/telegraf/telegraf.log" + logfile = "/opt/vespa/logs/telegraf/telegraf.log" logfile_rotation_interval = "1d" logfile_rotation_max_size = "20MB" logfile_rotation_max_archives = 5 diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index c790e73037e..782f8592350 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -66,6 +66,9 @@ public class DockerOperationsImpl implements DockerOperations { .withHostName(context.node().hostname()) .withResources(containerResources) .withManagedBy(MANAGER_NAME) + // The inet6 option is needed to prefer AAAA records with gethostbyname(3), used by (at least) a yca package + // TODO: Try to remove this + .withDnsOption("inet6") .withUlimit("nofile", 262_144, 262_144) // The nproc aka RLIMIT_NPROC resource limit works as follows: // - A process has a (soft) nproc limit, either inherited by the parent or changed with setrlimit(2). diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java index e2ad9e3de97..69bc9f5e092 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java @@ -181,6 +181,11 @@ public class DockerMock implements Docker { } @Override + public CreateContainerCommand withDnsOption(String dnsOption) { + return this; + } + + @Override public CreateContainerCommand withPrivileged(boolean privileged) { return this; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 7c0e0e7868b..efb2a71264a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -430,7 +430,11 @@ public final class Node { /** Returns whether this is a state where the node is assigned to an application */ public boolean isAllocated() { - return this == reserved || this == active || this == inactive || this == failed || this == parked; + return allocatedStates().contains(this); + } + + public static Set<State> allocatedStates() { + return Set.of(reserved, active, inactive, failed, parked); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java index 3392569d1f2..4d04409aaf0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java @@ -25,15 +25,20 @@ public class InfrastructureProvisioner extends Maintainer { this.infraDeployer = infraDeployer; } + public void maintainButThrowOnException() { + try { + infraDeployer.activateAllSupportedInfraApplications(true); + } catch (RuntimeException e) { + logger.log(LogLevel.INFO, "Failed to deploy supported infrastructure applications, " + + "will sleep 30s before propagating failure, to allow inspection of zk", + e.getMessage()); + try { Thread.sleep(30_000); } catch (InterruptedException ignored) { } + throw e; + } + } + @Override protected void maintain() { - infraDeployer.getSupportedInfraDeployments().forEach((application, deployment) -> { - try { - deployment.activate(); - } catch (RuntimeException e) { - logger.log(LogLevel.INFO, "Failed to activate " + application, e); - // loop around to activate the next application - } - }); + infraDeployer.activateAllSupportedInfraApplications(false); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 063b5ad2c2a..37620e17a95 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -87,7 +87,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { rebalancer = new Rebalancer(deployer, nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), provisionServiceProvider.getHostProvisioner(), metric, clock, defaults.rebalancerInterval); // The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now - infrastructureProvisioner.maintain(); + infrastructureProvisioner.maintainButThrowOnException(); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java index a56ace07c82..1086a3a7cd9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java @@ -20,7 +20,6 @@ import com.yahoo.vespa.service.monitor.DuperModelInfraApi; import com.yahoo.vespa.service.monitor.InfraApplicationApi; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -51,9 +50,22 @@ public class InfraDeployerImpl implements InfraDeployer { } @Override - public Map<ApplicationId, Deployment> getSupportedInfraDeployments() { - return duperModel.getSupportedInfraApplications().stream() - .collect(Collectors.toMap(InfraApplicationApi::getApplicationId, InfraDeployment::new)); + public void activateAllSupportedInfraApplications(boolean propagateException) { + duperModel.getSupportedInfraApplications().forEach(api -> { + var application = api.getApplicationId(); + var deployment = new InfraDeployment(api); + try { + deployment.activate(); + } catch (RuntimeException e) { + logger.log(LogLevel.INFO, "Failed to activate " + application, e); + if (propagateException) { + throw e; + } + // loop around to activate the next application + } + }); + + duperModel.infraApplicationsIsNowComplete(); } private class InfraDeployment implements Deployment { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index 72be68a7ee3..91c15cdb61b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -121,8 +121,7 @@ class Preparer { */ private int findHighestIndex(ApplicationId application, ClusterSpec cluster) { int highestIndex = -1; - for (Node node : nodeRepository.getNodes(application, - Node.State.active, Node.State.inactive, Node.State.parked, Node.State.failed)) { + for (Node node : nodeRepository.getNodes(application, Node.State.allocatedStates().toArray(new Node.State[0]))) { ClusterSpec nodeCluster = node.allocation().get().membership().cluster(); if ( ! nodeCluster.id().equals(cluster.id())) continue; if ( ! nodeCluster.type().equals(cluster.type())) continue; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java index 62e17ab63ad..915ef0d9125 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java @@ -49,4 +49,8 @@ public class MockDuperModel implements DuperModelInfraApi { public void infraApplicationRemoved(ApplicationId applicationId) { activeApps.remove(applicationId); } + + @Override + public void infraApplicationsIsNowComplete() { + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java index e7bf76986ca..742a863fb38 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Deployment; import com.yahoo.config.provision.InfraDeployer; -import java.util.Map; import java.util.Optional; public class MockInfraDeployer implements InfraDeployer { @@ -15,7 +14,5 @@ public class MockInfraDeployer implements InfraDeployer { } @Override - public Map<ApplicationId, Deployment> getSupportedInfraDeployments() { - return Map.of(); - } + public void activateAllSupportedInfraApplications(boolean propagateException) { } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java index 2b770625060..677aaf93336 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java @@ -6,10 +6,13 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; @@ -18,11 +21,15 @@ import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import org.junit.Test; +import java.time.Instant; import java.util.List; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.IntStream; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -103,6 +110,41 @@ public class DynamicDockerProvisionTest { assertEquals(4, tester.nodeRepository().getNodes(NodeType.tenant, Node.State.reserved).size()); } + @Test + public void node_indices_are_unique_even_when_a_node_is_left_in_reserved_state() { + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); + NodeResources resources = new NodeResources(10, 10, 10, 10); + ApplicationId app = tester.makeApplicationId(); + + Function<Node, Node> retireNode = node -> + tester.nodeRepository().write(node.withWantToRetire(true, Agent.system, Instant.now()), () -> {}); + Function<Integer, Node> getNodeInGroup = group -> tester.nodeRepository().getNodes(app).stream() + .filter(node -> node.allocation().get().membership().cluster().group().get().index() == group) + .findAny().orElseThrow(); + + // Allocate 10 hosts + tester.makeReadyNodes(10, resources, NodeType.host, 1); + tester.deployZoneApp(); + + // Prepare & activate an application with 8 nodes and 2 groups + tester.activate(app, tester.prepare(app, clusterSpec("content"), 8, 2, resources)); + + // Retire a node in group 1 and prepare the application + retireNode.apply(getNodeInGroup.apply(1)); + tester.prepare(app, clusterSpec("content"), 8, 2, resources); + // App is not activated, to leave node '8' in reserved state + + // Retire a node in group 0 and prepare the application + retireNode.apply(getNodeInGroup.apply(0)); + tester.prepare(app, clusterSpec("content"), 8, 2, resources); + + // Verify that nodes have unique indices from 0..9 + var indices = tester.nodeRepository().getNodes(app).stream() + .map(node -> node.allocation().get().membership().index()) + .collect(Collectors.toSet()); + assertTrue(indices.containsAll(IntStream.range(0, 10).boxed().collect(Collectors.toList()))); + } + private static void deployZoneApp(ProvisioningTester tester) { ApplicationId applicationId = tester.makeApplicationId(); List<HostSpec> list = tester.prepare(applicationId, diff --git a/orchestrator/pom.xml b/orchestrator/pom.xml index 5dd4e7ea87d..fed0e617c79 100644 --- a/orchestrator/pom.xml +++ b/orchestrator/pom.xml @@ -81,12 +81,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.easytesting</groupId> - <artifactId>fest-assert</artifactId> - <version>1.4</version> - <scope>test</scope> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>config-provisioning</artifactId> <version>${project.version}</version> diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java index e9bb4984c2e..65c45c8df76 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java @@ -26,7 +26,6 @@ public interface ClusterApi { Optional<StorageNode> storageNodeInGroup(); Optional<StorageNode> upStorageNodeInGroup(); - String servicesDownAndNotInGroupDescription(); - String nodesAllowedToBeDownNotInGroupDescription(); + String downDescription(); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java index b747d8c2e22..24f56eac85d 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java @@ -131,25 +131,56 @@ class ClusterApiImpl implements ClusterApi { return numberOfServicesDown * 100 / (serviceCluster.serviceInstances().size() + missingServices); } + /** + * A description of the hosts outside the group that are allowed to be down, + * and a description of the services outside the group and outside of the allowed services + * that are down. + */ @Override - public String servicesDownAndNotInGroupDescription() { - // Sort these for readability and testing stability - return Stream - .concat(servicesDownAndNotInGroup.stream().map(ServiceInstance::toString).sorted(), - missingServices > 0 ? Stream.of(descriptionOfMissingServices) : Stream.of()) - .collect(Collectors.toList()) - .toString(); - } + public String downDescription() { + StringBuilder description = new StringBuilder(); - @Override - public String nodesAllowedToBeDownNotInGroupDescription() { - return servicesNotInGroup.stream() + Set<HostName> suspended = servicesNotInGroup.stream() .map(ServiceInstance::hostName) .filter(hostName -> hostStatus(hostName).isSuspended()) - .sorted() - .distinct() - .collect(Collectors.toList()) - .toString(); + .collect(Collectors.toSet()); + + if (suspended.size() > 0) { + description.append(" "); + + final int nodeLimit = 3; + description.append("Suspended hosts: "); + description.append(suspended.stream().sorted().distinct().limit(nodeLimit).collect(Collectors.toList()).toString()); + if (suspended.size() > nodeLimit) { + description.append(", and " + (suspended.size() - nodeLimit) + " more"); + } + description.append("."); + } + + Set<ServiceInstance> downElsewhere = servicesDownAndNotInGroup.stream() + .filter(serviceInstance -> !suspended.contains(serviceInstance.hostName())) + .collect(Collectors.toSet()); + + final int downElsewhereTotal = downElsewhere.size() + missingServices; + if (downElsewhereTotal > 0) { + description.append(" "); + + final int serviceLimit = 2; // services info is verbose + description.append("Services down on resumed hosts: "); + description.append(Stream.concat( + downElsewhere.stream().map(ServiceInstance::toString).sorted(), + missingServices > 0 ? Stream.of(descriptionOfMissingServices) : Stream.of()) + .limit(serviceLimit) + .collect(Collectors.toList()) + .toString()); + + if (downElsewhereTotal > serviceLimit) { + description.append(", and " + (downElsewhereTotal - serviceLimit) + " more"); + } + description.append("."); + } + + return description.toString(); } private Optional<StorageNode> storageNodeInGroup(Predicate<ServiceInstance> storageServicePredicate) { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java index e13cf17d420..a6b3cbc87dc 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java @@ -2,55 +2,43 @@ package com.yahoo.vespa.orchestrator.policy; import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.orchestrator.OrchestrationException; import com.yahoo.vespa.orchestrator.model.NodeGroup; -import java.util.Optional; - /** * @author bakksjo */ public class HostStateChangeDeniedException extends OrchestrationException { private final String constraintName; - private final Optional<ServiceType> serviceType; public HostStateChangeDeniedException(HostName hostName, String constraintName, String message) { this(hostName, constraintName, message, null); } public HostStateChangeDeniedException(HostName hostName, String constraintName, String message, Exception e) { - this(hostName.s(), constraintName, Optional.empty(), message, e); + this(hostName.s(), constraintName, message, e); } public HostStateChangeDeniedException(NodeGroup nodeGroup, String constraintName, String message) { - this(nodeGroup.toCommaSeparatedString(), constraintName, Optional.empty(), message, null); + this(nodeGroup.toCommaSeparatedString(), constraintName, message, null); } private HostStateChangeDeniedException(String nodes, String constraintName, - Optional<ServiceType> serviceType, String message, Throwable cause) { - super(createMessage(nodes, constraintName, serviceType, message), cause); + super(createMessage(nodes, constraintName, message), cause); this.constraintName = constraintName; - this.serviceType = serviceType; } private static String createMessage(String nodes, String constraintName, - Optional<ServiceType> serviceType, String message) { return "Changing the state of " + nodes + " would violate " + constraintName - + (serviceType.isPresent() ? " for service type " + serviceType.get() : "") + ": " + message; } public String getConstraintName() { return constraintName; } - - public Optional<ServiceType> getServiceType() { - return serviceType; - } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java index 1e895d0e757..ccb0bb57186 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java @@ -26,13 +26,11 @@ public class HostedVespaClusterPolicy implements ClusterPolicy { throw new HostStateChangeDeniedException( clusterApi.getNodeGroup(), ENOUGH_SERVICES_UP_CONSTRAINT, - "Suspension percentage for service type " + clusterApi.serviceType() + "Suspension for service type " + clusterApi.serviceType() + " would increase from " + clusterApi.percentageOfServicesDown() + "% to " + clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() + "%, over the limit of " + percentageOfServicesAllowedToBeDown + "%." - + " These instances may be down: " + clusterApi.servicesDownAndNotInGroupDescription() - + " and these hosts are allowed to be down: " - + clusterApi.nodesAllowedToBeDownNotInGroupDescription()); + + clusterApi.downDescription()); } @Override @@ -56,9 +54,7 @@ public class HostedVespaClusterPolicy implements ClusterPolicy { "Down percentage for service type " + clusterApi.serviceType() + " would increase to " + clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() + "%, over the limit of " + percentageOfServicesAllowedToBeDown + "%." - + " These instances may be down: " + clusterApi.servicesDownAndNotInGroupDescription() - + " and these hosts are allowed to be down: " - + clusterApi.nodesAllowedToBeDownNotInGroupDescription()); + + clusterApi.downDescription()); } // Non-private for testing purposes diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java index 62925dc003e..a5cb5cfa630 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; -import java.util.HashMap; import java.util.List; import java.util.Optional; import java.util.Set; @@ -81,15 +80,10 @@ public class ClusterApiImplTest { assertEquals("{ clusterId=cluster, serviceType=service-type }", clusterApi.clusterInfo()); assertFalse(clusterApi.isStorageCluster()); - assertEquals("[ServiceInstance{configId=service-2, hostName=host2, serviceStatus=" + - "ServiceStatusInfo{status=DOWN, since=Optional.empty, lastChecked=Optional.empty}}, " - + "ServiceInstance{configId=service-3, hostName=host3, serviceStatus=" + - "ServiceStatusInfo{status=UP, since=Optional.empty, lastChecked=Optional.empty}}, " - + "ServiceInstance{configId=service-4, hostName=host4, serviceStatus=" + - "ServiceStatusInfo{status=DOWN, since=Optional.empty, lastChecked=Optional.empty}}]", - clusterApi.servicesDownAndNotInGroupDescription()); - assertEquals("[host3, host4]", - clusterApi.nodesAllowedToBeDownNotInGroupDescription()); + assertEquals(" Suspended hosts: [host3, host4]. Services down on resumed hosts: [" + + "ServiceInstance{configId=service-2, hostName=host2, serviceStatus=" + + "ServiceStatusInfo{status=DOWN, since=Optional.empty, lastChecked=Optional.empty}}].", + clusterApi.downDescription()); assertEquals(60, clusterApi.percentageOfServicesDown()); assertEquals(80, clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()); } @@ -110,9 +104,9 @@ public class ClusterApiImplTest { fail(); } catch (HostStateChangeDeniedException e) { assertThat(e.getMessage(), - containsString("Changing the state of cfg1 would violate enough-services-up: Suspension percentage " + - "for service type configserver would increase from 33% to 66%, over the limit of 10%. " + - "These instances may be down: [1 missing config server] and these hosts are allowed to be down: []")); + containsString("Changing the state of cfg1 would violate enough-services-up: " + + "Suspension for service type configserver would increase from 33% to 66%, " + + "over the limit of 10%. Services down on resumed hosts: [1 missing config server].")); } } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java index d834034c9a8..4462e886d1b 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java @@ -101,8 +101,7 @@ public class HostedVespaClusterPolicyTest { when(clusterApi.serviceType()).thenReturn(new ServiceType("service-type")); when(clusterApi.percentageOfServicesDown()).thenReturn(5); when(clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()).thenReturn(percentageOfServicesDownIfGroupIsAllowedToBeDown); - when(clusterApi.servicesDownAndNotInGroupDescription()).thenReturn("services-down-and-not-in-group"); - when(clusterApi.nodesAllowedToBeDownNotInGroupDescription()).thenReturn("allowed-to-be-down"); + when(clusterApi.downDescription()).thenReturn(" Down description"); NodeGroup nodeGroup = mock(NodeGroup.class); when(clusterApi.getNodeGroup()).thenReturn(nodeGroup); @@ -116,11 +115,9 @@ public class HostedVespaClusterPolicyTest { } } catch (HostStateChangeDeniedException e) { if (!expectSuccess) { - assertEquals("Changing the state of node-group would violate enough-services-up: " - + "Suspension percentage for service type service-type would increase from " - + "5% to 13%, over the limit of 10%. These instances may be down: " - + "services-down-and-not-in-group and these hosts are allowed to be down: " - + "allowed-to-be-down", e.getMessage()); + assertEquals("Changing the state of node-group would violate enough-services-up: " + + "Suspension for service type service-type would increase from 5% to 13%, " + + "over the limit of 10%. Down description", e.getMessage()); assertEquals("enough-services-up", e.getConstraintName()); } } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java index ff7413cd3bb..bfa68145828 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java @@ -60,9 +60,9 @@ import java.util.Optional; import java.util.Set; import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceClusterSet; -import static org.fest.assertions.Assertions.assertThat; -import static org.fest.assertions.Fail.fail; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -167,7 +167,7 @@ public class HostResourceTest { UpdateHostResponse response = hostResource.suspend(hostName); - assertThat(response.hostname()).isEqualTo(hostName); + assertEquals(hostName, response.hostname()); } @Test @@ -175,14 +175,14 @@ public class HostResourceTest { HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(alwaysAllowOrchestrator); BatchOperationResult response = hostSuspensionResource.suspendAll("parentHostname", Arrays.asList("hostname1", "hostname2")); - assertThat(response.success()); + assertTrue(response.success()); } @Test public void returns_200_empty_batch() { HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(alwaysAllowOrchestrator); BatchOperationResult response = hostSuspensionResource.suspendAll("parentHostname", List.of()); - assertThat(response.success()); + assertTrue(response.success()); } @Test @@ -193,7 +193,7 @@ public class HostResourceTest { hostResource.suspend("hostname"); fail(); } catch (WebApplicationException w) { - assertThat(w.getResponse().getStatus()).isEqualTo(404); + assertEquals(404, w.getResponse().getStatus()); } } @@ -207,7 +207,7 @@ public class HostResourceTest { hostSuspensionResource.suspendAll("parentHostname", Arrays.asList("hostname1", "hostname2")); fail(); } catch (WebApplicationException w) { - assertThat(w.getResponse().getStatus()).isEqualTo(400); + assertEquals(400, w.getResponse().getStatus()); } } @@ -259,7 +259,7 @@ public class HostResourceTest { hostResource.suspend("hostname"); fail(); } catch (WebApplicationException w) { - assertThat(w.getResponse().getStatus()).isEqualTo(409); + assertEquals(409, w.getResponse().getStatus()); } } @@ -280,7 +280,7 @@ public class HostResourceTest { hostSuspensionResource.suspendAll("parentHostname", Arrays.asList("hostname1", "hostname2")); fail(); } catch (WebApplicationException w) { - assertThat(w.getResponse().getStatus()).isEqualTo(409); + assertEquals(409, w.getResponse().getStatus()); } } @@ -377,7 +377,7 @@ public class HostResourceTest { hostResource.resume("hostname"); fail(); } catch (WebApplicationException w) { - assertThat(w.getResponse().getStatus()).isEqualTo(409); + assertEquals(409, w.getResponse().getStatus()); assertEquals("resume failed: Timeout Message [deadline]", w.getMessage()); } } @@ -392,7 +392,7 @@ public class HostResourceTest { resource.suspendAll("parenthost", Arrays.asList("h1", "h2", "h3")); fail(); } catch (WebApplicationException w) { - assertThat(w.getResponse().getStatus()).isEqualTo(409); + assertEquals(409, w.getResponse().getStatus()); } } } diff --git a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp index 340619f09bd..37ac5f0d65c 100644 --- a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp +++ b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp @@ -1,12 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("threading_service_config_test"); #include <vespa/searchcore/config/config-proton.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/server/threading_service_config.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/log/log.h> +LOG_SETUP("threading_service_config_test"); + using namespace proton; using ProtonConfig = vespa::config::search::core::ProtonConfig; using ProtonConfigBuilder = vespa::config::search::core::ProtonConfigBuilder; @@ -42,6 +43,7 @@ TEST_F("require that indexing threads are set based on cpu cores and feeding con TEST_DO(f.assertIndexingThreads(3, 18)); TEST_DO(f.assertIndexingThreads(4, 19)); TEST_DO(f.assertIndexingThreads(4, 24)); + TEST_DO(f.assertIndexingThreads(4, 64)); // Ensure it is capped at 4 } TEST_F("require that indexing threads is always >= 1", Fixture(0)) diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp index 7c5e7584eed..0e80d31a063 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp @@ -35,7 +35,7 @@ namespace { struct WaitTimer { double &wait_time_s; vespalib::Timer wait_time; - WaitTimer(double &wait_time_s_in) + explicit WaitTimer(double &wait_time_s_in) : wait_time_s(wait_time_s_in), wait_time() { } void done() { diff --git a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp index 8f1c3560e9b..55aa1a20ef6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp @@ -22,7 +22,10 @@ namespace { uint32_t calculateIndexingThreads(uint32_t cfgIndexingThreads, double concurrency, const HwInfo::Cpu &cpuInfo) { - double scaledCores = cpuInfo.cores() * concurrency; + // We are capping at 12 threads to reduce cost of waking up threads + // to achieve a better throughput. + // TODO: Fix this in a simpler/better way. + double scaledCores = std::min(12.0, cpuInfo.cores() * concurrency); uint32_t indexingThreads = std::max((uint32_t)std::ceil(scaledCores / 3), cfgIndexingThreads); return std::max(indexingThreads, 1u); } diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index a7689cd6b9f..60f3a6b7664 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -16,7 +16,6 @@ #include <vespa/searchlib/index/i_field_length_inspector.h> #include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/query/tree/simplequery.h> -#include <vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h> #include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/searchlib/queryeval/fake_search.h> #include <vespa/searchlib/queryeval/fake_searchable.h> @@ -24,7 +23,6 @@ #include <vespa/searchlib/test/index/mock_field_length_inspector.h> #include <vespa/searchlib/util/rand48.h> #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/log/log.h> diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java index 18f6c6f2ca2..1f27bc8750e 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java @@ -10,6 +10,7 @@ import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; +import com.yahoo.text.Text; import java.io.File; import java.io.FileNotFoundException; @@ -115,7 +116,7 @@ public class RankingExpression implements Serializable { root = parse(new StringReader(expression)); } catch (ParseException e) { - ParseException p = new ParseException("Could not parse '" + expression + "'"); + ParseException p = new ParseException("Could not parse '" + Text.truncate(expression, 50) + "'"); p.initCause(e); throw p; } diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 5089743a54a..1f3ad7c2fec 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -6,6 +6,7 @@ #include <vespa/eval/tensor/tensor.h> #include <vespa/fastos/file.h> #include <vespa/searchlib/attribute/attributeguard.h> +#include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/tensor/doc_vector_access.h> @@ -41,6 +42,7 @@ using vespalib::tensor::DenseTensor; using vespalib::tensor::Tensor; using DoubleVector = std::vector<double>; +using generation_t = vespalib::GenerationHandler::generation_t; namespace vespalib::tensor { @@ -80,12 +82,16 @@ private: const DocVectorAccess& _vectors; EntryVector _adds; EntryVector _removes; + generation_t _transfer_gen; + generation_t _trim_gen; public: MockNearestNeighborIndex(const DocVectorAccess& vectors) : _vectors(vectors), _adds(), - _removes() + _removes(), + _transfer_gen(std::numeric_limits<generation_t>::max()), + _trim_gen(std::numeric_limits<generation_t>::max()) { } void clear() { @@ -111,6 +117,9 @@ public: EXPECT_EQUAL(exp_docid, _removes.back().first); EXPECT_EQUAL(exp_vector, _removes.back().second); } + generation_t get_transfer_gen() const { return _transfer_gen; } + generation_t get_trim_gen() const { return _trim_gen; } + void add_document(uint32_t docid) override { auto vector = _vectors.get_vector(docid).typify<double>(); _adds.emplace_back(docid, DoubleVector(vector.begin(), vector.end())); @@ -119,6 +128,15 @@ public: auto vector = _vectors.get_vector(docid).typify<double>(); _removes.emplace_back(docid, DoubleVector(vector.begin(), vector.end())); } + void transfer_hold_lists(generation_t current_gen) override { + _transfer_gen = current_gen; + } + void trim_hold_lists(generation_t first_used_gen) override { + _trim_gen = first_used_gen; + } + vespalib::MemoryUsage memory_usage() const override { + return vespalib::MemoryUsage(); + } std::vector<Neighbor> find_top_k(uint32_t k, vespalib::tensor::TypedCells vector, uint32_t explore_k) const override { (void) k; (void) vector; @@ -232,6 +250,10 @@ struct Fixture _attr->commit(); } + generation_t get_current_gen() const { + return _attr->getCurrentGeneration(); + } + search::attribute::Status getStatus() { _attr->commit(true); return _attr->getStatus(); @@ -531,4 +553,33 @@ TEST_F("onLoad() updates nearest neighbor index", DenseTensorAttributeMockIndex) index.expect_adds({{1, {3, 5}}, {2, {7, 9}}}); } + +TEST_F("commit() ensures transfer and trim hold lists on nearest neighbor index", DenseTensorAttributeMockIndex) +{ + auto& index = f.mock_index(); + TensorSpec spec = vec_2d(3, 5); + + f.set_tensor(1, spec); + generation_t gen_1 = f.get_current_gen(); + EXPECT_EQUAL(gen_1 - 1, index.get_transfer_gen()); + EXPECT_EQUAL(gen_1, index.get_trim_gen()); + + generation_t gen_2 = 0; + { + // Takes guard on gen_1 + auto guard = f._attr->makeReadGuard(false); + f.set_tensor(2, spec); + gen_2 = f.get_current_gen(); + EXPECT_GREATER(gen_2, gen_1); + EXPECT_EQUAL(gen_2 - 1, index.get_transfer_gen()); + EXPECT_EQUAL(gen_1, index.get_trim_gen()); + } + + f.set_tensor(3, spec); + generation_t gen_3 = f.get_current_gen(); + EXPECT_GREATER(gen_3, gen_2); + EXPECT_EQUAL(gen_3 - 1, index.get_transfer_gen()); + EXPECT_EQUAL(gen_3, index.get_trim_gen()); +} + TEST_MAIN() { TEST_RUN_ALL(); vespalib::unlink("test.dat"); } diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/.gitignore b/searchlib/src/tests/common/sequencedtaskexecutor/.gitignore index 35d038b0b7c..4bd94f124fb 100644 --- a/searchlib/src/tests/common/sequencedtaskexecutor/.gitignore +++ b/searchlib/src/tests/common/sequencedtaskexecutor/.gitignore @@ -1 +1,2 @@ searchlib_sequencedtaskexecutor_test_app +searchlib_sequencedtaskexecutor_benchmark_app diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt b/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt index 6ba30a1647f..6c593d20683 100644 --- a/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt +++ b/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt @@ -1,4 +1,11 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_sequencedtaskexecutor_benchmark_app TEST + SOURCES + sequencedtaskexecutor_benchmark.cpp + DEPENDS + searchlib +) + vespa_add_executable(searchlib_sequencedtaskexecutor_test_app TEST SOURCES sequencedtaskexecutor_test.cpp diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp b/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp new file mode 100644 index 00000000000..a51becfbf13 --- /dev/null +++ b/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp @@ -0,0 +1,24 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchlib/common/sequencedtaskexecutor.h> +#include <vespa/vespalib/util/lambdatask.h> +#include <atomic> + +using search::SequencedTaskExecutor; +using ExecutorId = search::ISequencedTaskExecutor::ExecutorId; + +int main(int argc, char *argv[]) { + unsigned long numTasks = 1000000; + unsigned numThreads = 4; + std::atomic<long> counter(0); + if (argc > 1) + numTasks = atol(argv[1]); + if (argc > 2) + numThreads = atoi(argv[2]); + + SequencedTaskExecutor executor(numThreads); + for (unsigned long tid(0); tid < numTasks; tid++) { + executor.executeTask(ExecutorId(tid%numThreads), vespalib::makeLambdaTask([&counter] { counter++; })); + } + return 0; +} diff --git a/searchlib/src/tests/features/prod_features.cpp b/searchlib/src/tests/features/prod_features.cpp index 56aaf2dcbc9..de436dffff1 100644 --- a/searchlib/src/tests/features/prod_features.cpp +++ b/searchlib/src/tests/features/prod_features.cpp @@ -9,16 +9,15 @@ #include <vespa/searchlib/attribute/floatbase.h> #include <vespa/searchlib/attribute/integerbase.h> #include <vespa/searchlib/attribute/stringbase.h> +#include <vespa/searchlib/attribute/singleboolattribute.h> #include <vespa/searchlib/features/agefeature.h> #include <vespa/searchlib/features/array_parser.hpp> #include <vespa/searchlib/features/attributefeature.h> -#include <vespa/searchlib/features/attributematchfeature.h> #include <vespa/searchlib/features/closenessfeature.h> #include <vespa/searchlib/features/distancefeature.h> #include <vespa/searchlib/features/dotproductfeature.h> #include <vespa/searchlib/features/fieldlengthfeature.h> #include <vespa/searchlib/features/fieldmatchfeature.h> -#include <vespa/searchlib/features/fieldtermmatchfeature.h> #include <vespa/searchlib/features/firstphasefeature.h> #include <vespa/searchlib/features/foreachfeature.h> #include <vespa/searchlib/features/freshnessfeature.h> @@ -35,7 +34,6 @@ #include <vespa/searchlib/features/setup.h> #include <vespa/searchlib/features/termfeature.h> #include <vespa/searchlib/features/utils.h> -#include <vespa/searchlib/features/valuefeature.h> #include <vespa/searchlib/features/weighted_set_parser.hpp> #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/indexproperties.h> @@ -60,6 +58,7 @@ using search::AttributeFactory; using search::IntegerAttribute; using search::FloatingPointAttribute; using search::StringAttribute; +using search::SingleBoolAttribute; using search::WeightedSetStringExtAttribute; using search::attribute::WeightedEnumContent; @@ -212,7 +211,7 @@ Test::setupForAgeTest(FtFeatureTest & ft, uint64_t docTime) doctime->addReservedDoc(); doctime->addDocs(1); ft.getIndexEnv().getAttributeMap().add(doctime); - (static_cast<IntegerAttribute *>(doctime.get()))->update(1, docTime); + (dynamic_cast<IntegerAttribute *>(doctime.get()))->update(1, docTime); doctime->commit(); } @@ -240,7 +239,12 @@ Test::testAttribute() RankResult exp; exp.addScore("attribute(sint)", 10). addScore("attribute(sint,0)", 10). + addScore("attribute(slong)", 20). + addScore("attribute(sbyte)", 37). + addScore("attribute(sbool)", 1). + addScore("attribute(sebool)", 0). addScore("attribute(sfloat)", 60.5f). + addScore("attribute(sdouble)", 67.5f). addScore("attribute(sstr)", (feature_t)vespalib::hash_code("foo")). addScore("attribute(sint).count", 1). addScore("attribute(sfloat).count", 1). @@ -250,12 +254,18 @@ Test::testAttribute() addScore("attribute(udefstr)", (feature_t)vespalib::hash_code("")); FtFeatureTest ft(_factory, exp.getKeys()); - ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sint"). - addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sfloat"). - addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sstr"). - addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefint"). - addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udeffloat"). - addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefstr"); + ft.getIndexEnv().getBuilder() + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sint") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "slong") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sbyte") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sbool") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sebool") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sfloat") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sdouble") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sstr") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefint") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udeffloat") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefstr"); setupForAttributeTest(ft); ASSERT_TRUE(ft.setup()); ASSERT_TRUE(ft.execute(exp)); @@ -370,6 +380,11 @@ Test::setupForAttributeTest(FtFeatureTest &ft, bool setup_env) avs.push_back(AttributeFactory::createAttribute("udefint", AVC(AVBT::INT32, AVCT::SINGLE))); // 9 avs.push_back(AttributeFactory::createAttribute("udeffloat", AVC(AVBT::FLOAT, AVCT::SINGLE))); // 10 avs.push_back(AttributeFactory::createAttribute("udefstr", AVC(AVBT::STRING, AVCT::SINGLE))); // 11 + avs.push_back(AttributeFactory::createAttribute("sbyte", AVC(AVBT::INT64, AVCT::SINGLE))); // 12 + avs.push_back(AttributeFactory::createAttribute("slong", AVC(AVBT::INT64, AVCT::SINGLE))); // 13 + avs.push_back(AttributeFactory::createAttribute("sbool", AVC(AVBT::BOOL, AVCT::SINGLE))); // 14 + avs.push_back(AttributeFactory::createAttribute("sebool", AVC(AVBT::BOOL, AVCT::SINGLE))); // 15 + avs.push_back(AttributeFactory::createAttribute("sdouble", AVC(AVBT::DOUBLE, AVCT::SINGLE))); // 16 // simulate a unique only attribute as specified in sd AVC cfg(AVBT::INT32, AVCT::SINGLE); @@ -391,36 +406,46 @@ Test::setupForAttributeTest(FtFeatureTest &ft, bool setup_env) .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefint") .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udeffloat") .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefstr") - .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "unique"); + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "unique") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "slong") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sdouble") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sbyte") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sbool") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sebool"); } - for (uint32_t i = 0; i < avs.size(); ++i) { - avs[i]->addReservedDoc(); - avs[i]->addDocs(1); - ft.getIndexEnv().getAttributeMap().add(avs[i]); + for (const auto & attr : avs) { + attr->addReservedDoc(); + attr->addDocs(1); + ft.getIndexEnv().getAttributeMap().add(attr); } // integer attributes - (static_cast<IntegerAttribute *>(avs[0].get()))->update(1, 10); - (static_cast<IntegerAttribute *>(avs[1].get()))->append(1, 20, 0); - (static_cast<IntegerAttribute *>(avs[1].get()))->append(1, 30, 0); - (static_cast<IntegerAttribute *>(avs[2].get()))->append(1, 40, 10); - (static_cast<IntegerAttribute *>(avs[2].get()))->append(1, 50, 20); - (static_cast<IntegerAttribute *>(avs[9].get()))->update(1, search::attribute::getUndefined<int32_t>()); + (dynamic_cast<IntegerAttribute *>(avs[0].get()))->update(1, 10); + (dynamic_cast<IntegerAttribute *>(avs[12].get()))->update(1, 37); + (dynamic_cast<IntegerAttribute *>(avs[13].get()))->update(1, 20); + (dynamic_cast<SingleBoolAttribute *>(avs[14].get()))->update(1, 1); + (dynamic_cast<SingleBoolAttribute *>(avs[15].get()))->update(1, 0); + (dynamic_cast<IntegerAttribute *>(avs[1].get()))->append(1, 20, 0); + (dynamic_cast<IntegerAttribute *>(avs[1].get()))->append(1, 30, 0); + (dynamic_cast<IntegerAttribute *>(avs[2].get()))->append(1, 40, 10); + (dynamic_cast<IntegerAttribute *>(avs[2].get()))->append(1, 50, 20); + (dynamic_cast<IntegerAttribute *>(avs[9].get()))->update(1, search::attribute::getUndefined<int32_t>()); // feature_t attributes - (static_cast<FloatingPointAttribute *>(avs[3].get()))->update(1, 60.5f); - (static_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 70.5f, 0); - (static_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 80.5f, 0); - (static_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 90.5f, -30); - (static_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 100.5f, -40); - (static_cast<FloatingPointAttribute *>(avs[10].get()))->update(1, search::attribute::getUndefined<float>()); + (dynamic_cast<FloatingPointAttribute *>(avs[3].get()))->update(1, 60.5f); + (dynamic_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 70.5f, 0); + (dynamic_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 80.5f, 0); + (dynamic_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 90.5f, -30); + (dynamic_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 100.5f, -40); + (dynamic_cast<FloatingPointAttribute *>(avs[10].get()))->update(1, search::attribute::getUndefined<float>()); + (dynamic_cast<FloatingPointAttribute *>(avs[16].get()))->update(1, 67.5); // string attributes - (static_cast<StringAttribute *>(avs[6].get()))->update(1, "foo"); - (static_cast<StringAttribute *>(avs[7].get()))->append(1, "bar", 0); - (static_cast<StringAttribute *>(avs[7].get()))->append(1, "baz", 0); - (static_cast<StringAttribute *>(avs[8].get()))->append(1, "qux", 11); - (static_cast<StringAttribute *>(avs[8].get()))->append(1, "quux", 12); - (static_cast<StringAttribute *>(avs[11].get()))->update(1, ""); + (dynamic_cast<StringAttribute *>(avs[6].get()))->update(1, "foo"); + (dynamic_cast<StringAttribute *>(avs[7].get()))->append(1, "bar", 0); + (dynamic_cast<StringAttribute *>(avs[7].get()))->append(1, "baz", 0); + (dynamic_cast<StringAttribute *>(avs[8].get()))->append(1, "qux", 11); + (dynamic_cast<StringAttribute *>(avs[8].get()))->append(1, "quux", 12); + (dynamic_cast<StringAttribute *>(avs[11].get()))->update(1, ""); for (uint32_t i = 0; i < avs.size() - 1; ++i) { // do not commit the noupdate attribute avs[i]->commit(); @@ -475,7 +500,7 @@ Test::assertCloseness(feature_t exp, const vespalib::string & attr, double dista FtFeatureTest ft(_factory, feature); std::vector<std::pair<int32_t, int32_t> > positions; int32_t x = 0; - positions.push_back(std::make_pair(x, x)); + positions.emplace_back(x, x); setupForDistanceTest(ft, "pos", positions, false); ft.getQueryEnv().getLocation().setXPosition((int)distance); ft.getQueryEnv().getLocation().setValid(true); @@ -572,7 +597,7 @@ Test::assertFieldMatch(const vespalib::string & spec, const vespalib::string & field, uint32_t totalTermWeight) { - assertFieldMatch(spec, query, field, NULL, totalTermWeight); + assertFieldMatch(spec, query, field, nullptr, totalTermWeight); } void @@ -581,7 +606,7 @@ Test::assertFieldMatchTS(const vespalib::string & spec, const vespalib::string & field, feature_t totalSignificance) { - assertFieldMatch(spec, query, field, NULL, 0, totalSignificance); + assertFieldMatch(spec, query, field, nullptr, 0, totalSignificance); } @@ -871,12 +896,12 @@ Test::setupForDistanceTest(FtFeatureTest &ft, const vespalib::string & attrName, pos->addDocs(1); ft.getIndexEnv().getAttributeMap().add(pos); - IntegerAttribute * ia = static_cast<IntegerAttribute *>(pos.get()); - for (uint32_t i = 0; i < positions.size(); ++i) { + auto ia = dynamic_cast<IntegerAttribute *>(pos.get()); + for (const auto & p : positions) { if (zcurve) { - ia->append(1, vespalib::geo::ZCurve::encode(positions[i].first, positions[i].second), 0); + ia->append(1, vespalib::geo::ZCurve::encode(p.first, p.second), 0); } else { - ia->append(1, positions[i].first, 0); + ia->append(1, p.first, 0); } } @@ -891,11 +916,11 @@ Test::assert2DZDistance(feature_t exp, const vespalib::string & positions, FtFeatureTest ft(_factory, "distance(pos)"); std::vector<vespalib::string> ta = FtUtil::tokenize(positions, ","); std::vector<std::pair<int32_t, int32_t> > pos; - for (uint32_t i = 0; i < ta.size(); ++i) { - std::vector<vespalib::string> tb = FtUtil::tokenize(ta[i], ":"); - int32_t x = util::strToNum<int32_t>(tb[0]); - int32_t y = util::strToNum<int32_t>(tb[1]); - pos.push_back(std::make_pair(x, y)); + for (const auto & s : ta) { + std::vector<vespalib::string> tb = FtUtil::tokenize(s, ":"); + auto x = util::strToNum<int32_t>(tb[0]); + auto y = util::strToNum<int32_t>(tb[1]); + pos.emplace_back(x, y); } setupForDistanceTest(ft, "pos", pos, true); ft.getQueryEnv().getLocation().setXPosition(xquery); @@ -927,7 +952,7 @@ Test::testDistanceToPath() { // Test executor. std::vector<std::pair<int32_t, int32_t> > pos; - pos.push_back(std::make_pair(0, 0)); + pos.emplace_back(0, 0); // invalid path assertDistanceToPath(pos, "a"); @@ -965,7 +990,7 @@ Test::testDistanceToPath() assertDistanceToPath(pos, "(-3,2,2,2,2,-1,0,-1)", 1, 1, 2); // multiple document locations - pos.push_back(std::make_pair(0, 1)); + pos.emplace_back(0, 1); assertDistanceToPath(pos, "(-1,1,1,1)", 0, 0.5); assertDistanceToPath(pos, "(-2,-1,-1,1)", 1, 1, 2); assertDistanceToPath(pos, "(-1,0.25,1,0.25)", 0.25, 0.5, 0.5); @@ -1017,7 +1042,7 @@ Test::testDistanceToPath() } void -Test::assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > pos, +Test::assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > & pos, const vespalib::string &path, feature_t distance, feature_t traveled, feature_t product) { LOG(info, "Testing distance to path '%s' with %zd document locations.", path.c_str(), pos.size()); @@ -1033,20 +1058,6 @@ Test::assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > pos, .addScore("distanceToPath(pos).product", product))); } -void -Test::setupForDocumentTest(FtFeatureTest &ft, const vespalib::string & attrName, const vespalib::string & docType) -{ - AttributePtr type = AttributeFactory::createAttribute(attrName, AVC(AVBT::STRING, AVCT::SINGLE)); - - type->addReservedDoc(); - type->addDocs(1); - ft.getIndexEnv().getAttributeMap().add(type); - - (static_cast<StringAttribute *>(type.get()))->update(1, docType); - type->commit(); -} - - namespace { void @@ -1264,6 +1275,8 @@ void Test::setupForDotProductTest(FtFeatureTest & ft) { struct Config { + Config() : name(nullptr), dataType(AVBT::BOOL), collectionType(AVCT::SINGLE), fastSearch(false) {} + Config(const char *n, AVBT dt, AVCT ct, bool fs) : name(n), dataType(dt), collectionType(ct), fastSearch(fs) {} const char * name; AVBT dataType; AVCT collectionType; @@ -1296,11 +1309,11 @@ Test::setupForDotProductTest(FtFeatureTest & ft) baf->addDocs(2); ft.getIndexEnv().getAttributeMap().add(baf); for (size_t i(1); i < 6; i++) { - IntegerAttribute * ia = dynamic_cast<IntegerAttribute *>(baf.get()); + auto ia = dynamic_cast<IntegerAttribute *>(baf.get()); if (ia) { ia->append(1, i, i); } else { - FloatingPointAttribute * fa = dynamic_cast<FloatingPointAttribute *>(baf.get()); + auto fa = dynamic_cast<FloatingPointAttribute *>(baf.get()); fa->append(1, i, i); } } @@ -1315,14 +1328,14 @@ Test::setupForDotProductTest(FtFeatureTest & ft) ft.getIndexEnv().getAttributeMap().add(c); ft.getIndexEnv().getAttributeMap().add(d); - StringAttribute * sa = static_cast<StringAttribute *>(a.get()); + auto sa = dynamic_cast<StringAttribute *>(a.get()); sa->append(1, "a", 1); sa->append(1, "b", 2); sa->append(1, "c", 3); sa->append(1, "d", 4); sa->append(1, "e", 5); - WeightedSetStringExtAttribute * ea = static_cast<WeightedSetStringExtAttribute *>(d.get()); + auto ea = dynamic_cast<WeightedSetStringExtAttribute *>(d.get()); EXPECT_TRUE(!ea->hasEnum()); uint32_t docId; ea->addDoc(docId); // reserved doc @@ -1517,9 +1530,9 @@ Test::testMatchCount() ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "foo"); ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "bar"); ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "baz"); - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL); // query term 0, hit in foo - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != NULL); // query term 1, hit in bar - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL); // query term 2, hit in foo + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr); // query term 0, hit in foo + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != nullptr); // query term 1, hit in bar + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr); // query term 2, hit in foo ASSERT_TRUE(ft.setup()); MatchDataBuilder::UP mdb = ft.createMatchDataBuilder(); @@ -1577,9 +1590,9 @@ Test::testMatches() ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "foo"); ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "bar"); ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "baz"); - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL); // query term 0, hit in foo - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != NULL); // query term 1, hit in bar - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL); // query term 2, hit in foo + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr); // query term 0, hit in foo + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != nullptr); // query term 1, hit in bar + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr); // query term 2, hit in foo ASSERT_TRUE(ft.setup()); MatchDataBuilder::UP mdb = ft.createMatchDataBuilder(); @@ -1613,11 +1626,9 @@ Test::assertMatches(uint32_t output, ASSERT_TRUE(ft.execute(output, EPS, docId)); // Execute and compare results. - if (!EXPECT_TRUE(ft.execute(output, EPS, docId))) return false; - return true; + return EXPECT_TRUE(ft.execute(output, EPS, docId)); } - void Test::testQuery() { @@ -1721,7 +1732,7 @@ Test::testRandom() search::Rand48 rnd; rnd.srand48(100); for (uint32_t i = 0; i < 5; ++i) { - feature_t exp = rnd.lrand48() / (feature_t)0x80000000u; + feature_t exp = static_cast<feature_t>(rnd.lrand48()) / static_cast<feature_t>(0x80000000u); ASSERT_TRUE(ft.execute(exp, EPS, i + 1)); } } @@ -1744,7 +1755,7 @@ Test::testRandom() search::Rand48 rnd; for (uint32_t i = 1; i <= 5; ++i) { rnd.srand48(100 + i); // seed + lid - feature_t exp = rnd.lrand48() / (feature_t)0x80000000u; + feature_t exp = static_cast<feature_t>(rnd.lrand48()) / static_cast<feature_t>(0x80000000u); ASSERT_TRUE(ft.execute(exp, EPS, i)); } } @@ -2067,10 +2078,7 @@ Test::assertTermDistance(const TermDistanceCalculator::Result & exp, rr.addScore(feature + ".forwardTermPosition", exp.forwardTermPos); rr.addScore(feature + ".reverse", exp.reverseDist); rr.addScore(feature + ".reverseTermPosition", exp.reverseTermPos); - if (!EXPECT_TRUE(ft.execute(rr, docId))) { - return false; - } - return true; + return EXPECT_TRUE(ft.execute(rr, docId)); } void diff --git a/searchlib/src/tests/features/prod_features.h b/searchlib/src/tests/features/prod_features.h index 8e6578f34bd..6d30dd3fcd8 100644 --- a/searchlib/src/tests/features/prod_features.h +++ b/searchlib/src/tests/features/prod_features.h @@ -10,10 +10,10 @@ class Test : public FtTestApp { public: Test(); - ~Test(); + ~Test() override; int Main() override; void testFramework(); - void testFtLib(); + static void testFtLib(); void testAge(); void testAttribute(); void testAttributeMatch(); @@ -39,10 +39,9 @@ public: void testRankingExpression(); void testTerm(); void testTermDistance(); - void testUtils(); + static void testUtils(); static void setupForDotProductTest(FtFeatureTest & ft); - static void setupForDocumentTest(FtFeatureTest &ft, const vespalib::string & attrName, const vespalib::string & docType); private: void testFieldMatchBluePrint(); @@ -81,21 +80,21 @@ private: void testFieldMatchExecutorRemaining(); void assertAge(feature_t expAge, const vespalib::string & attr, uint64_t now, uint64_t docTime); - void setupForAgeTest(FtFeatureTest & ft, uint64_t docTime); - void setupForAttributeTest(FtFeatureTest &ft, bool setup_env = true); + static void setupForAgeTest(FtFeatureTest & ft, uint64_t docTime); + static void setupForAttributeTest(FtFeatureTest &ft, bool setup_env = true); void assertCloseness(feature_t exp, const vespalib::string & attr, double distance, double maxDistance = 0, double halfResponse = 0); - void setupForDistanceTest(FtFeatureTest & ft, const vespalib::string & attrName, - const std::vector<std::pair<int32_t, int32_t> > & positions, bool zcurve); + static void setupForDistanceTest(FtFeatureTest & ft, const vespalib::string & attrName, + const std::vector<std::pair<int32_t, int32_t> > & positions, bool zcurve); void assert2DZDistance(feature_t exp, const vespalib::string & positions, int32_t xquery, int32_t yquery, uint32_t xAspect = 0); - void assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > pos, const vespalib::string &path, + void assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > & pos, const vespalib::string &path, feature_t distance = search::features::DistanceToPathExecutor::DEFAULT_DISTANCE, feature_t traveled = 1, feature_t product = 0); void assertDotProduct(feature_t exp, const vespalib::string & vector, uint32_t docId = 1, const vespalib::string & attribute = "wsstr", const vespalib::string & attributeOverride=""); void assertFieldMatch(const vespalib::string & spec, const vespalib::string & query, const vespalib::string & field, - const search::features::fieldmatch::Params * params = NULL, uint32_t totalTermWeight = 0, feature_t totalSignificance = 0.0f); + const search::features::fieldmatch::Params * params = nullptr, uint32_t totalTermWeight = 0, feature_t totalSignificance = 0.0f); void assertFieldMatch(const vespalib::string & spec, const vespalib::string & query, const vespalib::string & field, uint32_t totalTermWeight); void assertFieldMatchTS(const vespalib::string & spec, const vespalib::string & query, const vespalib::string & field, diff --git a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp index 691e80aeb9f..ee8d4b787bd 100644 --- a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp +++ b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp @@ -195,7 +195,7 @@ TEST("require that NnsIndexIterator works as expected") { std::vector<NnsIndexIterator::Hit> hits{{2,4.0}, {3,9.0}, {5,1.0}, {8,16.0}, {9,36.0}}; auto md = MatchData::makeTestInstance(2, 2); auto &tfmd = *(md->resolveTermField(0)); - auto search = NnsIndexIterator::create(true, tfmd, hits); + auto search = NnsIndexIterator::create(tfmd, hits); uint32_t docid = 1; search->initFullRange(); bool match = search->seek(docid); diff --git a/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp b/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp index b10da86dd8c..861af3527ca 100644 --- a/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp +++ b/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.h> #include <vespa/searchlib/fef/test/indexenvironment.h> #include <vespa/searchlib/fef/test/queryenvironment.h> +#include <vespa/vespalib/util/stash.h> #include <set> using namespace search::features::rankingexpression; diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp index 1204ae1e9bc..37c4d02017f 100644 --- a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -6,11 +6,14 @@ #include <vespa/searchlib/tensor/hnsw_index.h> #include <vespa/searchlib/tensor/random_level_generator.h> #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/util/generationhandler.h> #include <vector> #include <vespa/log/log.h> LOG_SETUP("hnsw_index_test"); +using vespalib::GenerationHandler; +using vespalib::MemoryUsage; using namespace search::tensor; template <typename FloatType> @@ -49,11 +52,13 @@ class HnswIndexTest : public ::testing::Test { public: FloatVectors vectors; LevelGenerator* level_generator; + GenerationHandler gen_handler; HnswIndexUP index; HnswIndexTest() : vectors(), level_generator(), + gen_handler(), index() { vectors.set(1, {2, 2}).set(2, {3, 2}).set(3, {2, 3}) @@ -70,9 +75,23 @@ public: void add_document(uint32_t docid, uint32_t max_level = 0) { level_generator->level = max_level; index->add_document(docid); + commit(); } void remove_document(uint32_t docid) { index->remove_document(docid); + commit(); + } + void commit() { + index->transfer_hold_lists(gen_handler.getCurrentGeneration()); + gen_handler.incGeneration(); + gen_handler.updateFirstUsedGeneration(); + index->trim_hold_lists(gen_handler.getFirstUsedGeneration()); + } + GenerationHandler::Guard take_read_guard() { + return gen_handler.takeGuard(); + } + MemoryUsage memory_usage() const { + return index->memory_usage(); } void expect_entry_point(uint32_t exp_docid, uint32_t exp_level) { EXPECT_EQ(exp_docid, index->get_entry_docid()); @@ -83,6 +102,10 @@ public: ASSERT_EQ(1, node.size()); EXPECT_EQ(exp_links, node.level(0)); } + void expect_empty_level_0(uint32_t docid) { + auto node = index->get_node(docid); + EXPECT_TRUE(node.empty()); + } void expect_levels(uint32_t docid, const HnswNode::LevelArray& exp_levels) { auto act_node = index->get_node(docid); ASSERT_EQ(exp_levels.size(), act_node.size()); @@ -266,5 +289,83 @@ TEST_F(HnswIndexTest, 2d_vectors_inserted_in_hierarchic_graph_with_heuristic_sel expect_level_0(7, {3, 6}); } +TEST_F(HnswIndexTest, manual_insert) +{ + init(false); + + std::vector<uint32_t> nbl; + HnswNode empty{nbl}; + index->set_node(1, empty); + index->set_node(2, empty); + + HnswNode three{{1,2}}; + index->set_node(3, three); + expect_level_0(1, {3}); + expect_level_0(2, {3}); + expect_level_0(3, {1,2}); + + expect_entry_point(1, 0); + + HnswNode twolevels{{{1},nbl}}; + index->set_node(4, twolevels); + + expect_entry_point(4, 1); + expect_level_0(1, {3,4}); + + HnswNode five{{{1,2}, {4}}}; + index->set_node(5, five); + + expect_levels(1, {{3,4,5}}); + expect_levels(2, {{3,5}}); + expect_levels(3, {{1,2}}); + expect_levels(4, {{1}, {5}}); + expect_levels(5, {{1,2}, {4}}); +} + +TEST_F(HnswIndexTest, memory_is_reclaimed_when_doing_changes_to_graph) +{ + init(true); + + add_document(1); + add_document(3); + auto mem_1 = memory_usage(); + + add_document(2); + expect_level_0(1, {2,3}); + expect_level_0(2, {1,3}); + expect_level_0(3, {1,2}); + auto mem_2 = memory_usage(); + // We should use more memory with larger link arrays and extra document. + EXPECT_GT((mem_2.usedBytes() - mem_2.deadBytes()), (mem_1.usedBytes() - mem_1.deadBytes())); + EXPECT_EQ(0, mem_2.allocatedBytesOnHold()); + + remove_document(2); + expect_level_0(1, {3}); + expect_empty_level_0(2); + expect_level_0(3, {1}); + auto mem_3 = memory_usage(); + // We end up in the same state as before document 2 was added and effectively use the same amount of memory. + EXPECT_EQ((mem_1.usedBytes() - mem_1.deadBytes()), (mem_3.usedBytes() - mem_3.deadBytes())); + EXPECT_EQ(0, mem_3.allocatedBytesOnHold()); +} + +TEST_F(HnswIndexTest, memory_is_put_on_hold_while_read_guard_is_held) +{ + init(true); + + add_document(1); + add_document(3); + { + auto guard = take_read_guard(); + add_document(2); + auto mem = memory_usage(); + // As read guard is held memory to reclaim is put on hold + EXPECT_GT(mem.allocatedBytesOnHold(), 0); + } + commit(); + auto mem = memory_usage(); + EXPECT_EQ(0, mem.allocatedBytesOnHold()); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp index bb779b659ab..617e4bf5c85 100644 --- a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp +++ b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp @@ -2,7 +2,6 @@ #include "sequencedtaskexecutor.h" #include <vespa/vespalib/util/blockingthreadstackexecutor.h> -#include <vespa/vespalib/stllike/hash_map.hpp> using vespalib::BlockingThreadStackExecutor; diff --git a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h index 2b7e70d69c7..9337f393150 100644 --- a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h +++ b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h @@ -2,7 +2,6 @@ #pragma once #include "isequencedtaskexecutor.h" -#include <vespa/vespalib/stllike/hash_map.h> #include <vector> namespace vespalib { @@ -16,7 +15,7 @@ namespace search { * Class to run multiple tasks in parallel, but tasks with same * id has to be run in sequence. */ -class SequencedTaskExecutor : public ISequencedTaskExecutor +class SequencedTaskExecutor final : public ISequencedTaskExecutor { using Stats = vespalib::ExecutorStats; std::vector<std::shared_ptr<vespalib::BlockingThreadStackExecutor>> _executors; diff --git a/searchlib/src/vespa/searchlib/features/agefeature.cpp b/searchlib/src/vespa/searchlib/features/agefeature.cpp index 258648408f8..e93691e7241 100644 --- a/searchlib/src/vespa/searchlib/features/agefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/agefeature.cpp @@ -4,6 +4,7 @@ #include "valuefeature.h" #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/matchdata.h> +#include <vespa/vespalib/util/stash.h> using search::attribute::IAttributeVector; @@ -18,20 +19,18 @@ AgeExecutor::AgeExecutor(const IAttributeVector *attribute) : _attribute(attribute), _buf() { - if (_attribute != NULL) { + if (_attribute != nullptr) { _buf.allocate(attribute->getMaxValueCount()); } } -AgeBlueprint::~AgeBlueprint() -{ -} +AgeBlueprint::~AgeBlueprint() = default; void AgeExecutor::execute(uint32_t docId) { feature_t age = 10000000000.0; - if (_attribute != NULL) { + if (_attribute != nullptr) { _buf.fill(*_attribute, docId); int64_t docTime = _buf[0]; feature_t currTime = inputs().get_number(0); @@ -65,7 +64,7 @@ AgeBlueprint::setup(const search::fef::IIndexEnvironment &env, search::fef::Blueprint::UP AgeBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new AgeBlueprint()); + return std::make_unique<AgeBlueprint>(); } search::fef::FeatureExecutor & diff --git a/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp b/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp index 4776437a14b..c26d18eb11c 100644 --- a/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp @@ -8,6 +8,7 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/parameterdescriptions.h> #include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/vespalib/util/stash.h> #include <vespa/log/log.h> LOG_SETUP(".features.attributematchfeature"); diff --git a/searchlib/src/vespa/searchlib/features/bm25_feature.cpp b/searchlib/src/vespa/searchlib/features/bm25_feature.cpp index e16b4bba996..64e365b25ac 100644 --- a/searchlib/src/vespa/searchlib/features/bm25_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/bm25_feature.cpp @@ -6,8 +6,8 @@ #include <vespa/searchlib/fef/itermfielddata.h> #include <vespa/searchlib/fef/objectstore.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <cmath> -#include <memory> #include <stdexcept> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/features/closenessfeature.cpp b/searchlib/src/vespa/searchlib/features/closenessfeature.cpp index 2358e54e9f5..ccd76f5435f 100644 --- a/searchlib/src/vespa/searchlib/features/closenessfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/closenessfeature.cpp @@ -3,14 +3,14 @@ #include "closenessfeature.h" #include "utils.h" #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <vespa/log/log.h> LOG_SETUP(".features.closenessfeature"); using namespace search::fef; -namespace search { -namespace features { +namespace search::features { ClosenessExecutor::ClosenessExecutor(feature_t maxDistance, feature_t scaleDistance) : FeatureExecutor(), @@ -96,7 +96,7 @@ ClosenessBlueprint::setup(const IIndexEnvironment & env, Blueprint::UP ClosenessBlueprint::createInstance() const { - return Blueprint::UP(new ClosenessBlueprint()); + return std::make_unique<ClosenessBlueprint>(); } FeatureExecutor & @@ -105,6 +105,4 @@ ClosenessBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash &s return stash.create<ClosenessExecutor>(_maxDistance, _scaleDistance); } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/constant_feature.cpp b/searchlib/src/vespa/searchlib/features/constant_feature.cpp index ced9d95fb33..7fc0c5c05fc 100644 --- a/searchlib/src/vespa/searchlib/features/constant_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/constant_feature.cpp @@ -4,14 +4,14 @@ #include "valuefeature.h" #include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/eval/eval/value_cache/constant_value.h> +#include <vespa/vespalib/util/stash.h> #include <vespa/log/log.h> LOG_SETUP(".features.constant_feature"); using namespace search::fef; -namespace search { -namespace features { +namespace search::features { /** * Feature executor that returns a constant value. @@ -25,8 +25,8 @@ public: ConstantFeatureExecutor(const vespalib::eval::Value &value) : _value(value) {} - virtual bool isPure() override { return true; } - virtual void execute(uint32_t) override { + bool isPure() override { return true; } + void execute(uint32_t) override { outputs().set_object(0, _value); } static FeatureExecutor &create(const vespalib::eval::Value &value, vespalib::Stash &stash) { @@ -41,9 +41,7 @@ ConstantBlueprint::ConstantBlueprint() { } -ConstantBlueprint::~ConstantBlueprint() -{ -} +ConstantBlueprint::~ConstantBlueprint() = default; void ConstantBlueprint::visitDumpFeatures(const IIndexEnvironment &, @@ -54,7 +52,7 @@ ConstantBlueprint::visitDumpFeatures(const IIndexEnvironment &, Blueprint::UP ConstantBlueprint::createInstance() const { - return Blueprint::UP(new ConstantBlueprint()); + return std::make_unique<ConstantBlueprint>(); } bool @@ -88,5 +86,4 @@ ConstantBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash } } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h b/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h index 2ab7b98fabe..785eb357795 100644 --- a/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h +++ b/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h @@ -7,7 +7,7 @@ #include <vespa/eval/eval/value.h> #include <vespa/eval/eval/value_type.h> #include <vespa/eval/tensor/default_tensor_engine.h> -#include <memory> +#include <vespa/vespalib/util/stash.h> namespace search::features { diff --git a/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp b/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp index 86a71184f43..8cc75a9a424 100644 --- a/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp +++ b/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp @@ -2,6 +2,8 @@ #include "debug_attribute_wait.h" #include <vespa/vespalib/util/time.h> +#include <vespa/vespalib/util/stash.h> + using search::attribute::IAttributeVector; using namespace search::fef; diff --git a/searchlib/src/vespa/searchlib/features/debug_wait.cpp b/searchlib/src/vespa/searchlib/features/debug_wait.cpp index fb002564572..57d19618ba4 100644 --- a/searchlib/src/vespa/searchlib/features/debug_wait.cpp +++ b/searchlib/src/vespa/searchlib/features/debug_wait.cpp @@ -2,6 +2,7 @@ #include "debug_wait.h" #include <vespa/vespalib/util/time.h> +#include <vespa/vespalib/util/stash.h> using namespace search::fef; diff --git a/searchlib/src/vespa/searchlib/features/distancefeature.cpp b/searchlib/src/vespa/searchlib/features/distancefeature.cpp index 501bfd7cd14..4d7d77fe315 100644 --- a/searchlib/src/vespa/searchlib/features/distancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/distancefeature.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/fef/matchdata.h> #include <vespa/document/datatype/positiondatatype.h> #include <vespa/vespalib/geo/zcurve.h> +#include <vespa/vespalib/util/stash.h> #include <cmath> #include <limits> @@ -13,13 +14,12 @@ LOG_SETUP(".features.distancefeature"); using namespace search::fef; -namespace search { -namespace features { +namespace search::features { feature_t DistanceExecutor::calculateDistance(uint32_t docId) { - if (_location.isValid() && _pos != NULL) { + if (_location.isValid() && _pos != nullptr) { return calculate2DZDistance(docId); } return DEFAULT_DISTANCE; @@ -66,7 +66,7 @@ DistanceExecutor::DistanceExecutor(const Location & location, _pos(pos), _intBuf() { - if (_pos != NULL) { + if (_pos != nullptr) { _intBuf.allocate(_pos->getMaxValueCount()); } } @@ -86,9 +86,7 @@ DistanceBlueprint::DistanceBlueprint() : { } -DistanceBlueprint::~DistanceBlueprint() -{ -} +DistanceBlueprint::~DistanceBlueprint() = default; void DistanceBlueprint::visitDumpFeatures(const IIndexEnvironment &, @@ -99,7 +97,7 @@ DistanceBlueprint::visitDumpFeatures(const IIndexEnvironment &, Blueprint::UP DistanceBlueprint::createInstance() const { - return Blueprint::UP(new DistanceBlueprint()); + return std::make_unique<DistanceBlueprint>(); } bool @@ -116,26 +114,26 @@ DistanceBlueprint::setup(const IIndexEnvironment & env, FeatureExecutor & DistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const { - const search::attribute::IAttributeVector * pos = NULL; + const search::attribute::IAttributeVector * pos = nullptr; const Location & location = env.getLocation(); LOG(debug, "DistanceBlueprint::createExecutor location.valid='%s', '%s', alternatively '%s'", location.isValid() ? "true" : "false", _posAttr.c_str(), document::PositionDataType::getZCurveFieldName(_posAttr).c_str()); if (location.isValid()) { pos = env.getAttributeContext().getAttribute(_posAttr); - if (pos == NULL) { + if (pos == nullptr) { LOG(debug, "Failed to find attribute '%s', resorting too '%s'", _posAttr.c_str(), document::PositionDataType::getZCurveFieldName(_posAttr).c_str()); pos = env.getAttributeContext().getAttribute(document::PositionDataType::getZCurveFieldName(_posAttr)); } - if (pos != NULL) { + if (pos != nullptr) { if (!pos->isIntegerType()) { LOG(warning, "The position attribute '%s' is not an integer attribute. Will use default distance.", pos->getName().c_str()); - pos = NULL; + pos = nullptr; } else if (pos->getCollectionType() == attribute::CollectionType::WSET) { LOG(warning, "The position attribute '%s' is a weighted set attribute. Will use default distance.", pos->getName().c_str()); - pos = NULL; + pos = nullptr; } } else { LOG(warning, "The position attribute '%s' was not found. Will use default distance.", _posAttr.c_str()); @@ -145,7 +143,4 @@ DistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash return stash.create<DistanceExecutor>(location, pos); } - - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp b/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp index 834f5913af9..a2f9225d3c4 100644 --- a/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp @@ -6,10 +6,10 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/document/datatype/positiondatatype.h> #include <vespa/vespalib/geo/zcurve.h> +#include <vespa/vespalib/util/stash.h> #include <boost/algorithm/string/split.hpp> #include <boost/algorithm/string/classification.hpp> #include <cmath> -#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".features.distancetopathfeature"); @@ -25,7 +25,7 @@ DistanceToPathExecutor::DistanceToPathExecutor(std::vector<Vector2> &path, _path(), _pos(pos) { - if (_pos != NULL) { + if (_pos != nullptr) { _intBuf.allocate(_pos->getMaxValueCount()); } _path.swap(path); // avoid copy @@ -34,7 +34,7 @@ DistanceToPathExecutor::DistanceToPathExecutor(std::vector<Vector2> &path, void DistanceToPathExecutor::execute(uint32_t docId) { - if (_path.size() > 1 && _pos != NULL) { + if (_path.size() > 1 && _pos != nullptr) { double pos = -1, trip = 0, product = 0; double minSqDist = std::numeric_limits<double>::max(); _intBuf.fill(*_pos, docId); @@ -145,21 +145,21 @@ DistanceToPathBlueprint::createExecutor(const search::fef::IQueryEnvironment &en } // Lookup the attribute vector that holds document positions. - const search::attribute::IAttributeVector *pos = NULL; + const search::attribute::IAttributeVector *pos = nullptr; if (path.size() > 1) { pos = env.getAttributeContext().getAttribute(_posAttr); - if (pos == NULL) { + if (pos == nullptr) { pos = env.getAttributeContext().getAttribute(document::PositionDataType::getZCurveFieldName(_posAttr)); } - if (pos != NULL) { + if (pos != nullptr) { if (!pos->isIntegerType()) { LOG(warning, "The position attribute '%s' is not an integer attribute. Will use default distance.", pos->getName().c_str()); - pos = NULL; + pos = nullptr; } else if (pos->getCollectionType() == attribute::CollectionType::WSET) { LOG(warning, "The position attribute '%s' is a weighted set attribute. Will use default distance.", pos->getName().c_str()); - pos = NULL; + pos = nullptr; } } else { LOG(warning, "The position attribute '%s' was not found. Will use default distance.", _posAttr.c_str()); diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp index 8998f01b59e..1072607aa8a 100644 --- a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp @@ -10,12 +10,11 @@ #include <vespa/searchlib/attribute/floatbase.h> #include <vespa/searchlib/attribute/multinumericattribute.h> #include <vespa/searchlib/attribute/multienumattribute.h> -#include <type_traits> - -#include <vespa/log/log.h> #include <vespa/eval/tensor/serialization/typed_binary_format.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/stash.h> +#include <vespa/log/log.h> LOG_SETUP(".features.dotproduct"); using namespace search::attribute; diff --git a/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp b/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp index 1622a87e733..1014fe4679a 100644 --- a/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/locale/c.h> +#include <vespa/vespalib/util/stash.h> #include <cassert> namespace search::features { @@ -116,7 +117,7 @@ ElementCompletenessBlueprint::visitDumpFeatures(const fef::IIndexEnvironment &en fef::Blueprint::UP ElementCompletenessBlueprint::createInstance() const { - return Blueprint::UP(new ElementCompletenessBlueprint()); + return std::make_unique<ElementCompletenessBlueprint>(); } bool diff --git a/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp b/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp index 0676b0a46c4..45fcd013fbf 100644 --- a/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/eval/eval/llvm/compiled_function.h> #include <vespa/eval/eval/llvm/compile_cache.h> +#include <vespa/vespalib/util/stash.h> #include <vespa/log/log.h> LOG_SETUP(".features.elementsimilarity"); diff --git a/searchlib/src/vespa/searchlib/features/euclidean_distance_feature.cpp b/searchlib/src/vespa/searchlib/features/euclidean_distance_feature.cpp index cd007f1396f..9d71d358b46 100644 --- a/searchlib/src/vespa/searchlib/features/euclidean_distance_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/euclidean_distance_feature.cpp @@ -4,9 +4,9 @@ #include "euclidean_distance_feature.h" #include "array_parser.hpp" #include <vespa/searchlib/attribute/integerbase.h> -#include <vespa/searchlib/attribute/floatbase.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/vespalib/util/stash.h> #include <cmath> #include <vespa/log/log.h> @@ -15,8 +15,7 @@ LOG_SETUP(".features.euclidean_distance_feature"); using namespace search::attribute; using namespace search::fef; -namespace search { -namespace features { +namespace search::features { template <typename DataType> @@ -57,7 +56,7 @@ EuclideanDistanceBlueprint::EuclideanDistanceBlueprint() : { } -EuclideanDistanceBlueprint::~EuclideanDistanceBlueprint() {} +EuclideanDistanceBlueprint::~EuclideanDistanceBlueprint() = default; void EuclideanDistanceBlueprint::visitDumpFeatures(const IIndexEnvironment &, IDumpFeatureVisitor &) const @@ -78,7 +77,7 @@ EuclideanDistanceBlueprint::setup(const IIndexEnvironment &env, const ParameterL Blueprint::UP EuclideanDistanceBlueprint::createInstance() const { - return Blueprint::UP(new EuclideanDistanceBlueprint()); + return std::make_unique<EuclideanDistanceBlueprint>(); } namespace { @@ -97,7 +96,7 @@ FeatureExecutor & EuclideanDistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const { const IAttributeVector * attribute = env.getAttributeContext().getAttribute(_attributeName); - if (attribute == NULL) { + if (attribute == nullptr) { LOG(warning, "The attribute vector '%s' was not found in the attribute manager, returning executor with default value.", _attributeName.c_str()); return stash.create<SingleZeroValueExecutor>(); @@ -118,6 +117,5 @@ EuclideanDistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespali } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp b/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp index d7e17187ff4..18a15ccf541 100644 --- a/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp @@ -7,12 +7,11 @@ #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/fieldtype.h> #include <vespa/searchlib/fef/featurenamebuilder.h> -#include <vespa/searchlib/fef/itermdata.h> #include <vespa/searchlib/fef/handle.h> +#include <vespa/vespalib/util/stash.h> #include <sstream> -namespace search { -namespace features { +namespace search::features { IndexFieldInfoExecutor::IndexFieldInfoExecutor(feature_t type, feature_t isFilter, [[maybe_unused]] uint32_t field, uint32_t fieldHandle) @@ -238,5 +237,4 @@ FieldInfoBlueprint::createExecutor(const fef::IQueryEnvironment &queryEnv, vespa return stash.create<ValueExecutor>(values); } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp b/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp index d0680e8fc19..74bfa156b5f 100644 --- a/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp @@ -6,13 +6,12 @@ #include <vespa/searchlib/fef/itermdata.h> #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> -#include <vespa/searchlib/fef/fieldtype.h> -#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> + using namespace search::fef; -namespace search { -namespace features { +namespace search::features { FieldLengthExecutor:: FieldLengthExecutor(const IQueryEnvironment &env, @@ -63,7 +62,7 @@ FieldLengthExecutor::handle_bind_match_data(const MatchData &md) FieldLengthBlueprint::FieldLengthBlueprint() : Blueprint("fieldLength"), - _field(NULL) + _field(nullptr) { } @@ -86,7 +85,7 @@ FieldLengthBlueprint::setup(const IIndexEnvironment &env, Blueprint::UP FieldLengthBlueprint::createInstance() const { - return Blueprint::UP(new FieldLengthBlueprint()); + return std::make_unique<FieldLengthBlueprint>(); } FeatureExecutor & @@ -100,4 +99,4 @@ FieldLengthBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Sta return stash.create<FieldLengthExecutor>(env, _field->id()); } -}} +} diff --git a/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp b/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp index 583afa6e698..7ad9eef0506 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp @@ -7,6 +7,7 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/locale/c.h> +#include <vespa/vespalib/util/stash.h> using namespace search::fef; using CollectionType = FieldInfo::CollectionType; @@ -92,14 +93,12 @@ FieldMatchExecutor::handle_bind_match_data(const fef::MatchData &md) FieldMatchBlueprint::FieldMatchBlueprint() : Blueprint("fieldMatch"), - _field(NULL), + _field(nullptr), _params() { } -FieldMatchBlueprint::~FieldMatchBlueprint() -{ -} +FieldMatchBlueprint::~FieldMatchBlueprint() = default; void FieldMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env, @@ -158,7 +157,7 @@ FieldMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env, Blueprint::UP FieldMatchBlueprint::createInstance() const { - return Blueprint::UP(new FieldMatchBlueprint()); + return std::make_unique<FieldMatchBlueprint>(); } bool @@ -306,5 +305,4 @@ FieldMatchBlueprint::createExecutor(const IQueryEnvironment & env, vespalib::Sta return stash.create<FieldMatchExecutor>(env, *_field, _params); } - } diff --git a/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp b/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp index a7a00bee956..2f065fee289 100644 --- a/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp @@ -8,9 +8,10 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/itermdata.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> -namespace search { -namespace features { + +namespace search::features { FieldTermMatchExecutor::FieldTermMatchExecutor(const search::fef::IQueryEnvironment &env, uint32_t fieldId, uint32_t termId) : @@ -122,7 +123,7 @@ FieldTermMatchBlueprint::setup(const search::fef::IIndexEnvironment &env, search::fef::Blueprint::UP FieldTermMatchBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new FieldTermMatchBlueprint()); + return std::make_unique<FieldTermMatchBlueprint>(); } search::fef::FeatureExecutor & @@ -131,4 +132,4 @@ FieldTermMatchBlueprint::createExecutor(const search::fef::IQueryEnvironment &en return stash.create<FieldTermMatchExecutor>(env, _fieldId, _termId); } -}} +} diff --git a/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp b/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp index 2e6bae14a44..9d1831c6102 100644 --- a/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp @@ -4,11 +4,12 @@ #include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> + using namespace search::fef; -namespace search { -namespace features { +namespace search::features { void FirstPhaseExecutor::execute(uint32_t) @@ -34,7 +35,7 @@ FirstPhaseBlueprint::visitDumpFeatures(const IIndexEnvironment &, Blueprint::UP FirstPhaseBlueprint::createInstance() const { - return Blueprint::UP(new FirstPhaseBlueprint()); + return std::make_unique<FirstPhaseBlueprint>(); } bool @@ -54,5 +55,4 @@ FirstPhaseBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash & } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp b/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp index eda83b991bf..e43faaec4e1 100644 --- a/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp @@ -7,6 +7,8 @@ #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/locale/c.h> +#include <vespa/vespalib/util/stash.h> + #include <cassert> #include <vespa/log/log.h> @@ -285,7 +287,7 @@ FlowCompletenessBlueprint::visitDumpFeatures(const fef::IIndexEnvironment &env, fef::Blueprint::UP FlowCompletenessBlueprint::createInstance() const { - return Blueprint::UP(new FlowCompletenessBlueprint()); + return std::make_unique<FlowCompletenessBlueprint>(); } bool @@ -318,6 +320,4 @@ FlowCompletenessBlueprint::createExecutor(const fef::IQueryEnvironment &env, ves return stash.create<FlowCompletenessExecutor>(env, _params); } -//----------------------------------------------------------------------------- - } diff --git a/searchlib/src/vespa/searchlib/features/foreachfeature.cpp b/searchlib/src/vespa/searchlib/features/foreachfeature.cpp index a67d8001a36..21167dd23d4 100644 --- a/searchlib/src/vespa/searchlib/features/foreachfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/foreachfeature.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> #include <boost/algorithm/string/replace.hpp> #include <vespa/log/log.h> @@ -120,9 +121,7 @@ ForeachBlueprint::ForeachBlueprint() : { } -ForeachBlueprint::~ForeachBlueprint() -{ -} +ForeachBlueprint::~ForeachBlueprint() = default; void ForeachBlueprint::visitDumpFeatures(const IIndexEnvironment &, @@ -171,13 +170,13 @@ ForeachBlueprint::setup(const IIndexEnvironment & env, Blueprint::UP ForeachBlueprint::createInstance() const { - return Blueprint::UP(new ForeachBlueprint()); + return std::make_unique<ForeachBlueprint>(); } FeatureExecutor & ForeachBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash &stash) const { - if (_executorCreator.get() != NULL) { + if (_executorCreator) { return _executorCreator->create(_num_inputs, stash); } return stash.create<SingleZeroValueExecutor>(); diff --git a/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp b/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp index 11ae8305e16..6e621f61034 100644 --- a/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp @@ -3,14 +3,14 @@ #include "freshnessfeature.h" #include "utils.h" #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <vespa/log/log.h> LOG_SETUP(".features.freshnessfeature"); using namespace search::fef; -namespace search { -namespace features { +namespace search::features { FreshnessExecutor::FreshnessExecutor(feature_t maxAge, feature_t scaleAge) : FeatureExecutor(), @@ -86,7 +86,7 @@ FreshnessBlueprint::setup(const IIndexEnvironment & env, Blueprint::UP FreshnessBlueprint::createInstance() const { - return Blueprint::UP(new FreshnessBlueprint()); + return std::make_unique<FreshnessBlueprint>(); } fef::ParameterDescriptions @@ -101,7 +101,4 @@ FreshnessBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash &s return stash.create<FreshnessExecutor>(_maxAge, _scaleAge); } - -} // namespace features -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp index fd1faeae5ea..e2e8a206099 100644 --- a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp @@ -5,12 +5,13 @@ #include "weighted_set_parser.h" #include "dotproductfeature.h" -#include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/attribute/imported_attribute_vector_read_guard.h> #include <vespa/searchlib/attribute/multinumericattribute.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/searchcommon/common/datatype.h> +#include <vespa/vespalib/util/stash.h> + #include <vespa/log/log.h> LOG_SETUP(".features.internalmaxreduceprodjoin"); diff --git a/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp b/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp index 45baf646656..89f7751a369 100644 --- a/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp @@ -3,11 +3,11 @@ #include "item_raw_score_feature.h" #include "valuefeature.h" #include "utils.h" +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { void ItemRawScoreExecutor::execute(uint32_t docId) @@ -89,6 +89,4 @@ ItemRawScoreBlueprint::resolve(const IQueryEnvironment &env, return handles; } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp b/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp index a5e3e2da5ba..dc689459ff3 100644 --- a/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp @@ -6,6 +6,8 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/locale/c.h> +#include <vespa/vespalib/util/stash.h> + namespace search::features { @@ -168,7 +170,7 @@ JaroWinklerDistanceBlueprint::setup(const search::fef::IIndexEnvironment &env, search::fef::Blueprint::UP JaroWinklerDistanceBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new JaroWinklerDistanceBlueprint()); + return std::make_unique<JaroWinklerDistanceBlueprint>(); } search::fef::FeatureExecutor & diff --git a/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp b/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp index fd453e17eb1..c061ace4854 100644 --- a/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp @@ -3,11 +3,11 @@ #include "matchcountfeature.h" #include "utils.h" #include "valuefeature.h" +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { MatchCountExecutor::MatchCountExecutor(uint32_t fieldId, const IQueryEnvironment &env) : FeatureExecutor(), @@ -43,7 +43,7 @@ MatchCountExecutor::handle_bind_match_data(const MatchData &md) MatchCountBlueprint::MatchCountBlueprint() : Blueprint("matchCount"), - _field(NULL) + _field(nullptr) { } @@ -63,7 +63,7 @@ MatchCountBlueprint::setup(const IIndexEnvironment &, const ParameterList & para Blueprint::UP MatchCountBlueprint::createInstance() const { - return Blueprint::UP(new MatchCountBlueprint()); + return std::make_unique<MatchCountBlueprint>(); } FeatureExecutor & @@ -75,5 +75,4 @@ MatchCountBlueprint::createExecutor(const IQueryEnvironment & queryEnv, vespalib return stash.create<MatchCountExecutor>(_field->id(), queryEnv); } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/matchesfeature.cpp b/searchlib/src/vespa/searchlib/features/matchesfeature.cpp index f4788ee74c8..a99c2330ee3 100644 --- a/searchlib/src/vespa/searchlib/features/matchesfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/matchesfeature.cpp @@ -4,11 +4,12 @@ #include "utils.h" #include "valuefeature.h" #include <vespa/searchlib/fef/fieldinfo.h> +#include <vespa/vespalib/util/stash.h> + using namespace search::fef; -namespace search { -namespace features { +namespace search::features { MatchesExecutor::MatchesExecutor(uint32_t fieldId, const search::fef::IQueryEnvironment &env, @@ -47,7 +48,7 @@ MatchesExecutor::handle_bind_match_data(const MatchData &md) MatchesBlueprint::MatchesBlueprint() : Blueprint("matches"), - _field(NULL), + _field(nullptr), _termIdx(std::numeric_limits<uint32_t>::max()) { } @@ -73,7 +74,7 @@ MatchesBlueprint::setup(const IIndexEnvironment &, Blueprint::UP MatchesBlueprint::createInstance() const { - return Blueprint::UP(new MatchesBlueprint()); + return std::make_unique<MatchesBlueprint>(); } FeatureExecutor & @@ -89,5 +90,4 @@ MatchesBlueprint::createExecutor(const IQueryEnvironment & queryEnv, vespalib::S } } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/matchfeature.cpp b/searchlib/src/vespa/searchlib/features/matchfeature.cpp index 7210b8b67e9..f6843df1a2f 100644 --- a/searchlib/src/vespa/searchlib/features/matchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/matchfeature.cpp @@ -2,17 +2,16 @@ #include "matchfeature.h" #include "utils.h" -#include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/properties.h> -#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> + using namespace search::fef; using CollectionType = FieldInfo::CollectionType; -namespace search { -namespace features { +namespace search::features { MatchExecutor::MatchExecutor(const MatchParams & params) : FeatureExecutor(), @@ -46,9 +45,7 @@ MatchBlueprint::MatchBlueprint() : { } -MatchBlueprint::~MatchBlueprint() -{ -} +MatchBlueprint::~MatchBlueprint() = default; void MatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env, @@ -61,7 +58,7 @@ MatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env, Blueprint::UP MatchBlueprint::createInstance() const { - return Blueprint::UP(new MatchBlueprint()); + return std::make_unique<MatchBlueprint>(); } bool @@ -101,6 +98,4 @@ MatchBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &st return stash.create<MatchExecutor>(_params); } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp b/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp index 7865e32849f..5ede14130ec 100644 --- a/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp @@ -2,11 +2,11 @@ #include "native_dot_product_feature.h" #include "utils.h" +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { NativeDotProductExecutor::NativeDotProductExecutor(const search::fef::IQueryEnvironment &env) : FeatureExecutor(), @@ -80,5 +80,4 @@ NativeDotProductBlueprint::createExecutor(const IQueryEnvironment &queryEnv, ves } } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp b/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp index 5dda159f629..865ea9fc3c4 100644 --- a/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp @@ -7,11 +7,11 @@ #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/itablemanager.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { feature_t NativeAttributeMatchExecutor::calculateScore(const CachedTermData &td, const TermFieldMatchData &tfmd) @@ -115,7 +115,7 @@ NativeAttributeMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env, Blueprint::UP NativeAttributeMatchBlueprint::createInstance() const { - return Blueprint::UP(new NativeAttributeMatchBlueprint()); + return std::make_unique<NativeAttributeMatchBlueprint>(); } fef::ParameterDescriptions @@ -160,6 +160,4 @@ NativeAttributeMatchBlueprint::createExecutor(const IQueryEnvironment &env, vesp return NativeAttributeMatchExecutor::createExecutor(env, _params, stash); } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp b/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp index 089a8102d6e..2b6841750ad 100644 --- a/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp @@ -7,11 +7,11 @@ #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/itablemanager.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { const uint32_t NativeFieldMatchParam::NOT_DEF_FIELD_LENGTH(std::numeric_limits<uint32_t>::max()); @@ -95,9 +95,7 @@ NativeFieldMatchBlueprint::NativeFieldMatchBlueprint() : { } -NativeFieldMatchBlueprint::~NativeFieldMatchBlueprint() -{ -} +NativeFieldMatchBlueprint::~NativeFieldMatchBlueprint() = default; void NativeFieldMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env, @@ -110,7 +108,7 @@ NativeFieldMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env, Blueprint::UP NativeFieldMatchBlueprint::createInstance() const { - return Blueprint::UP(new NativeFieldMatchBlueprint()); + return std::make_unique<NativeFieldMatchBlueprint>(); } bool @@ -181,5 +179,4 @@ NativeFieldMatchBlueprint::createExecutor(const IQueryEnvironment &env, vespalib } } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp b/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp index a31d9207e05..2809417e382 100644 --- a/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp @@ -7,12 +7,12 @@ #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/itablemanager.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <map> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { feature_t NativeProximityExecutor::calculateScoreForField(const FieldSetup & fs, uint32_t docId) @@ -136,9 +136,7 @@ NativeProximityBlueprint::NativeProximityBlueprint() : { } -NativeProximityBlueprint::~NativeProximityBlueprint() -{ -} +NativeProximityBlueprint::~NativeProximityBlueprint() = default; void NativeProximityBlueprint::visitDumpFeatures(const IIndexEnvironment & env, @@ -151,7 +149,7 @@ NativeProximityBlueprint::visitDumpFeatures(const IIndexEnvironment & env, Blueprint::UP NativeProximityBlueprint::createInstance() const { - return Blueprint::UP(new NativeProximityBlueprint()); + return std::make_unique<NativeProximityBlueprint>(); } bool @@ -168,12 +166,12 @@ NativeProximityBlueprint::setup(const IIndexEnvironment & env, NativeProximityParam & param = _params.vector[fieldId]; param.field = true; if ((param.proximityTable = - util::lookupTable(env, getBaseName(), "proximityTable", info->name(), _defaultProximityBoost)) == NULL) + util::lookupTable(env, getBaseName(), "proximityTable", info->name(), _defaultProximityBoost)) == nullptr) { return false; } if ((param.revProximityTable = - util::lookupTable(env, getBaseName(), "reverseProximityTable", info->name(), _defaultRevProximityBoost)) == NULL) + util::lookupTable(env, getBaseName(), "reverseProximityTable", info->name(), _defaultRevProximityBoost)) == nullptr) { return false; } @@ -190,7 +188,7 @@ NativeProximityBlueprint::setup(const IIndexEnvironment & env, if (NativeRankBlueprint::useTableNormalization(env)) { const Table * fp = param.proximityTable; const Table * rp = param.revProximityTable; - if (fp != NULL && rp != NULL) { + if (fp != nullptr && rp != nullptr) { double value = (fp->max() * param.proximityImportance) + (rp->max() * (1 - param.proximityImportance)); _params.setMaxTableSums(fieldId, value); @@ -217,6 +215,4 @@ NativeProximityBlueprint::createExecutor(const IQueryEnvironment &env, vespalib: } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp b/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp index b519c4f4b7f..a980c265484 100644 --- a/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp @@ -4,6 +4,7 @@ #include "valuefeature.h" #include "utils.h" #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <sstream> #include <vespa/log/log.h> @@ -30,8 +31,7 @@ buildFeatureName(const vespalib::string & baseName, const search::features::Fiel } -namespace search { -namespace features { +namespace search::features { FieldWrapper::FieldWrapper(const IIndexEnvironment & env, const ParameterList & fields, @@ -93,7 +93,7 @@ NativeRankBlueprint::visitDumpFeatures(const IIndexEnvironment & env, Blueprint::UP NativeRankBlueprint::createInstance() const { - return Blueprint::UP(new NativeRankBlueprint()); + return std::make_unique<NativeRankBlueprint>(); } bool @@ -168,6 +168,4 @@ NativeRankBlueprint::useTableNormalization(const search::fef::IIndexEnvironment return (!(norm.found() && (norm.get() == vespalib::string("false")))); } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/nowfeature.cpp b/searchlib/src/vespa/searchlib/features/nowfeature.cpp index 074acb2e890..d6059592cf3 100644 --- a/searchlib/src/vespa/searchlib/features/nowfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nowfeature.cpp @@ -3,6 +3,7 @@ #include "nowfeature.h" #include <vespa/searchlib/fef/queryproperties.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <chrono> namespace search::features { diff --git a/searchlib/src/vespa/searchlib/features/proximityfeature.cpp b/searchlib/src/vespa/searchlib/features/proximityfeature.cpp index f625e30f378..daeb4af6569 100644 --- a/searchlib/src/vespa/searchlib/features/proximityfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/proximityfeature.cpp @@ -2,13 +2,11 @@ #include "proximityfeature.h" #include "utils.h" -#include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/itermdata.h> -#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> -namespace search { -namespace features { +namespace search::features { ProximityConfig::ProximityConfig() : fieldId(search::fef::IllegalHandle), @@ -139,7 +137,7 @@ ProximityBlueprint::setup(const search::fef::IIndexEnvironment &env, search::fef::Blueprint::UP ProximityBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new ProximityBlueprint()); + return std::make_unique<ProximityBlueprint>(); } search::fef::FeatureExecutor & @@ -148,4 +146,4 @@ ProximityBlueprint::createExecutor(const search::fef::IQueryEnvironment &env, ve return stash.create<ProximityExecutor>(env, _config); } -}} +} diff --git a/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp b/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp index b4b6a1b0eb4..56e8810e14b 100644 --- a/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp @@ -3,15 +3,14 @@ #include "querycompletenessfeature.h" #include "utils.h" #include <vespa/searchlib/fef/featurenamebuilder.h> -#include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/itermdata.h> +#include <vespa/vespalib/util/stash.h> #include <limits> #include <vespa/log/log.h> LOG_SETUP(".features.querycompleteness"); -namespace search { -namespace features { +namespace search::features { QueryCompletenessConfig::QueryCompletenessConfig() : fieldId(search::fef::IllegalHandle), @@ -106,7 +105,7 @@ QueryCompletenessBlueprint::setup(const search::fef::IIndexEnvironment &env, search::fef::Blueprint::UP QueryCompletenessBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new QueryCompletenessBlueprint()); + return std::make_unique<QueryCompletenessBlueprint>(); } search::fef::FeatureExecutor & @@ -115,4 +114,4 @@ QueryCompletenessBlueprint::createExecutor(const search::fef::IQueryEnvironment return stash.create<QueryCompletenessExecutor>(env, _config); } -}} +} diff --git a/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp b/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp index dfc4af059a1..cbaf9e97cb6 100644 --- a/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp @@ -4,15 +4,14 @@ #include "valuefeature.h" #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/fieldinfo.h> -#include <vespa/searchlib/fef/fieldtype.h> #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/itermdata.h> -#include <vespa/searchlib/fef/handle.h> +#include <vespa/vespalib/util/stash.h> + using namespace search::fef; -namespace search { -namespace features { +namespace search::features { QueryTermCountBlueprint::QueryTermCountBlueprint() : Blueprint("queryTermCount") @@ -30,7 +29,7 @@ QueryTermCountBlueprint::visitDumpFeatures(const IIndexEnvironment & env, Blueprint::UP QueryTermCountBlueprint::createInstance() const { - return Blueprint::UP(new QueryTermCountBlueprint()); + return std::make_unique<QueryTermCountBlueprint>(); } bool @@ -49,6 +48,4 @@ QueryTermCountBlueprint::createExecutor(const IQueryEnvironment &env, vespalib:: return stash.create<ValueExecutor>(values); } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp index dd0c67df45c..8d4af8fd88d 100644 --- a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp @@ -3,6 +3,7 @@ #include "random_normal_feature.h" #include "utils.h" #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <chrono> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp index bde0cedaed0..f3d33c7dc29 100644 --- a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp @@ -3,6 +3,7 @@ #include "random_normal_stable_feature.h" #include "utils.h" #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <vespa/log/log.h> LOG_SETUP(".features.randomnormalstablefeature"); @@ -41,7 +42,7 @@ RandomNormalStableBlueprint::visitDumpFeatures(const search::fef::IIndexEnvironm search::fef::Blueprint::UP RandomNormalStableBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new RandomNormalStableBlueprint()); + return std::make_unique<RandomNormalStableBlueprint>(); } bool @@ -75,5 +76,4 @@ RandomNormalStableBlueprint::createExecutor(const search::fef::IQueryEnvironment return stash.create<RandomNormalStableExecutor>(seed, _mean, _stddev); } - } diff --git a/searchlib/src/vespa/searchlib/features/randomfeature.cpp b/searchlib/src/vespa/searchlib/features/randomfeature.cpp index 18b0cf616d4..95daebd0452 100644 --- a/searchlib/src/vespa/searchlib/features/randomfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/randomfeature.cpp @@ -3,7 +3,9 @@ #include "randomfeature.h" #include "utils.h" #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <chrono> + #include <vespa/log/log.h> LOG_SETUP(".features.randomfeature"); @@ -75,5 +77,4 @@ RandomBlueprint::createExecutor(const fef::IQueryEnvironment &env, vespalib::Sta return stash.create<RandomExecutor>(seed, matchSeed); } - } diff --git a/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp b/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp index 61355581214..ea696c75eff 100644 --- a/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp @@ -2,6 +2,7 @@ #include "raw_score_feature.h" #include "utils.h" +#include <vespa/vespalib/util/stash.h> using namespace search::fef; diff --git a/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp b/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp index c27936332d2..436e9bc0a0c 100644 --- a/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp @@ -2,14 +2,11 @@ #include "reverseproximityfeature.h" #include "utils.h" -#include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> -#include <vespa/searchlib/fef/fieldtype.h> #include <vespa/searchlib/fef/itermdata.h> -#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> -namespace search { -namespace features { +namespace search::features { ReverseProximityConfig::ReverseProximityConfig() : fieldId(search::fef::IllegalHandle), @@ -126,7 +123,7 @@ ReverseProximityBlueprint::setup(const search::fef::IIndexEnvironment &env, search::fef::Blueprint::UP ReverseProximityBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new ReverseProximityBlueprint()); + return std::make_unique<ReverseProximityBlueprint>(); } search::fef::FeatureExecutor & @@ -135,4 +132,4 @@ ReverseProximityBlueprint::createExecutor(const search::fef::IQueryEnvironment & return stash.create<ReverseProximityExecutor>(env, _config); } -}} +} diff --git a/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp b/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp index 6c52b6edb76..aaff46af93e 100644 --- a/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp @@ -2,11 +2,11 @@ #include "subqueries_feature.h" #include "utils.h" +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { SubqueriesExecutor::SubqueriesExecutor(const IQueryEnvironment &env, uint32_t fieldId) @@ -59,5 +59,4 @@ SubqueriesBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib: return stash.create<SubqueriesExecutor>(queryEnv, _field->id()); } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp b/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp index a4ca8524140..7fd487d8bdd 100644 --- a/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp @@ -2,10 +2,10 @@ #include "term_field_md_feature.h" #include "utils.h" -#include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/itablemanager.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <cassert> using namespace search::fef; @@ -83,7 +83,7 @@ TermFieldMdBlueprint::visitDumpFeatures(const IIndexEnvironment &, Blueprint::UP TermFieldMdBlueprint::createInstance() const { - return Blueprint::UP(new TermFieldMdBlueprint()); + return std::make_unique<TermFieldMdBlueprint>(); } bool diff --git a/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp b/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp index fb38a49d6eb..e9f48421fcf 100644 --- a/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp @@ -5,11 +5,11 @@ #include "utils.h" #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { TermDistanceExecutor::TermDistanceExecutor(const IQueryEnvironment & env, @@ -61,7 +61,7 @@ TermDistanceBlueprint::visitDumpFeatures(const IIndexEnvironment &, Blueprint::UP TermDistanceBlueprint::createInstance() const { - return Blueprint::UP(new TermDistanceBlueprint()); + return std::make_unique<TermDistanceBlueprint>(); } bool @@ -97,6 +97,4 @@ TermDistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::St } } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp b/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp index 5990d62cb25..433bf6134b8 100644 --- a/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp @@ -6,6 +6,8 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/locale/c.h> +#include <vespa/vespalib/util/stash.h> + #include <vespa/log/log.h> LOG_SETUP(".features.termeditdistance"); @@ -219,7 +221,7 @@ TermEditDistanceBlueprint::setup(const search::fef::IIndexEnvironment &env, search::fef::Blueprint::UP TermEditDistanceBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new TermEditDistanceBlueprint()); + return std::make_unique<TermEditDistanceBlueprint>(); } search::fef::FeatureExecutor & diff --git a/searchlib/src/vespa/searchlib/features/termfeature.cpp b/searchlib/src/vespa/searchlib/features/termfeature.cpp index d6df25cc2b9..90540726227 100644 --- a/searchlib/src/vespa/searchlib/features/termfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/termfeature.cpp @@ -4,15 +4,14 @@ #include "utils.h" #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> -#include <vespa/searchlib/fef/fieldtype.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/itermdata.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { TermExecutor::TermExecutor(const search::fef::IQueryEnvironment &env, uint32_t termId) : @@ -21,7 +20,7 @@ TermExecutor::TermExecutor(const search::fef::IQueryEnvironment &env, _connectedness(util::lookupConnectedness(env, termId)), _significance(0) { - if (_termData != NULL) { + if (_termData != nullptr) { feature_t fallback = util::getSignificance(*_termData); _significance = util::lookupSignificance(env, termId, fallback); } @@ -30,7 +29,7 @@ TermExecutor::TermExecutor(const search::fef::IQueryEnvironment &env, void TermExecutor::execute(uint32_t) { - if (_termData == NULL) { // this query term is not present in the query + if (_termData == nullptr) { // this query term is not present in the query outputs().set_number(0, 0.0f); // connectedness outputs().set_number(1, 0.0f); // significance (1 - frequency) outputs().set_number(2, 0.0f); // weight @@ -76,7 +75,7 @@ TermBlueprint::setup(const search::fef::IIndexEnvironment &, search::fef::Blueprint::UP TermBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new TermBlueprint()); + return std::make_unique<TermBlueprint>(); } search::fef::FeatureExecutor & @@ -85,4 +84,4 @@ TermBlueprint::createExecutor(const search::fef::IQueryEnvironment &env, vespali return stash.create<TermExecutor>(env, _termId); } -}} +} diff --git a/searchlib/src/vespa/searchlib/features/terminfofeature.cpp b/searchlib/src/vespa/searchlib/features/terminfofeature.cpp index 4f32cda8c86..ca7a5d8e248 100644 --- a/searchlib/src/vespa/searchlib/features/terminfofeature.cpp +++ b/searchlib/src/vespa/searchlib/features/terminfofeature.cpp @@ -3,30 +3,28 @@ #include "terminfofeature.h" #include "valuefeature.h" #include <vespa/searchlib/fef/properties.h> -#include <vespa/searchlib/fef/fieldinfo.h> -#include <vespa/searchlib/fef/fieldtype.h> #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/itermdata.h> -#include <vespa/searchlib/fef/handle.h> +#include <vespa/vespalib/util/stash.h> -namespace search { -namespace features { +using namespace search::fef; +namespace search::features { TermInfoBlueprint::TermInfoBlueprint() - : search::fef::Blueprint("termInfo"), + : Blueprint("termInfo"), _termIdx(0) { } void -TermInfoBlueprint::visitDumpFeatures(const search::fef::IIndexEnvironment &, - search::fef::IDumpFeatureVisitor &) const +TermInfoBlueprint::visitDumpFeatures(const IIndexEnvironment &, + IDumpFeatureVisitor &) const { } bool -TermInfoBlueprint::setup(const search::fef::IIndexEnvironment &, - const search::fef::ParameterList & params) +TermInfoBlueprint::setup(const IIndexEnvironment &, + const ParameterList & params) { _termIdx = params[0].asInteger(); describeOutput("queryidx", "The index of the first term with the given " @@ -34,8 +32,8 @@ TermInfoBlueprint::setup(const search::fef::IIndexEnvironment &, return true; } -search::fef::FeatureExecutor & -TermInfoBlueprint::createExecutor(const search::fef::IQueryEnvironment &queryEnv, vespalib::Stash &stash) const +FeatureExecutor & +TermInfoBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stash &stash) const { feature_t queryIdx = -1.0; if (queryEnv.getNumTerms() > _termIdx) { @@ -46,5 +44,4 @@ TermInfoBlueprint::createExecutor(const search::fef::IQueryEnvironment &queryEnv return stash.create<ValueExecutor>(values); } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp b/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp index 3cdb20a16e5..a0e0a8759a0 100644 --- a/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp @@ -3,6 +3,7 @@ #include "text_similarity_feature.h" #include <vespa/searchlib/fef/itermdata.h> #include <vespa/searchlib/fef/featurenamebuilder.h> +#include <vespa/vespalib/util/stash.h> namespace search::features { @@ -194,7 +195,7 @@ TextSimilarityBlueprint::visitDumpFeatures(const fef::IIndexEnvironment &env, fef::Blueprint::UP TextSimilarityBlueprint::createInstance() const { - return Blueprint::UP(new TextSimilarityBlueprint()); + return std::make_unique<TextSimilarityBlueprint>(); } bool @@ -218,6 +219,4 @@ TextSimilarityBlueprint::createExecutor(const fef::IQueryEnvironment &env, vespa return stash.create<TextSimilarityExecutor>(env, _field_id); } -//----------------------------------------------------------------------------- - } diff --git a/searchlib/src/vespa/searchlib/features/valuefeature.cpp b/searchlib/src/vespa/searchlib/features/valuefeature.cpp index 339cc5431f1..2b91cf1688b 100644 --- a/searchlib/src/vespa/searchlib/features/valuefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/valuefeature.cpp @@ -2,12 +2,13 @@ #include "valuefeature.h" #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/stash.h> -namespace search { -namespace features { +using namespace search::fef; +namespace search::features { ValueExecutor::ValueExecutor(const std::vector<feature_t> & values) : - search::fef::FeatureExecutor(), + FeatureExecutor(), _values(values) { } @@ -27,22 +28,20 @@ SingleZeroValueExecutor::execute(uint32_t) } ValueBlueprint::ValueBlueprint() : - search::fef::Blueprint("value"), + Blueprint("value"), _values() { } -ValueBlueprint::~ValueBlueprint() {} +ValueBlueprint::~ValueBlueprint() = default; void -ValueBlueprint::visitDumpFeatures(const search::fef::IIndexEnvironment &, - search::fef::IDumpFeatureVisitor &) const +ValueBlueprint::visitDumpFeatures(const IIndexEnvironment &, IDumpFeatureVisitor &) const { } bool -ValueBlueprint::setup(const search::fef::IIndexEnvironment &, - const search::fef::ParameterList & params) +ValueBlueprint::setup(const IIndexEnvironment &, const ParameterList & params) { for (uint32_t i = 0; i < params.size(); ++i) { _values.push_back(params[i].asDouble()); @@ -56,13 +55,12 @@ ValueBlueprint::setup(const search::fef::IIndexEnvironment &, return true; } -search::fef::FeatureExecutor & -ValueBlueprint::createExecutor(const search::fef::IQueryEnvironment &queryEnv, vespalib::Stash &stash) const +FeatureExecutor & +ValueBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stash &stash) const { (void) queryEnv; return stash.create<ValueExecutor>(_values); } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/rank_program.cpp b/searchlib/src/vespa/searchlib/fef/rank_program.cpp index d39be693806..bd8abe41afb 100644 --- a/searchlib/src/vespa/searchlib/fef/rank_program.cpp +++ b/searchlib/src/vespa/searchlib/fef/rank_program.cpp @@ -6,6 +6,9 @@ #include <algorithm> #include <cassert> +#include <vespa/log/log.h> +LOG_SETUP(".fef.rankprogram"); + using vespalib::Stash; namespace search::fef { @@ -42,7 +45,7 @@ struct OverrideVisitor : public IPropertiesVisitor { auto pos = feature_map.find(key); if (pos != feature_map.end()) { - overrides.push_back(Override(pos->second, vespalib::locale::c::strtod(values.get().c_str(), nullptr))); + overrides.emplace_back(pos->second, vespalib::locale::c::strtod(values.get().c_str(), nullptr)); } } }; @@ -175,6 +178,7 @@ RankProgram::setup(const MatchData &md, auto override_end = overrides.end(); const auto &specs = _resolver->getExecutorSpecs(); + _executors.reserve(specs.size()); for (uint32_t i = 0; i < specs.size(); ++i) { vespalib::ArrayRef<NumberOrObject> outputs = _hot_stash.create_array<NumberOrObject>(specs[i].output_types.size()); StashSelector stash(_hot_stash, _cold_stash); @@ -216,6 +220,8 @@ RankProgram::setup(const MatchData &md, } } assert(_executors.size() == specs.size()); + LOG(debug, "Num executors = %ld, hot stash = %ld, cold stash = %ld, match data fields = %d", + _executors.size(), _hot_stash.count_used(), _cold_stash.count_used(), md.getNumTermFields()); } FeatureResolver diff --git a/searchlib/src/vespa/searchlib/fef/rank_program.h b/searchlib/src/vespa/searchlib/fef/rank_program.h index 0e5c390162a..aa6f77abce4 100644 --- a/searchlib/src/vespa/searchlib/fef/rank_program.h +++ b/searchlib/src/vespa/searchlib/fef/rank_program.h @@ -9,8 +9,8 @@ #include "feature_resolver.h" #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/array.h> +#include <vespa/vespalib/util/stash.h> #include <set> -#include <vector> namespace search::fef { diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp index ee88be8ad00..88f4a07d95d 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp @@ -12,7 +12,7 @@ class VisitorAdapter : public search::fef::IDumpFeatureVisitor { search::fef::BlueprintResolver &_resolver; public: - VisitorAdapter(search::fef::BlueprintResolver &resolver) + explicit VisitorAdapter(search::fef::BlueprintResolver &resolver) : _resolver(resolver) {} void visitDumpFeature(const vespalib::string &name) override { _resolver.addSeed(name); @@ -60,7 +60,8 @@ RankSetup::RankSetup(const BlueprintFactory &factory, const IIndexEnvironment &i _diversityCutoffFactor(10.0), _diversityCutoffStrategy("loose"), _softTimeoutEnabled(false), - _softTimeoutTailCost(0.1) + _softTimeoutTailCost(0.1), + _softTimeoutFactor(0.5) { } RankSetup::~RankSetup() = default; @@ -71,13 +72,13 @@ RankSetup::configure() setFirstPhaseRank(rank::FirstPhase::lookup(_indexEnv.getProperties())); setSecondPhaseRank(rank::SecondPhase::lookup(_indexEnv.getProperties())); std::vector<vespalib::string> summaryFeatures = summary::Feature::lookup(_indexEnv.getProperties()); - for (uint32_t i = 0; i < summaryFeatures.size(); ++i) { - addSummaryFeature(summaryFeatures[i]); + for (const auto & feature : summaryFeatures) { + addSummaryFeature(feature); } setIgnoreDefaultRankFeatures(dump::IgnoreDefaultFeatures::check(_indexEnv.getProperties())); std::vector<vespalib::string> dumpFeatures = dump::Feature::lookup(_indexEnv.getProperties()); - for (uint32_t i = 0; i < dumpFeatures.size(); ++i) { - addDumpFeature(dumpFeatures[i]); + for (const auto & feature : dumpFeatures) { + addDumpFeature(feature); } split_unpacking_iterators(matching::SplitUnpackingIterators::check(_indexEnv.getProperties())); delay_unpacking_iterators(matching::DelayUnpackingIterators::check(_indexEnv.getProperties())); @@ -159,15 +160,15 @@ RankSetup::compile() _compileError = true; } } - for (uint32_t i = 0; i < _summaryFeatures.size(); ++i) { - _summary_resolver->addSeed(_summaryFeatures[i]); + for (const auto & feature :_summaryFeatures) { + _summary_resolver->addSeed(feature); } if (!_ignoreDefaultRankFeatures) { VisitorAdapter adapter(*_dumpResolver); _factory.visitDumpFeatures(_indexEnv, adapter); } - for (uint32_t i = 0; i < _dumpFeatures.size(); ++i) { - _dumpResolver->addSeed(_dumpFeatures[i]); + for (const auto & feature : _dumpFeatures) { + _dumpResolver->addSeed(feature); } _indexEnv.hintFeatureMotivation(IIndexEnvironment::RANK); _compileError |= !_first_phase_resolver->compile(); diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.h b/searchlib/src/vespa/searchlib/fef/ranksetup.h index d543794b347..e1cd78d41a9 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.h +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.h @@ -397,10 +397,10 @@ public: // them to be ready to use. Also keep in mind that creating a rank // program is cheap while setting it up is more expensive. - RankProgram::UP create_first_phase_program() const { return RankProgram::UP(new RankProgram(_first_phase_resolver)); } - RankProgram::UP create_second_phase_program() const { return RankProgram::UP(new RankProgram(_second_phase_resolver)); } - RankProgram::UP create_summary_program() const { return RankProgram::UP(new RankProgram(_summary_resolver)); } - RankProgram::UP create_dump_program() const { return RankProgram::UP(new RankProgram(_dumpResolver)); } + RankProgram::UP create_first_phase_program() const { return std::make_unique<RankProgram>(_first_phase_resolver); } + RankProgram::UP create_second_phase_program() const { return std::make_unique<RankProgram>(_second_phase_resolver); } + RankProgram::UP create_summary_program() const { return std::make_unique<RankProgram>(_summary_resolver); } + RankProgram::UP create_dump_program() const { return std::make_unique<RankProgram>(_dumpResolver); } /** * Here you can do some preprocessing. State must be stored in the IObjectStore. diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp index 31e99ef9953..8be0961f999 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp @@ -2,11 +2,10 @@ #include "cfgvalue.h" #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <sstream> -namespace search { -namespace fef { -namespace test { +namespace search::fef::test { CfgValueBlueprint::CfgValueBlueprint() : Blueprint("test_cfgvalue"), @@ -14,9 +13,7 @@ CfgValueBlueprint::CfgValueBlueprint() : { } -CfgValueBlueprint::~CfgValueBlueprint() -{ -} +CfgValueBlueprint::~CfgValueBlueprint() = default; void CfgValueBlueprint::visitDumpFeatures(const IIndexEnvironment &indexEnv, IDumpFeatureVisitor &visitor) const @@ -59,6 +56,4 @@ CfgValueBlueprint::createExecutor(const IQueryEnvironment & queryEnv, vespalib:: return stash.create<search::features::ValueExecutor>(_values); } -} // namespace test -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp index 86754c2c22d..017b916ad76 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp @@ -1,11 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "chain.h" +#include <vespa/vespalib/util/stash.h> #include <sstream> -namespace search { -namespace fef { -namespace test { +namespace search::fef::test { ChainExecutor::ChainExecutor() : FeatureExecutor() @@ -67,6 +66,4 @@ ChainBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stas return stash.create<ChainExecutor>(); } -} // namespace test -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp index d9ec8b13e57..6f8ebd57fb0 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp @@ -3,6 +3,7 @@ #include "double.h" #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/stash.h> #include <cassert> namespace search::fef::test { diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp index 4b4b10c4d25..1e57d6252ee 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp @@ -4,6 +4,7 @@ #include <vespa/searchlib/features/valuefeature.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/locale/c.h> +#include <vespa/vespalib/util/stash.h> #include <sstream> namespace search::fef::test { @@ -32,7 +33,7 @@ QueryBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stas { std::vector<feature_t> values; std::string val = queryEnv.getProperties().lookup(_key).get("0.0"); - values.push_back(vespalib::locale::c::strtod(val.data(), NULL)); + values.push_back(vespalib::locale::c::strtod(val.data(), nullptr)); return stash.create<search::features::ValueExecutor>(values); } diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp index c1b8b940245..c5871b23e77 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp @@ -2,10 +2,9 @@ #include "staticrank.h" #include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/vespalib/util/stash.h> -namespace search { -namespace fef { -namespace test { +namespace search::fef::test { StaticRankExecutor::StaticRankExecutor(const search::attribute::IAttributeVector * attribute) : FeatureExecutor(), @@ -17,7 +16,7 @@ void StaticRankExecutor::execute(uint32_t docId) { search::attribute::FloatContent staticRank; - if (_attribute != NULL) { + if (_attribute != nullptr) { staticRank.allocate(_attribute->getMaxValueCount()); staticRank.fill(*_attribute, docId); } @@ -31,9 +30,7 @@ StaticRankBlueprint::StaticRankBlueprint() : { } -StaticRankBlueprint::~StaticRankBlueprint() -{ -} +StaticRankBlueprint::~StaticRankBlueprint() = default; bool StaticRankBlueprint::setup(const IIndexEnvironment & indexEnv, const StringVector & params) @@ -54,6 +51,4 @@ StaticRankBlueprint::createExecutor(const IQueryEnvironment & queryEnv, vespalib return stash.create<StaticRankExecutor>(av); } -} // namespace test -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp index b5025d53cbd..4362a7f0860 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp @@ -2,10 +2,9 @@ #include "sum.h" #include <vespa/searchlib/fef/featurenamebuilder.h> +#include <vespa/vespalib/util/stash.h> -namespace search { -namespace fef { -namespace test { +namespace search::fef::test { void SumExecutor::execute(uint32_t) @@ -73,6 +72,4 @@ SumBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stash return stash.create<SumExecutor>(); } -} // namespace test -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp index 7b3876fada0..e30b4893e15 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "unbox.h" +#include <vespa/vespalib/util/stash.h> namespace search::fef::test { diff --git a/searchlib/src/vespa/searchlib/fef/test/test_features.cpp b/searchlib/src/vespa/searchlib/fef/test/test_features.cpp index f924acd65de..c32776dc88d 100644 --- a/searchlib/src/vespa/searchlib/fef/test/test_features.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/test_features.cpp @@ -3,6 +3,8 @@ #include <vespa/vespalib/testkit/test_kit.h> #include "test_features.h" #include <vespa/vespalib/locale/c.h> +#include <vespa/vespalib/util/stash.h> + using vespalib::eval::DoubleValue; using vespalib::eval::ValueType; diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp index f9bce4bf7d1..d4aa2aaa1d7 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp @@ -21,7 +21,11 @@ NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& f _distance_heap(target_num_hits), _found_hits() { - setEstimate(HitEstimate(_attr_tensor.getNumDocs(), false)); + uint32_t est_hits = _attr_tensor.getNumDocs(); + if (_attr_tensor.nearest_neighbor_index()) { + est_hits = std::min(target_num_hits, est_hits); + } + setEstimate(HitEstimate(est_hits, false)); } NearestNeighborBlueprint::~NearestNeighborBlueprint() = default; @@ -56,7 +60,7 @@ NearestNeighborBlueprint::createLeafSearch(const search::fef::TermFieldMatchData assert(tfmda.size() == 1); fef::TermFieldMatchData &tfmd = *tfmda[0]; // always search in only one field if (strict && ! _found_hits.empty()) { - return NnsIndexIterator::create(strict, tfmd, _found_hits); + return NnsIndexIterator::create(tfmd, _found_hits); } const vespalib::tensor::DenseTensorView &qT = *_query_tensor; return NearestNeighborIterator::create(strict, tfmd, qT, _attr_tensor, _distance_heap); diff --git a/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp index 7ee985a0ba5..18e0213e092 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp @@ -4,7 +4,7 @@ #include <vespa/searchlib/tensor/nearest_neighbor_index.h> #include <cmath> -using Hit = search::tensor::NearestNeighborIndex::Neighbor; +using Neighbor = search::tensor::NearestNeighborIndex::Neighbor; namespace search::queryeval { @@ -17,12 +17,12 @@ class NeighborVectorIterator : public NnsIndexIterator { private: fef::TermFieldMatchData &_tfmd; - const std::vector<Hit> &_hits; + const std::vector<Neighbor> &_hits; uint32_t _idx; double _last_sq_dist; public: NeighborVectorIterator(fef::TermFieldMatchData &tfmd, - const std::vector<Hit> &hits) + const std::vector<Neighbor> &hits) : _tfmd(tfmd), _hits(hits), _idx(0), @@ -59,11 +59,9 @@ public: std::unique_ptr<NnsIndexIterator> NnsIndexIterator::create( - bool strict, fef::TermFieldMatchData &tfmd, - const std::vector<Hit> &hits) + const std::vector<Neighbor> &hits) { - assert(strict); return std::make_unique<NeighborVectorIterator>(tfmd, hits); } diff --git a/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h index 62fa49aac46..9ffd0df94eb 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h @@ -13,7 +13,6 @@ class NnsIndexIterator : public SearchIterator public: using Hit = search::tensor::NearestNeighborIndex::Neighbor; static std::unique_ptr<NnsIndexIterator> create( - bool strict, fef::TermFieldMatchData &tfmd, const std::vector<Hit> &hits); }; diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index 171340e07f1..229c6c2c34f 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -183,6 +183,26 @@ DenseTensorAttribute::getVersion() const return DENSE_TENSOR_ATTRIBUTE_VERSION; } +void +DenseTensorAttribute::onGenerationChange(generation_t next_gen) +{ + // TODO: Change onGenerationChange() to send current generation instead of next generation. + // This applies for entire attribute vector code. + TensorAttribute::onGenerationChange(next_gen); + if (_index) { + _index->transfer_hold_lists(next_gen - 1); + } +} + +void +DenseTensorAttribute::removeOldGenerations(generation_t first_used_gen) +{ + TensorAttribute::removeOldGenerations(first_used_gen); + if (_index) { + _index->trim_hold_lists(first_used_gen); + } +} + vespalib::tensor::TypedCells DenseTensorAttribute::get_vector(uint32_t docid) const { diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h index f9a8a81b56b..84a60376fe7 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h @@ -29,7 +29,7 @@ public: DenseTensorAttribute(vespalib::stringref baseFileName, const Config& cfg, const NearestNeighborIndexFactory& index_factory = DefaultNearestNeighborIndexFactory()); virtual ~DenseTensorAttribute(); - // Implements TensorAttribute + // Implements AttributeVector and ITensorAttribute uint32_t clearDoc(DocId docId) override; void setTensor(DocId docId, const Tensor &tensor) override; std::unique_ptr<Tensor> getTensor(DocId docId) const override; @@ -38,6 +38,8 @@ public: std::unique_ptr<AttributeSaver> onInitSave(vespalib::stringref fileName) override; void compactWorst() override; uint32_t getVersion() const override; + void onGenerationChange(generation_t next_gen) override; + void removeOldGenerations(generation_t first_used_gen) override; // Implements DocVectorAccess vespalib::tensor::TypedCells get_vector(uint32_t docid) const override; diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index 0d308206761..467e41d83fc 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -9,6 +9,8 @@ namespace search::tensor { +using search::datastore::EntryRef; + namespace { // TODO: Move this to MemoryAllocator, with name PAGE_SIZE. @@ -44,6 +46,9 @@ HnswIndex::max_links_for_level(uint32_t level) const uint32_t HnswIndex::make_node_for_document(uint32_t docid) { + // A document cannot be added twice. + assert(!_node_refs[docid].load_acquire().valid()); + uint32_t max_level = _level_generator->max_level(); // TODO: Add capping on num_levels uint32_t num_levels = max_level + 1; @@ -54,6 +59,15 @@ HnswIndex::make_node_for_document(uint32_t docid) return max_level; } +void +HnswIndex::remove_node_for_document(uint32_t docid) +{ + auto node_ref = _node_refs[docid].load_acquire(); + _nodes.remove(node_ref); + EntryRef invalid; + _node_refs[docid].store_release(invalid); +} + HnswIndex::LevelArrayRef HnswIndex::get_level_array(uint32_t docid) const { @@ -72,10 +86,12 @@ HnswIndex::get_link_array(uint32_t docid, uint32_t level) const void HnswIndex::set_link_array(uint32_t docid, uint32_t level, const LinkArrayRef& links) { - auto links_ref = _links.add(links); + auto new_links_ref = _links.add(links); auto node_ref = _node_refs[docid].load_acquire(); auto levels = _nodes.get_writable(node_ref); - levels[level].store_release(links_ref); + auto old_links_ref = levels[level].load_acquire(); + levels[level].store_release(new_links_ref); + _links.remove(old_links_ref); } bool @@ -136,7 +152,7 @@ HnswIndex::select_neighbors(const HnswCandidateVector& neighbors, uint32_t max_l } void -HnswIndex::connect_new_node(uint32_t docid, const LinkArray& neighbors, uint32_t level) +HnswIndex::connect_new_node(uint32_t docid, const LinkArrayRef &neighbors, uint32_t level) { set_link_array(docid, level, neighbors); for (uint32_t neighbor_docid : neighbors) { @@ -248,8 +264,6 @@ HnswIndex::add_document(uint32_t docid) { auto input = get_vector(docid); _node_refs.ensure_size(docid + 1, AtomicEntryRef()); - // A document cannot be added twice. - assert(!_node_refs[docid].load_acquire().valid()); int level = make_node_for_document(docid); if (_entry_docid == 0) { _entry_docid = docid; @@ -306,8 +320,35 @@ HnswIndex::remove_document(uint32_t docid) _entry_docid = 0; _entry_level = -1; } - search::datastore::EntryRef invalid; - _node_refs[docid].store_release(invalid); + remove_node_for_document(docid); +} + +void +HnswIndex::transfer_hold_lists(generation_t current_gen) +{ + // Note: RcuVector transfers hold lists as part of reallocation based on current generation. + // We need to set the next generation here, as it is incremented on a higher level right after this call. + _node_refs.setGeneration(current_gen + 1); + _nodes.transferHoldLists(current_gen); + _links.transferHoldLists(current_gen); +} + +void +HnswIndex::trim_hold_lists(generation_t first_used_gen) +{ + _node_refs.removeOldGenerations(first_used_gen); + _nodes.trimHoldLists(first_used_gen); + _links.trimHoldLists(first_used_gen); +} + +vespalib::MemoryUsage +HnswIndex::memory_usage() const +{ + vespalib::MemoryUsage result; + result.merge(_node_refs.getMemoryUsage()); + result.merge(_nodes.getMemoryUsage()); + result.merge(_links.getMemoryUsage()); + return result; } struct NeighborsByDocId { @@ -371,5 +412,28 @@ HnswIndex::get_node(uint32_t docid) const return HnswNode(result); } +void +HnswIndex::set_node(uint32_t docid, const HnswNode &node) +{ + _node_refs.ensure_size(docid + 1, AtomicEntryRef()); + // A document cannot be added twice. + assert(!_node_refs[docid].load_acquire().valid()); + + // make new node + size_t num_levels = node.size(); + assert(num_levels > 0); + LevelArray levels(num_levels, AtomicEntryRef()); + auto node_ref = _nodes.add(levels); + _node_refs[docid].store_release(node_ref); + + for (size_t level = 0; level < num_levels; ++level) { + connect_new_node(docid, node.level(level), level); + } + int max_level = num_levels - 1; + if (_entry_level < max_level) { + _entry_docid = docid; + _entry_level = max_level; + } } +} diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h index 800b88923b5..89c45d6b50c 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -95,6 +95,7 @@ protected: uint32_t max_links_for_level(uint32_t level) const; uint32_t make_node_for_document(uint32_t docid); + void remove_node_for_document(uint32_t docid); LevelArrayRef get_level_array(uint32_t docid) const; LinkArrayRef get_link_array(uint32_t docid, uint32_t level) const; void set_link_array(uint32_t docid, uint32_t level, const LinkArrayRef& links); @@ -110,7 +111,7 @@ protected: LinkArray select_neighbors_heuristic(const HnswCandidateVector& neighbors, uint32_t max_links) const; LinkArray select_neighbors_simple(const HnswCandidateVector& neighbors, uint32_t max_links) const; LinkArray select_neighbors(const HnswCandidateVector& neighbors, uint32_t max_links) const; - void connect_new_node(uint32_t docid, const LinkArray& neighbors, uint32_t level); + void connect_new_node(uint32_t docid, const LinkArrayRef &neighbors, uint32_t level); void remove_link_to(uint32_t remove_from, uint32_t remove_id, uint32_t level); inline TypedCells get_vector(uint32_t docid) const { @@ -133,20 +134,22 @@ public: const Config& config() const { return _cfg; } + // Implements NearestNeighborIndex void add_document(uint32_t docid) override; void remove_document(uint32_t docid) override; + void transfer_hold_lists(generation_t current_gen) override; + void trim_hold_lists(generation_t first_used_gen) override; + vespalib::MemoryUsage memory_usage() const override; std::vector<Neighbor> find_top_k(uint32_t k, TypedCells vector, uint32_t explore_k) const override; - FurthestPriQ top_k_candidates(const TypedCells &vector, uint32_t k) const; - // TODO: Add support for generation handling and cleanup (transfer_hold_lists, trim_hold_lists) + FurthestPriQ top_k_candidates(const TypedCells &vector, uint32_t k) const; uint32_t get_entry_docid() const { return _entry_docid; } uint32_t get_entry_level() const { return _entry_level; } // Should only be used by unit tests. HnswNode get_node(uint32_t docid) const; - - // TODO: Implement set_node() as well for use in unit tests. + void set_node(uint32_t docid, const HnswNode &node); }; } diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h index f933af0147e..bd98623bdd3 100644 --- a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h +++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h @@ -5,6 +5,8 @@ #include <cstdint> #include <vector> #include <vespa/eval/tensor/dense/typed_cells.h> +#include <vespa/vespalib/util/generationhandler.h> +#include <vespa/vespalib/util/memoryusage.h> namespace search::tensor { @@ -13,6 +15,7 @@ namespace search::tensor { */ class NearestNeighborIndex { public: + using generation_t = vespalib::GenerationHandler::generation_t; struct Neighbor { uint32_t docid; double distance; @@ -24,9 +27,14 @@ public: virtual ~NearestNeighborIndex() {} virtual void add_document(uint32_t docid) = 0; virtual void remove_document(uint32_t docid) = 0; + virtual void transfer_hold_lists(generation_t current_gen) = 0; + virtual void trim_hold_lists(generation_t first_used_gen) = 0; + virtual vespalib::MemoryUsage memory_usage() const = 0; + virtual std::vector<Neighbor> find_top_k(uint32_t k, vespalib::tensor::TypedCells vector, uint32_t explore_k) const = 0; + }; } diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp index 89b8e77e136..0b9628e6872 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp @@ -71,7 +71,6 @@ TensorAttribute::TensorAttribute(vespalib::stringref name, const Config &cfg, Te { } - TensorAttribute::~TensorAttribute() = default; const ITensorAttribute * @@ -93,7 +92,6 @@ TensorAttribute::clearDoc(DocId docId) return 0u; } - void TensorAttribute::onCommit() { @@ -110,7 +108,6 @@ TensorAttribute::onCommit() } } - void TensorAttribute::onUpdateStat() { @@ -126,7 +123,6 @@ TensorAttribute::onUpdateStat() total.allocatedBytesOnHold()); } - void TensorAttribute::removeOldGenerations(generation_t firstUsed) { @@ -141,7 +137,6 @@ TensorAttribute::onGenerationChange(generation_t generation) _tensorStore.transferHoldLists(generation - 1); } - bool TensorAttribute::addDoc(DocId &docId) { @@ -209,7 +204,6 @@ TensorAttribute::clearDocs(DocId lidLow, DocId lidLimit) } } - void TensorAttribute::onShrinkLidSpace() { @@ -220,14 +214,12 @@ TensorAttribute::onShrinkLidSpace() setNumDocs(committedDocIdLimit); } - uint32_t TensorAttribute::getVersion() const { return TENSOR_ATTRIBUTE_VERSION; } - TensorAttribute::RefCopyVector TensorAttribute::getRefCopy() const { diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java index f559e9336c8..1cfb70560b8 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java @@ -4,9 +4,9 @@ package com.yahoo.vespa.service.duper; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.log.LogLevel; +import com.yahoo.vespa.service.monitor.DuperModelListener; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -22,12 +22,16 @@ public class DuperModel { private final Map<ApplicationId, ApplicationInfo> applications = new TreeMap<>(); private final List<DuperModelListener> listeners = new ArrayList<>(); + private boolean isComplete = false; public void registerListener(DuperModelListener listener) { applications.values().forEach(listener::applicationActivated); listeners.add(listener); } + public void setCompleteness(boolean isComplete) { this.isComplete = isComplete; } + public boolean isComplete() { return isComplete; } + public boolean contains(ApplicationId applicationId) { return applications.containsKey(applicationId); } @@ -47,6 +51,6 @@ public class DuperModel { public List<ApplicationInfo> getApplicationInfos() { logger.log(LogLevel.DEBUG, "Applications in duper model: " + applications.values().size()); - return Collections.unmodifiableList(new ArrayList<>(applications.values())); + return List.copyOf(applications.values()); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java index 885368810a8..15c461c7f59 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java @@ -12,6 +12,8 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.service.monitor.DuperModelInfraApi; +import com.yahoo.vespa.service.monitor.DuperModelListener; +import com.yahoo.vespa.service.monitor.DuperModelProvider; import com.yahoo.vespa.service.monitor.InfraApplicationApi; import java.util.ArrayList; @@ -27,7 +29,7 @@ import java.util.stream.Stream; /** * @author hakonhall */ -public class DuperModelManager implements DuperModelInfraApi { +public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi { // Infrastructure applications static final ControllerHostApplication controllerHostApplication = new ControllerHostApplication(); @@ -45,6 +47,8 @@ public class DuperModelManager implements DuperModelInfraApi { // The set of active infrastructure ApplicationInfo. Not all are necessarily in the DuperModel for historical reasons. private final Set<ApplicationId> activeInfraInfos = new HashSet<>(10); + private boolean superModelIsComplete = false; + private boolean infraApplicationsIsComplete = false; @Inject public DuperModelManager(ConfigserverConfig configServerConfig, FlagSource flagSource, SuperModelProvider superModelProvider) { @@ -53,7 +57,7 @@ public class DuperModelManager implements DuperModelInfraApi { superModelProvider, new DuperModel(), flagSource, SystemName.from(configServerConfig.system())); } - /** For testing */ + /** Non-private for testing */ DuperModelManager(boolean multitenant, boolean isController, SuperModelProvider superModelProvider, DuperModel duperModel, FlagSource flagSource, SystemName system) { this.duperModel = duperModel; @@ -86,6 +90,16 @@ public class DuperModelManager implements DuperModelInfraApi { duperModel.remove(applicationId); } } + + @Override + public void notifyOfCompleteness(SuperModel superModel) { + synchronized (monitor) { + if (!superModelIsComplete) { + superModelIsComplete = true; + maybeSetDuperModelAsComplete(); + } + } + } }); } @@ -93,6 +107,7 @@ public class DuperModelManager implements DuperModelInfraApi { * Synchronously call {@link DuperModelListener#applicationActivated(ApplicationInfo) listener.applicationActivated()} * for each currently active application, and forward future changes. */ + @Override public void registerListener(DuperModelListener listener) { synchronized (monitor) { duperModel.registerListener(listener); @@ -148,9 +163,25 @@ public class DuperModelManager implements DuperModelInfraApi { } } + @Override + public void infraApplicationsIsNowComplete() { + synchronized (monitor) { + if (!infraApplicationsIsComplete) { + infraApplicationsIsComplete = true; + maybeSetDuperModelAsComplete(); + } + } + } + public List<ApplicationInfo> getApplicationInfos() { synchronized (monitor) { return duperModel.getApplicationInfos(); } } + + private void maybeSetDuperModelAsComplete() { + if (superModelIsComplete && infraApplicationsIsComplete) { + duperModel.setCompleteness(true); + } + } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java index 3cc7010e209..d6e15f6af4e 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java @@ -92,6 +92,10 @@ public class HealthMonitorManager implements MonitorManager, HealthMonitorApi { } @Override + public void bootstrapComplete() { + } + + @Override public ServiceStatusInfo getStatus(ApplicationId applicationId, ClusterId clusterId, ServiceType serviceType, diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java index dd781a02cef..a7579d3f0da 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.manager; -import com.yahoo.vespa.service.duper.DuperModelListener; +import com.yahoo.vespa.service.monitor.DuperModelListener; import com.yahoo.vespa.service.monitor.ServiceStatusProvider; /** diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java index 3490ad4a5d2..2aacc3eadac 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java @@ -51,4 +51,8 @@ public class UnionMonitorManager implements MonitorManager { slobrokMonitorManager.applicationRemoved(id); healthMonitorManager.applicationRemoved(id); } + + @Override + public void bootstrapComplete() { + } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelCache.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelCache.java index c50f5e6c2d5..1b37555a554 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelCache.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelCache.java @@ -17,6 +17,7 @@ public class ServiceModelCache implements Supplier<ServiceModel> { private final Supplier<ServiceModel> expensiveSupplier; private final Timer timer; + private final boolean useCache; private volatile ServiceModel snapshot; private boolean updatePossiblyInProgress = false; @@ -24,13 +25,18 @@ public class ServiceModelCache implements Supplier<ServiceModel> { private final Object updateMonitor = new Object(); private long snapshotMillis; - public ServiceModelCache(Supplier<ServiceModel> expensiveSupplier, Timer timer) { + public ServiceModelCache(Supplier<ServiceModel> expensiveSupplier, Timer timer, boolean useCache) { this.expensiveSupplier = expensiveSupplier; this.timer = timer; + this.useCache = useCache; } @Override public ServiceModel get() { + if (!useCache) { + return expensiveSupplier.get(); + } + if (snapshot == null) { synchronized (updateMonitor) { if (snapshot == null) { diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java index 0a40555036c..67b4e890c29 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java @@ -5,16 +5,12 @@ import com.google.inject.Inject; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Timer; -import com.yahoo.vespa.applicationmodel.ApplicationInstance; -import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.service.duper.DuperModelManager; -import com.yahoo.vespa.service.health.HealthMonitorManager; import com.yahoo.vespa.service.manager.UnionMonitorManager; import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ServiceMonitor; -import com.yahoo.vespa.service.slobrok.SlobrokMonitorManagerImpl; - -import java.util.Map; public class ServiceMonitorImpl implements ServiceMonitor { @@ -25,7 +21,8 @@ public class ServiceMonitorImpl implements ServiceMonitor { UnionMonitorManager monitorManager, Metric metric, Timer timer, - Zone zone) { + Zone zone, + FlagSource flagSource) { duperModelManager.registerListener(monitorManager); ServiceModelProvider uncachedServiceModelProvider = new ServiceModelProvider( @@ -34,7 +31,8 @@ public class ServiceMonitorImpl implements ServiceMonitor { duperModelManager, new ModelGenerator(), zone); - serviceModelProvider = new ServiceModelCache(uncachedServiceModelProvider, timer); + boolean cache = Flags.SERVICE_MODEL_CACHE.bindTo(flagSource).value(); + serviceModelProvider = new ServiceModelCache(uncachedServiceModelProvider, timer, cache); } @Override diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java index d08bba2bd3d..f9e47b6b80a 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java @@ -27,4 +27,7 @@ public interface DuperModelInfraApi { /** Update the DuperModel: A supported infrastructure application has been removed or is not active. */ void infraApplicationRemoved(ApplicationId applicationId); + + /** All infra applications that are supposed to activate on config server bootstrap has been activated. */ + void infraApplicationsIsNowComplete(); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelListener.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelListener.java index a969b6c3f40..f664e5246ca 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelListener.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelListener.java @@ -1,34 +1,45 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.duper; +package com.yahoo.vespa.service.monitor; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.SuperModel; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.service.duper.DuperModel; /** * Interface for listening for changes to the {@link DuperModel}. * - * @author hakon + * @author hakonhall */ public interface DuperModelListener { /** * An application has been activated: * * <ul> - * <li>A synthetic application like the config server application has been added/"activated" + * <li>A synthetic application like the config server application has been added/activated * <li>A super model application has been activated (see * {@link com.yahoo.config.model.api.SuperModelListener#applicationActivated(SuperModel, ApplicationInfo) * SuperModelListener} * </ul> * - * No other threads will concurrently call any methods on this interface. + * <p>No other threads will concurrently call any methods on this interface.</p> */ void applicationActivated(ApplicationInfo application); /** * Application has been removed. * - * No other threads will concurrently call any methods on this interface. + * <p>No other threads will concurrently call any methods on this interface.</p> */ void applicationRemoved(ApplicationId id); + + /** + * During bootstrap of the config server, a number of applications are activated before + * resuming normal operations: The normal "tenant" application (making the super model) and + * the relevant infrastructure applications. Once all of these have been activated, this method + * will be invoked. + * + * <p>No other threads will concurrently call any methods on this interface.</p> + */ + void bootstrapComplete(); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelProvider.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelProvider.java new file mode 100644 index 00000000000..a90fa418054 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelProvider.java @@ -0,0 +1,6 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor; + +public interface DuperModelProvider { + void registerListener(DuperModelListener listener); +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java index e3ea48ca9fe..e7a8d33ea14 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java @@ -73,6 +73,10 @@ public class SlobrokMonitorManagerImpl implements SlobrokApi, MonitorManager { } @Override + public void bootstrapComplete() { + } + + @Override public List<Mirror.Entry> lookup(ApplicationId id, String pattern) { synchronized (monitor) { SlobrokMonitor slobrokMonitor = slobrokMonitors.get(id); diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java index 31fd266649a..dc90035be71 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.service.duper; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.service.monitor.DuperModelListener; import org.junit.Before; import org.junit.Test; diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java index 0f7c0dde357..3fb10f1f24e 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java @@ -47,7 +47,7 @@ public class ExampleModel { Map<ApplicationId, ApplicationInfo> applicationInfos = new HashMap<>(); applicationInfos.put(applicationInfo.getApplicationId(), applicationInfo); - return new SuperModel(applicationInfos); + return new SuperModel(applicationInfos, true); } public static ApplicationBuilder createApplication(String tenant, diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelCacheTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelCacheTest.java index 2d6921df374..c2314be1e0f 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelCacheTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelCacheTest.java @@ -18,7 +18,7 @@ public class ServiceModelCacheTest { @SuppressWarnings("unchecked") private final Supplier<ServiceModel> rawSupplier = mock(Supplier.class); private final Timer timer = mock(Timer.class); - private final ServiceModelCache cache = new ServiceModelCache(rawSupplier, timer); + private final ServiceModelCache cache = new ServiceModelCache(rawSupplier, timer, true); @Test public void sanityCheck() { diff --git a/storage/src/tests/distributor/garbagecollectiontest.cpp b/storage/src/tests/distributor/garbagecollectiontest.cpp index 65c1ac726b5..776cfc14d84 100644 --- a/storage/src/tests/distributor/garbagecollectiontest.cpp +++ b/storage/src/tests/distributor/garbagecollectiontest.cpp @@ -3,6 +3,7 @@ #include <vespa/storageapi/message/removelocation.h> #include <vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h> #include <vespa/storage/distributor/idealstatemanager.h> +#include <vespa/storage/distributor/idealstatemetricsset.h> #include <tests/distributor/distributortestutil.h> #include <vespa/storage/distributor/distributor.h> #include <vespa/document/test/make_document_bucket.h> @@ -35,11 +36,13 @@ struct GarbageCollectionOperationTest : Test, DistributorTestUtil { } // FIXME fragile to assume that send order == node index, but that's the way it currently works - void reply_to_nth_request(GarbageCollectionOperation& op, size_t n, uint32_t bucket_info_checksum) { + void reply_to_nth_request(GarbageCollectionOperation& op, size_t n, + uint32_t bucket_info_checksum, uint32_t n_docs_removed) { auto msg = _sender.command(n); assert(msg->getType() == api::MessageType::REMOVELOCATION); std::shared_ptr<api::StorageReply> reply(msg->makeReply()); auto& gc_reply = dynamic_cast<api::RemoveLocationReply&>(*reply); + gc_reply.set_documents_removed(n_docs_removed); gc_reply.setBucketInfo(api::BucketInfo(bucket_info_checksum, 90, 500)); op.receive(_sender, reply); @@ -56,6 +59,13 @@ struct GarbageCollectionOperationTest : Test, DistributorTestUtil { << entry->getNode(i)->getBucketInfo(); } } + + uint32_t gc_removed_documents_metric() { + auto metric_base = getIdealStateManager().getMetrics().operations[IdealStateOperation::GARBAGE_COLLECTION]; + auto gc_metrics = std::dynamic_pointer_cast<GcMetricSet>(metric_base); + assert(gc_metrics); + return gc_metrics->documents_removed.getValue(); + } }; TEST_F(GarbageCollectionOperationTest, simple) { @@ -63,29 +73,34 @@ TEST_F(GarbageCollectionOperationTest, simple) { op->start(_sender, framework::MilliSecTime(0)); ASSERT_EQ(2, _sender.commands().size()); + EXPECT_EQ(0u, gc_removed_documents_metric()); for (uint32_t i = 0; i < 2; ++i) { std::shared_ptr<api::StorageCommand> msg = _sender.command(i); ASSERT_EQ(msg->getType(), api::MessageType::REMOVELOCATION); auto& tmp = dynamic_cast<api::RemoveLocationCommand&>(*msg); EXPECT_EQ("music.date < 34", tmp.getDocumentSelection()); - reply_to_nth_request(*op, i, 777 + i); + reply_to_nth_request(*op, i, 777 + i, 50); } ASSERT_NO_FATAL_FAILURE(assert_bucket_db_contains({api::BucketInfo(777, 90, 500), api::BucketInfo(778, 90, 500)}, 34)); + EXPECT_EQ(50u, gc_removed_documents_metric()); } TEST_F(GarbageCollectionOperationTest, replica_bucket_info_not_added_to_db_until_all_replies_received) { auto op = create_op(); op->start(_sender, framework::MilliSecTime(0)); ASSERT_EQ(2, _sender.commands().size()); + EXPECT_EQ(0u, gc_removed_documents_metric()); // Respond to 1st request. Should _not_ cause bucket info to be merged into the database yet - reply_to_nth_request(*op, 0, 1234); + reply_to_nth_request(*op, 0, 1234, 70); ASSERT_NO_FATAL_FAILURE(assert_bucket_db_contains({api::BucketInfo(250, 50, 300), api::BucketInfo(250, 50, 300)}, 0)); // Respond to 2nd request. This _should_ cause bucket info to be merged into the database. - reply_to_nth_request(*op, 1, 4567); + reply_to_nth_request(*op, 1, 4567, 60); ASSERT_NO_FATAL_FAILURE(assert_bucket_db_contains({api::BucketInfo(1234, 90, 500), api::BucketInfo(4567, 90, 500)}, 34)); + + EXPECT_EQ(70u, gc_removed_documents_metric()); // Use max of received metrics } TEST_F(GarbageCollectionOperationTest, gc_bucket_info_does_not_overwrite_later_sequenced_bucket_info_writes) { @@ -93,10 +108,10 @@ TEST_F(GarbageCollectionOperationTest, gc_bucket_info_does_not_overwrite_later_s op->start(_sender, framework::MilliSecTime(0)); ASSERT_EQ(2, _sender.commands().size()); - reply_to_nth_request(*op, 0, 1234); + reply_to_nth_request(*op, 0, 1234, 0); // Change to replica on node 0 happens after GC op, but before GC info is merged into the DB. Must not be lost. insertBucketInfo(op->getBucketId(), 0, 7777, 100, 2000); - reply_to_nth_request(*op, 1, 4567); + reply_to_nth_request(*op, 1, 4567, 0); // Bucket info for node 0 is that of the later sequenced operation, _not_ from the earlier GC op. ASSERT_NO_FATAL_FAILURE(assert_bucket_db_contains({api::BucketInfo(7777, 100, 2000), api::BucketInfo(4567, 90, 500)}, 34)); } diff --git a/storage/src/tests/persistence/processalltest.cpp b/storage/src/tests/persistence/processalltest.cpp index 8c0f8853d2d..83f243ed1b2 100644 --- a/storage/src/tests/persistence/processalltest.cpp +++ b/storage/src/tests/persistence/processalltest.cpp @@ -23,11 +23,15 @@ TEST_F(ProcessAllHandlerTest, remove_location) { api::RemoveLocationCommand removeLocation("id.user == 4", makeDocumentBucket(bucketId)); ProcessAllHandler handler(getEnv(), getPersistenceProvider()); spi::Context context(documentapi::LoadType::DEFAULT, 0, 0); - handler.handleRemoveLocation(removeLocation, context); + auto tracker = handler.handleRemoveLocation(removeLocation, context); EXPECT_EQ("DocEntry(1234, 1, id:mail:testdoctype1:n=4:3619.html)\n" "DocEntry(2345, 1, id:mail:testdoctype1:n=4:4008.html)\n", dumpBucket(bucketId)); + + auto reply = std::dynamic_pointer_cast<api::RemoveLocationReply>(tracker->getReply()); + ASSERT_TRUE(reply.get() != nullptr); + EXPECT_EQ(2u, reply->documents_removed()); } TEST_F(ProcessAllHandlerTest, remove_location_document_subset) { @@ -44,7 +48,7 @@ TEST_F(ProcessAllHandlerTest, remove_location_document_subset) { api::RemoveLocationCommand removeLocation("testdoctype1.headerval % 2 == 0", makeDocumentBucket(bucketId)); spi::Context context(documentapi::LoadType::DEFAULT, 0, 0); - handler.handleRemoveLocation(removeLocation, context); + auto tracker = handler.handleRemoveLocation(removeLocation, context); EXPECT_EQ("DocEntry(100, 1, id:mail:testdoctype1:n=4:3619.html)\n" "DocEntry(101, 0, Doc(id:mail:testdoctype1:n=4:33113.html))\n" @@ -57,6 +61,10 @@ TEST_F(ProcessAllHandlerTest, remove_location_document_subset) { "DocEntry(108, 1, id:mail:testdoctype1:n=4:42967.html)\n" "DocEntry(109, 0, Doc(id:mail:testdoctype1:n=4:6925.html))\n", dumpBucket(bucketId)); + + auto reply = std::dynamic_pointer_cast<api::RemoveLocationReply>(tracker->getReply()); + ASSERT_TRUE(reply.get() != nullptr); + EXPECT_EQ(5u, reply->documents_removed()); } TEST_F(ProcessAllHandlerTest, remove_location_throws_exception_on_unknown_doc_type) { diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp index 61e67b40f44..c211e775326 100644 --- a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp +++ b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp @@ -4,8 +4,7 @@ #include <vespa/storage/distributor/distributormetricsset.h> #include <vespa/storage/distributor/idealstatemetricsset.h> -namespace storage { -namespace distributor { +namespace storage::distributor { BucketDBMetricUpdater::Stats::Stats() : _docCount(0), @@ -27,9 +26,7 @@ BucketDBMetricUpdater::BucketDBMetricUpdater() { } -BucketDBMetricUpdater::~BucketDBMetricUpdater() -{ -} +BucketDBMetricUpdater::~BucketDBMetricUpdater() = default; void BucketDBMetricUpdater::resetStats() @@ -148,5 +145,4 @@ BucketDBMetricUpdater::reset() resetStats(); } -} // distributor -} // storage +} // storage::distributor diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h index 6e15ee03d12..7ef8479866f 100644 --- a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h +++ b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h @@ -7,12 +7,9 @@ #include <unordered_map> -namespace storage { +namespace storage::distributor { class DistributorMetricSet; - -namespace distributor { - class IdealStateMetricSet; class BucketDBMetricUpdater { @@ -107,5 +104,4 @@ private: void resetStats(); }; -} // distributor -} // storage +} // storage::distributor diff --git a/storage/src/vespa/storage/distributor/distributorinterface.h b/storage/src/vespa/storage/distributor/distributorinterface.h index aba58e112dc..b17bcd56d19 100644 --- a/storage/src/vespa/storage/distributor/distributorinterface.h +++ b/storage/src/vespa/storage/distributor/distributorinterface.h @@ -12,10 +12,10 @@ namespace storage::api { class MergeBucketReply; } namespace storage::lib { class ClusterStateBundle; } namespace storage { class DistributorConfiguration; - class DistributorMetricSet; } namespace storage::distributor { +class DistributorMetricSet; class PendingMessageTracker; class DistributorInterface : public DistributorMessageSender diff --git a/storage/src/vespa/storage/distributor/distributormetricsset.cpp b/storage/src/vespa/storage/distributor/distributormetricsset.cpp index 244406ca6fb..98e96f9294f 100644 --- a/storage/src/vespa/storage/distributor/distributormetricsset.cpp +++ b/storage/src/vespa/storage/distributor/distributormetricsset.cpp @@ -3,7 +3,7 @@ #include <vespa/metrics/loadmetric.hpp> #include <vespa/metrics/summetric.hpp> -namespace storage { +namespace storage::distributor { using metrics::MetricSet; diff --git a/storage/src/vespa/storage/distributor/distributormetricsset.h b/storage/src/vespa/storage/distributor/distributormetricsset.h index 1e4730b8de6..b5be72e8c14 100644 --- a/storage/src/vespa/storage/distributor/distributormetricsset.h +++ b/storage/src/vespa/storage/distributor/distributormetricsset.h @@ -7,7 +7,7 @@ #include <vespa/metrics/metrics.h> #include <vespa/documentapi/loadtypes/loadtypeset.h> -namespace storage { +namespace storage::distributor { class DistributorMetricSet : public metrics::MetricSet { diff --git a/storage/src/vespa/storage/distributor/externaloperationhandler.h b/storage/src/vespa/storage/distributor/externaloperationhandler.h index 96875a3644a..60cad15a791 100644 --- a/storage/src/vespa/storage/distributor/externaloperationhandler.h +++ b/storage/src/vespa/storage/distributor/externaloperationhandler.h @@ -13,11 +13,11 @@ namespace storage { -class DistributorMetricSet; class PersistenceOperationMetricSet; namespace distributor { +class DistributorMetricSet; class Distributor; class MaintenanceOperationGenerator; class DirectDispatchSender; diff --git a/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp b/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp index d72f4a80ef4..fd193ad6fd8 100644 --- a/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp +++ b/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp @@ -6,7 +6,7 @@ namespace storage { namespace distributor { OperationMetricSet::OperationMetricSet(const std::string& name, metrics::Metric::Tags tags, const std::string& description, MetricSet* owner) - : MetricSet(name, tags, description, owner), + : MetricSet(name, std::move(tags), description, owner), pending("pending", {{"logdefault"},{"yamasdefault"}}, "The number of operations pending", this), @@ -16,14 +16,25 @@ OperationMetricSet::OperationMetricSet(const std::string& name, metrics::Metric: failed("done_failed", {{"logdefault"},{"yamasdefault"}}, "The number of operations that failed", this) -{ } +{} -OperationMetricSet::~OperationMetricSet() { } +OperationMetricSet::~OperationMetricSet() = default; + +GcMetricSet::GcMetricSet(const std::string& name, metrics::Metric::Tags tags, const std::string& description, MetricSet* owner) + : OperationMetricSet(name, std::move(tags), description, owner), + documents_removed("documents_removed", + {{"logdefault"},{"yamasdefault"}}, + "Number of documents removed by GC operations", this) +{} + +GcMetricSet::~GcMetricSet() = default; void IdealStateMetricSet::createOperationMetrics() { typedef IdealStateOperation ISO; operations.resize(ISO::OPERATION_COUNT); + // Note: naked new is used instead of make_shared due to the latter not being + // able to properly transitively deduce the types for the tag initializer lists. operations[ISO::DELETE_BUCKET] = std::shared_ptr<OperationMetricSet>( new OperationMetricSet("delete_bucket", {{"logdefault"},{"yamasdefault"}}, @@ -45,9 +56,9 @@ IdealStateMetricSet::createOperationMetrics() { {{"logdefault"},{"yamasdefault"}}, "Operations to set active/ready state for bucket copies", this)); operations[ISO::GARBAGE_COLLECTION] = std::shared_ptr<OperationMetricSet>( - new OperationMetricSet("garbage_collection", - {{"logdefault"},{"yamasdefault"}}, - "Operations to garbage collect data from buckets", this)); + new GcMetricSet("garbage_collection", + {{"logdefault"},{"yamasdefault"}}, + "Operations to garbage collect data from buckets", this)); } IdealStateMetricSet::IdealStateMetricSet() @@ -81,7 +92,7 @@ IdealStateMetricSet::IdealStateMetricSet() createOperationMetrics(); } -IdealStateMetricSet::~IdealStateMetricSet() { } +IdealStateMetricSet::~IdealStateMetricSet() = default; void IdealStateMetricSet::setPendingOperations(const std::vector<uint64_t>& newMetrics) { for (uint32_t i = 0; i < IdealStateOperation::OPERATION_COUNT; i++) { diff --git a/storage/src/vespa/storage/distributor/idealstatemetricsset.h b/storage/src/vespa/storage/distributor/idealstatemetricsset.h index 7bb472b4a2c..2679da17598 100644 --- a/storage/src/vespa/storage/distributor/idealstatemetricsset.h +++ b/storage/src/vespa/storage/distributor/idealstatemetricsset.h @@ -16,13 +16,21 @@ public: metrics::LongCountMetric failed; OperationMetricSet(const std::string& name, metrics::Metric::Tags tags, const std::string& description, MetricSet* owner); - ~OperationMetricSet(); + ~OperationMetricSet() override; +}; + +struct GcMetricSet : OperationMetricSet { + metrics::LongCountMetric documents_removed; + + GcMetricSet(const std::string& name, metrics::Metric::Tags tags, + const std::string& description, MetricSet* owner); + ~GcMetricSet() override; }; class IdealStateMetricSet : public metrics::MetricSet { public: - std::vector<std::shared_ptr<OperationMetricSet> > operations; + std::vector<std::shared_ptr<OperationMetricSet>> operations; metrics::LongValueMetric idealstate_diff; metrics::LongValueMetric buckets_toofewcopies; metrics::LongValueMetric buckets_toomanycopies; @@ -35,7 +43,7 @@ public: void createOperationMetrics(); IdealStateMetricSet(); - ~IdealStateMetricSet(); + ~IdealStateMetricSet() override; void setPendingOperations(const std::vector<uint64_t>& newMetrics); }; diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp index c674add80f7..fc127c2e0eb 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp @@ -2,6 +2,7 @@ #include "garbagecollectionoperation.h" #include <vespa/storage/distributor/idealstatemanager.h> +#include <vespa/storage/distributor/idealstatemetricsset.h> #include <vespa/storage/distributor/distributor.h> #include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/storageapi/message/removelocation.h> @@ -9,19 +10,18 @@ #include <vespa/log/log.h> LOG_SETUP(".distributor.operation.idealstate.remove"); -using namespace storage::distributor; +namespace storage::distributor { GarbageCollectionOperation::GarbageCollectionOperation(const std::string& clusterName, const BucketAndNodes& nodes) : IdealStateOperation(nodes), _tracker(clusterName), - _replica_info() + _replica_info(), + _max_documents_removed(0) {} GarbageCollectionOperation::~GarbageCollectionOperation() = default; -void -GarbageCollectionOperation::onStart(DistributorMessageSender& sender) -{ +void GarbageCollectionOperation::onStart(DistributorMessageSender& sender) { BucketDatabase::Entry entry = _bucketSpace->getBucketDatabase().get(getBucketId()); std::vector<uint16_t> nodes = entry->getNodes(); @@ -43,7 +43,7 @@ GarbageCollectionOperation::onStart(DistributorMessageSender& sender) void GarbageCollectionOperation::onReceive(DistributorMessageSender&, - const std::shared_ptr<api::StorageReply>& reply) + const std::shared_ptr<api::StorageReply>& reply) { auto* rep = dynamic_cast<api::RemoveLocationReply*>(reply.get()); assert(rep != nullptr); @@ -53,6 +53,7 @@ GarbageCollectionOperation::onReceive(DistributorMessageSender&, if (!rep->getResult().failed()) { _replica_info.emplace_back(_manager->getDistributorComponent().getUniqueTimestamp(), node, rep->getBucketInfo()); + _max_documents_removed = std::max(_max_documents_removed, rep->documents_removed()); } else { _ok = false; } @@ -61,6 +62,7 @@ GarbageCollectionOperation::onReceive(DistributorMessageSender&, if (_ok) { merge_received_bucket_info_into_db(); } + update_gc_metrics(); done(); } } @@ -76,8 +78,16 @@ void GarbageCollectionOperation::merge_received_bucket_info_into_db() { } } +void GarbageCollectionOperation::update_gc_metrics() { + auto metric_base = _manager->getMetrics().operations[IdealStateOperation::GARBAGE_COLLECTION]; + auto gc_metrics = std::dynamic_pointer_cast<GcMetricSet>(metric_base); + assert(gc_metrics); + gc_metrics->documents_removed.inc(_max_documents_removed); +} + bool -GarbageCollectionOperation::shouldBlockThisOperation(uint32_t, uint8_t) const -{ +GarbageCollectionOperation::shouldBlockThisOperation(uint32_t, uint8_t) const { return true; } + +} diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h index 47ea11bb328..28de9592a63 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h @@ -26,8 +26,10 @@ protected: MessageTracker _tracker; private: std::vector<BucketCopy> _replica_info; + uint32_t _max_documents_removed; void merge_received_bucket_info_into_db(); + void update_gc_metrics(); }; } diff --git a/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h b/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h index d951d7ceba2..1299fdad2ad 100644 --- a/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h +++ b/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h @@ -16,7 +16,7 @@ class PersistenceFailuresMetricSet : public metrics::MetricSet { public: explicit PersistenceFailuresMetricSet(metrics::MetricSet* owner); - ~PersistenceFailuresMetricSet(); + ~PersistenceFailuresMetricSet() override; metrics::SumMetric<metrics::LongCountMetric> sum; metrics::LongCountMetric notready; @@ -44,7 +44,7 @@ public: PersistenceFailuresMetricSet failures; PersistenceOperationMetricSet(const std::string& name, metrics::MetricSet* owner = nullptr); - ~PersistenceOperationMetricSet(); + ~PersistenceOperationMetricSet() override; MetricSet * clone(std::vector<Metric::UP>& ownerList, CopyType copyType, metrics::MetricSet* owner, bool includeUnused) const override; diff --git a/storage/src/vespa/storage/persistence/processallhandler.cpp b/storage/src/vespa/storage/persistence/processallhandler.cpp index 8c951a9f50d..5b94a3da027 100644 --- a/storage/src/vespa/storage/persistence/processallhandler.cpp +++ b/storage/src/vespa/storage/persistence/processallhandler.cpp @@ -23,6 +23,7 @@ public: spi::PersistenceProvider& _provider; const spi::Bucket& _bucket; spi::Context& _context; + uint32_t _n_removed; UnrevertableRemoveEntryProcessor( spi::PersistenceProvider& provider, @@ -30,7 +31,9 @@ public: spi::Context& context) : _provider(provider), _bucket(bucket), - _context(context) {} + _context(context), + _n_removed(0) + {} void process(spi::DocEntry& entry) override { spi::RemoveResult removeResult = _provider.remove( @@ -45,13 +48,14 @@ public: << removeResult.getErrorMessage(); throw std::runtime_error(ss.str()); } + ++_n_removed; } }; class StatEntryProcessor : public BucketProcessor::EntryProcessor { public: std::ostream& ost; - StatEntryProcessor(std::ostream& o) + explicit StatEntryProcessor(std::ostream& o) : ost(o) {}; void process(spi::DocEntry& e) override { @@ -97,7 +101,9 @@ ProcessAllHandler::handleRemoveLocation(api::RemoveLocationCommand& cmd, context); spi::Result result = _spi.flush(bucket, context); uint32_t code = _env.convertErrorCode(result); - if (code != 0) { + if (code == 0) { + tracker->setReply(std::make_shared<api::RemoveLocationReply>(cmd, processor._n_removed)); + } else { tracker->fail(code, result.getErrorMessage()); } diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp index 2f959e40e2a..2e5eb115844 100644 --- a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp +++ b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp @@ -522,8 +522,15 @@ TEST_P(StorageProtocolTest, remove_location) { EXPECT_EQ("id.group == \"mygroup\"", cmd2->getDocumentSelection()); EXPECT_EQ(_bucket, cmd2->getBucket()); - auto reply = std::make_shared<RemoveLocationReply>(*cmd2); + uint32_t n_docs_removed = 12345; + auto reply = std::make_shared<RemoveLocationReply>(*cmd2, n_docs_removed); auto reply2 = copyReply(reply); + if (GetParam().getMajor() == 7) { + // Statistics are only available for protobuf-enabled version. + EXPECT_EQ(n_docs_removed, reply2->documents_removed()); + } else { + EXPECT_EQ(0, reply2->documents_removed()); + } } TEST_P(StorageProtocolTest, create_visitor) { diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto b/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto index 810f88f588f..12dbaf59146 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto +++ b/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto @@ -90,7 +90,12 @@ message RemoveLocationRequest { bytes document_selection = 2; } +message RemoveLocationStats { + uint32 documents_removed = 1; +} + message RemoveLocationResponse { BucketInfo bucket_info = 1; BucketId remapped_bucket_id = 2; + RemoveLocationStats stats = 3; } diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp index 9751fd1be98..90c8d1c7d2a 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp @@ -643,7 +643,9 @@ void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveLocationComma } void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveLocationReply& msg) const { - encode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, msg, no_op_encode); + encode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, msg, [&](auto& res) { + res.mutable_stats()->set_documents_removed(msg.documents_removed()); + }); } api::StorageCommand::UP ProtocolSerialization7::onDecodeRemoveLocationCommand(BBuf& buf) const { @@ -653,8 +655,11 @@ api::StorageCommand::UP ProtocolSerialization7::onDecodeRemoveLocationCommand(BB } api::StorageReply::UP ProtocolSerialization7::onDecodeRemoveLocationReply(const SCmd& cmd, BBuf& buf) const { - return decode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, [&]([[maybe_unused]] auto& res) { - return std::make_unique<api::RemoveLocationReply>(static_cast<const api::RemoveLocationCommand&>(cmd)); + return decode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, [&](auto& res) { + uint32_t documents_removed = (res.has_stats() ? res.stats().documents_removed() : 0u); + return std::make_unique<api::RemoveLocationReply>( + static_cast<const api::RemoveLocationCommand&>(cmd), + documents_removed); }); } diff --git a/storageapi/src/vespa/storageapi/message/removelocation.cpp b/storageapi/src/vespa/storageapi/message/removelocation.cpp index b53584601ef..49c9d22f5ee 100644 --- a/storageapi/src/vespa/storageapi/message/removelocation.cpp +++ b/storageapi/src/vespa/storageapi/message/removelocation.cpp @@ -25,8 +25,9 @@ RemoveLocationCommand::print(std::ostream& out, bool verbose, const std::string& BucketInfoCommand::print(out, verbose, indent); } -RemoveLocationReply::RemoveLocationReply(const RemoveLocationCommand& cmd) - : BucketInfoReply(cmd) +RemoveLocationReply::RemoveLocationReply(const RemoveLocationCommand& cmd, uint32_t docs_removed) + : BucketInfoReply(cmd), + _documents_removed(docs_removed) { } diff --git a/storageapi/src/vespa/storageapi/message/removelocation.h b/storageapi/src/vespa/storageapi/message/removelocation.h index 46555497035..812cc8c413b 100644 --- a/storageapi/src/vespa/storageapi/message/removelocation.h +++ b/storageapi/src/vespa/storageapi/message/removelocation.h @@ -11,7 +11,7 @@ class RemoveLocationCommand : public BucketInfoCommand { public: RemoveLocationCommand(vespalib::stringref documentSelection, const document::Bucket &bucket); - ~RemoveLocationCommand(); + ~RemoveLocationCommand() override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; const vespalib::string& getDocumentSelection() const { return _documentSelection; } @@ -22,8 +22,13 @@ private: class RemoveLocationReply : public BucketInfoReply { + uint32_t _documents_removed; public: - RemoveLocationReply(const RemoveLocationCommand& cmd); + explicit RemoveLocationReply(const RemoveLocationCommand& cmd, uint32_t docs_removed = 0); + void set_documents_removed(uint32_t docs_removed) noexcept { + _documents_removed = docs_removed; + } + uint32_t documents_removed() const noexcept { return _documents_removed; } DECLARE_STORAGEREPLY(RemoveLocationReply, onRemoveLocationReply) }; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java index 4cc92828b0e..97a04e21d4b 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java @@ -80,6 +80,7 @@ public abstract class ClientBase implements AutoCloseable { .setRetryHandler(new DefaultHttpRequestRetryHandler(3, /*requestSentRetryEnabled*/true)) .setUserAgent(userAgent) .setSSLSocketFactory(new SSLConnectionSocketFactory(new ServiceIdentitySslSocketFactory(sslContextSupplier), hostnameVerifier)) + .setMaxConnPerRoute(8) .setDefaultRequestConfig(RequestConfig.custom() .setConnectTimeout((int) Duration.ofSeconds(10).toMillis()) .setConnectionRequestTimeout((int)Duration.ofSeconds(10).toMillis()) diff --git a/vespabase/src/rhel-prestart.sh b/vespabase/src/rhel-prestart.sh index 3a770cdd785..d84122898a3 100755 --- a/vespabase/src/rhel-prestart.sh +++ b/vespabase/src/rhel-prestart.sh @@ -96,32 +96,32 @@ fixdir () { # BEGIN directory fixups -fixdir root wheel 1777 logs -fixdir root wheel 1777 tmp -fixdir root wheel 1777 var/run -fixdir ${VESPA_USER} wheel 1777 var/crash -fixdir ${VESPA_USER} wheel 1777 logs/vespa -fixdir ${VESPA_USER} wheel 1777 tmp/vespa -fixdir ${VESPA_USER} wheel 755 var -fixdir ${VESPA_USER} wheel 755 libexec/vespa/plugins/qrs -fixdir ${VESPA_USER} wheel 755 logs/vespa/configserver -fixdir ${VESPA_USER} wheel 755 logs/vespa/qrs -fixdir ${VESPA_USER} wheel 755 logs/vespa/search -fixdir ${VESPA_USER} wheel 755 var/db/vespa -fixdir ${VESPA_USER} wheel 755 var/db/vespa/tmp -fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server -fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server/serverdb -fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server/serverdb/tenants -fixdir ${VESPA_USER} wheel 755 var/db/vespa/filedistribution -fixdir ${VESPA_USER} wheel 755 var/db/vespa/index -fixdir ${VESPA_USER} wheel 755 var/db/vespa/logcontrol -fixdir ${VESPA_USER} wheel 755 var/db/vespa/search -fixdir ${VESPA_USER} wheel 755 var/jdisc_container -fixdir ${VESPA_USER} wheel 755 var/vespa -fixdir ${VESPA_USER} wheel 755 var/vespa/application -fixdir ${VESPA_USER} wheel 755 var/vespa/bundlecache -fixdir ${VESPA_USER} wheel 755 var/vespa/bundlecache/configserver -fixdir ${VESPA_USER} wheel 755 var/vespa/cache/config/ +fixdir root root 1777 logs +fixdir root root 1777 tmp +fixdir root root 1777 var/run +fixdir ${VESPA_USER} root 1777 var/crash +fixdir ${VESPA_USER} root 1777 logs/vespa +fixdir ${VESPA_USER} root 1777 tmp/vespa +fixdir root root 755 var +fixdir ${VESPA_USER} root 755 libexec/vespa/plugins/qrs +fixdir ${VESPA_USER} root 755 logs/vespa/configserver +fixdir ${VESPA_USER} root 755 logs/vespa/qrs +fixdir ${VESPA_USER} root 755 logs/vespa/search +fixdir ${VESPA_USER} root 755 var/db/vespa +fixdir ${VESPA_USER} root 755 var/db/vespa/tmp +fixdir ${VESPA_USER} root 755 var/db/vespa/config_server +fixdir ${VESPA_USER} root 755 var/db/vespa/config_server/serverdb +fixdir ${VESPA_USER} root 755 var/db/vespa/config_server/serverdb/tenants +fixdir ${VESPA_USER} root 755 var/db/vespa/filedistribution +fixdir ${VESPA_USER} root 755 var/db/vespa/index +fixdir ${VESPA_USER} root 755 var/db/vespa/logcontrol +fixdir ${VESPA_USER} root 755 var/db/vespa/search +fixdir ${VESPA_USER} root 755 var/jdisc_container +fixdir ${VESPA_USER} root 755 var/vespa +fixdir ${VESPA_USER} root 755 var/vespa/application +fixdir ${VESPA_USER} root 755 var/vespa/bundlecache +fixdir ${VESPA_USER} root 755 var/vespa/bundlecache/configserver +fixdir ${VESPA_USER} root 755 var/vespa/cache/config/ if [ "${VESPA_UNPRIVILEGED}" != yes ]; then chown -hR ${VESPA_USER} logs/vespa diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index b8b6716d879..04f859e2802 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -3022,7 +3022,8 @@ "public static boolean isTextCharacter(int)", "public static java.util.OptionalInt validateTextString(java.lang.String)", "public static boolean isDisplayable(int)", - "public static java.lang.String stripInvalidCharacters(java.lang.String)" + "public static java.lang.String stripInvalidCharacters(java.lang.String)", + "public static java.lang.String truncate(java.lang.String, int)" ], "fields": [] }, diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/CachedThreadPoolWithFallback.java b/vespajlib/src/main/java/com/yahoo/concurrent/CachedThreadPoolWithFallback.java new file mode 100644 index 00000000000..42e86aad1ba --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/CachedThreadPoolWithFallback.java @@ -0,0 +1,63 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.concurrent; + +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * An executor that will first try a bounded cached threadpool before falling back to a unbounded + * single threaded threadpool that will take over dispatching to the primary pool. + * + */ +public class CachedThreadPoolWithFallback implements AutoCloseable, Executor { + private final ExecutorService primary; + private final ExecutorService secondary; + public CachedThreadPoolWithFallback(String baseName, int corePoolSize, int maximumPoolSize, long keepAlimeTime, TimeUnit timeUnit) { + primary = new ThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAlimeTime, timeUnit, + new SynchronousQueue<>(), ThreadFactoryFactory.getDaemonThreadFactory(baseName + ".primary")); + secondary = Executors.newSingleThreadExecutor(ThreadFactoryFactory.getDaemonThreadFactory(baseName + ".secondary")); + } + @Override + public void execute(Runnable command) { + try { + primary.execute(command); + } catch (RejectedExecutionException e1) { + secondary.execute(() -> retryForever(command)); + } + } + private void retryForever(Runnable command) { + while (true) { + try { + primary.execute(command); + return; + } catch (RejectedExecutionException rejected) { + try { + Thread.sleep(1); + } catch (InterruptedException silenced) { } + } + } + } + + @Override + public void close() { + secondary.shutdown(); + join(secondary); + primary.shutdown(); + join(primary); + } + private static void join(ExecutorService executor) { + while (true) { + try { + if (executor.awaitTermination(60, TimeUnit.SECONDS)) { + return; + } + } catch (InterruptedException e) {} + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/text/Text.java b/vespajlib/src/main/java/com/yahoo/text/Text.java index 706fd1583a3..85b28639d89 100644 --- a/vespajlib/src/main/java/com/yahoo/text/Text.java +++ b/vespajlib/src/main/java/com/yahoo/text/Text.java @@ -174,4 +174,16 @@ public final class Text { return stripped != null ? stripped.toString() : string; } + /** + * Returns a string which is never larger than the given number of characters. + * If the string is longer than the given length it will be truncated. + * If length is 4 or less the string will be truncated to length. + * If length is longer than 4, it will be truncated at length-4 with " ..." added at the end. + */ + public static String truncate(String s, int length) { + if (s.length() <= length) return s; + if (length <= 4) return s.substring(0, length); + return s.substring(0, length - 4) + " ..."; + } + } diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/CachedThreadPoolWithFallbackTest.java b/vespajlib/src/test/java/com/yahoo/concurrent/CachedThreadPoolWithFallbackTest.java new file mode 100644 index 00000000000..52e17631a34 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/concurrent/CachedThreadPoolWithFallbackTest.java @@ -0,0 +1,43 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.concurrent; + +import org.junit.Test; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.Assert.assertEquals; + +public class CachedThreadPoolWithFallbackTest { + private static void countAndBlock(AtomicLong counter, long waitLimit) { + counter.incrementAndGet(); + try { + synchronized (counter) { + while (counter.get() < waitLimit) { + counter.wait(); + } + } + } catch (InterruptedException e) {} + } + + @Test + public void testThatTaskAreQueued() throws InterruptedException { + CachedThreadPoolWithFallback executor = new CachedThreadPoolWithFallback("test", 1, 30, 1, TimeUnit.SECONDS); + AtomicLong counter = new AtomicLong(0); + for (int i = 0; i < 1000; i++) { + executor.execute(() -> countAndBlock(counter, 100)); + } + while (counter.get() < 30) { + Thread.sleep(1); + } + Thread.sleep(1); + assertEquals(30L, counter.get()); + counter.set(100); + synchronized (counter) { + counter.notifyAll(); + } + executor.close(); + assertEquals(1070L, counter.get()); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java b/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java index e733b838c39..8bb8b2aaad5 100644 --- a/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java @@ -61,4 +61,15 @@ public class TextTestCase { assertFalse(Text.isDisplayable(0)); } + @Test + public void testTruncate() { + assertEquals("ab", Text.truncate("ab", 5)); + assertEquals("ab", Text.truncate("ab", 6)); + assertEquals("ab", Text.truncate("ab", 2)); + assertEquals("a", Text.truncate("ab", 1)); + assertEquals("", Text.truncate("ab", 0)); + assertEquals("ab c", Text.truncate("ab cde", 4)); + assertEquals("a ...", Text.truncate("ab cde", 5)); + } + } diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 9339cdacea0..979184acbae 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -30,6 +30,7 @@ vespa_define_module( src/tests/component src/tests/compress src/tests/compression + src/tests/crypto src/tests/data/databuffer src/tests/data/input_reader src/tests/data/lz4_encode_decode @@ -139,6 +140,7 @@ vespa_define_module( src/vespa/vespalib src/vespa/vespalib/btree src/vespa/vespalib/component + src/vespa/vespalib/crypto src/vespa/vespalib/data src/vespa/vespalib/data/slime src/vespa/vespalib/datastore diff --git a/vespalib/src/tests/crypto/CMakeLists.txt b/vespalib/src/tests/crypto/CMakeLists.txt new file mode 100644 index 00000000000..b930b5715b5 --- /dev/null +++ b/vespalib/src/tests/crypto/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_crypto_crypto_test_app TEST + SOURCES + crypto_test.cpp + DEPENDS + vespalib + gtest +) +vespa_add_test(NAME vespalib_crypto_crypto_test_app COMMAND vespalib_crypto_crypto_test_app) + diff --git a/vespalib/src/tests/crypto/crypto_test.cpp b/vespalib/src/tests/crypto/crypto_test.cpp new file mode 100644 index 00000000000..8daba954793 --- /dev/null +++ b/vespalib/src/tests/crypto/crypto_test.cpp @@ -0,0 +1,37 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/crypto/private_key.h> +#include <vespa/vespalib/crypto/x509_certificate.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <gmock/gmock.h> + +using namespace ::testing; + +namespace vespalib::crypto { + +// FIXME these tests are very high level and simple since the current crypto utility API we provide +// is extremely simple and does not support loading PEMs, signing or verifying. + +TEST(CryptoTest, generated_p256_ec_private_key_can_be_exported_to_pem_format) { + auto key = PrivateKey::generate_p256_ec_key(); + auto pem = key->private_to_pem(); + EXPECT_THAT(pem, StartsWith("-----BEGIN PRIVATE KEY-----")); +} + +TEST(CryptoTest, generated_x509_certificate_can_be_exported_to_pem_format) { + auto dn = X509Certificate::DistinguishedName() + .country("NO").locality("Trondheim") + .organization("Cool Unit Test Writers") + .organizational_unit("Only the finest tests, yes") + .add_common_name("cooltests.example.com"); + auto subject = X509Certificate::SubjectInfo(std::move(dn)); + auto key = PrivateKey::generate_p256_ec_key(); + auto params = X509Certificate::Params::self_signed(std::move(subject), key); + auto cert = X509Certificate::generate_from(std::move(params)); + auto pem = cert->to_pem(); + EXPECT_THAT(pem, StartsWith("-----BEGIN CERTIFICATE-----")); +} + +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/net/tls/direct_buffer_bio/direct_buffer_bio_test.cpp b/vespalib/src/tests/net/tls/direct_buffer_bio/direct_buffer_bio_test.cpp index 134df9b17eb..a51cfb688c1 100644 --- a/vespalib/src/tests/net/tls/direct_buffer_bio/direct_buffer_bio_test.cpp +++ b/vespalib/src/tests/net/tls/direct_buffer_bio/direct_buffer_bio_test.cpp @@ -5,6 +5,7 @@ #include <cassert> using namespace vespalib; +using namespace vespalib::crypto; using namespace vespalib::net::tls::impl; struct Fixture { diff --git a/vespalib/src/tests/net/tls/openssl_impl/CMakeLists.txt b/vespalib/src/tests/net/tls/openssl_impl/CMakeLists.txt index e8f77d36e16..799e2291d7c 100644 --- a/vespalib/src/tests/net/tls/openssl_impl/CMakeLists.txt +++ b/vespalib/src/tests/net/tls/openssl_impl/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(vespalib_net_tls_openssl_impl_test_app TEST SOURCES - crypto_utils.cpp openssl_impl_test.cpp DEPENDS vespalib diff --git a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp index 54c8c19fc64..4586beef910 100644 --- a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp +++ b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp @@ -1,5 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "crypto_utils.h" +#include <vespa/vespalib/crypto/private_key.h> +#include <vespa/vespalib/crypto/x509_certificate.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/smart_buffer.h> #include <vespa/vespalib/net/tls/authorization_mode.h> @@ -11,11 +12,11 @@ #include <vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h> #include <vespa/vespalib/test/make_tls_options_for_testing.h> #include <vespa/vespalib/test/peer_policy_utils.h> -#include <iostream> #include <stdexcept> #include <stdlib.h> using namespace vespalib; +using namespace vespalib::crypto; using namespace vespalib::net::tls; using namespace vespalib::net::tls::impl; diff --git a/vespalib/src/vespa/vespalib/CMakeLists.txt b/vespalib/src/vespa/vespalib/CMakeLists.txt index 4249f6333a4..95f6a407914 100644 --- a/vespalib/src/vespa/vespalib/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(vespalib SOURCES $<TARGET_OBJECTS:vespalib_vespalib_btree> $<TARGET_OBJECTS:vespalib_vespalib_component> + $<TARGET_OBJECTS:vespalib_vespalib_crypto> $<TARGET_OBJECTS:vespalib_vespalib_data> $<TARGET_OBJECTS:vespalib_vespalib_data_slime> $<TARGET_OBJECTS:vespalib_vespalib_datastore> diff --git a/vespalib/src/vespa/vespalib/crypto/CMakeLists.txt b/vespalib/src/vespa/vespalib/crypto/CMakeLists.txt new file mode 100644 index 00000000000..6000156fcfa --- /dev/null +++ b/vespalib/src/vespa/vespalib/crypto/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(vespalib_vespalib_crypto OBJECT + SOURCES + crypto_exception.cpp + openssl_crypto_impl.cpp + private_key.cpp + x509_certificate.cpp + DEPENDS +) +find_package(OpenSSL) +target_include_directories(vespalib_vespalib_crypto PUBLIC ${OPENSSL_INCLUDE_DIR}) diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_exception.cpp b/vespalib/src/vespa/vespalib/crypto/crypto_exception.cpp index 41bb2060c04..226d8664de6 100644 --- a/vespalib/src/vespa/vespalib/net/tls/crypto_exception.cpp +++ b/vespalib/src/vespa/vespalib/crypto/crypto_exception.cpp @@ -2,7 +2,7 @@ #include "crypto_exception.h" -namespace vespalib::net::tls { +namespace vespalib::crypto { VESPA_IMPLEMENT_EXCEPTION(CryptoException, Exception); diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_exception.h b/vespalib/src/vespa/vespalib/crypto/crypto_exception.h index 696a158e058..0d0dcc8ceec 100644 --- a/vespalib/src/vespa/vespalib/net/tls/crypto_exception.h +++ b/vespalib/src/vespa/vespalib/crypto/crypto_exception.h @@ -3,7 +3,7 @@ #include <vespa/vespalib/util/exception.h> -namespace vespalib::net::tls { +namespace vespalib::crypto { VESPA_DEFINE_EXCEPTION(CryptoException, Exception); diff --git a/vespalib/src/tests/net/tls/openssl_impl/crypto_utils.cpp b/vespalib/src/vespa/vespalib/crypto/openssl_crypto_impl.cpp index 14755360b51..72efbb841c2 100644 --- a/vespalib/src/tests/net/tls/openssl_impl/crypto_utils.cpp +++ b/vespalib/src/vespa/vespalib/crypto/openssl_crypto_impl.cpp @@ -1,13 +1,12 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "crypto_utils.h" -#include <vespa/vespalib/net/tls/crypto_exception.h> +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "openssl_crypto_impl.h" +#include <vespa/vespalib/crypto/crypto_exception.h> #include <cassert> #include <openssl/bn.h> #include <openssl/rand.h> #include <openssl/x509v3.h> -namespace vespalib::net::tls::impl { +namespace vespalib::crypto::openssl_impl { namespace { @@ -61,6 +60,77 @@ BioPtr new_memory_bio() { return bio; } +} // anonymous namespace + +vespalib::string PrivateKeyImpl::private_to_pem() const { + BioPtr bio = new_memory_bio(); + // TODO this API is const-broken even on 1.1.1, revisit in the future... + auto* mutable_pkey = const_cast<::EVP_PKEY*>(_pkey.get()); + if (::PEM_write_bio_PrivateKey(bio.get(), mutable_pkey, nullptr, nullptr, + 0, nullptr, nullptr) != 1) { + throw CryptoException("PEM_write_bio_PrivateKey"); + } + return bio_to_string(*bio); +} + +std::shared_ptr<PrivateKeyImpl> PrivateKeyImpl::generate_openssl_p256_ec_key() { + // We first have to generate an EVP context for the keygen _parameters_... + EvpPkeyCtxPtr params_ctx(::EVP_PKEY_CTX_new_id(EVP_PKEY_EC, nullptr)); + if (!params_ctx) { + throw CryptoException("EVP_PKEY_CTX_new_id"); + } + if (::EVP_PKEY_paramgen_init(params_ctx.get()) != 1) { + throw CryptoException("EVP_PKEY_paramgen_init"); + } + // Set EC keygen parameters to use prime256v1, which is the same as P-256 + if (EVP_PKEY_CTX_set_ec_paramgen_curve_nid(params_ctx.get(), NID_X9_62_prime256v1) <= 0) { + throw CryptoException("EVP_PKEY_CTX_set_ec_paramgen_curve_nid"); + } +#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) + // Must tag _explicitly_ as a named curve or many won't accept our pretty keys. + // If we don't do this, explicit curve parameters are included with the key, + // and this is not widely supported nor needed since we're generating a key on + // a standardized curve. + if (EVP_PKEY_CTX_set_ec_param_enc(params_ctx.get(), OPENSSL_EC_NAMED_CURVE) <= 0) { + throw CryptoException("EVP_PKEY_CTX_set_ec_param_enc"); + } +#endif + // Note: despite being an EVP_PKEY this is not an actual key, just key parameters! + ::EVP_PKEY* params_raw = nullptr; + if (::EVP_PKEY_paramgen(params_ctx.get(), ¶ms_raw) != 1) { + throw CryptoException("EVP_PKEY_paramgen"); + } + EvpPkeyPtr params(params_raw); + // Now we can create a context for the proper key generation + EvpPkeyCtxPtr key_ctx(::EVP_PKEY_CTX_new(params.get(), nullptr)); + if (!params_ctx) { + throw CryptoException("EVP_PKEY_CTX_new"); + } + if (::EVP_PKEY_keygen_init(key_ctx.get()) != 1) { + throw CryptoException("EVP_PKEY_keygen_init"); + } + // Finally, it's time to generate the key pair itself. + ::EVP_PKEY* pkey_raw = nullptr; + if (::EVP_PKEY_keygen(key_ctx.get(), &pkey_raw) != 1) { + throw CryptoException("EVP_PKEY_keygen"); + } + EvpPkeyPtr generated_key(pkey_raw); +#if (OPENSSL_VERSION_NUMBER < 0x10100000L) + // On OpenSSL versions prior to 1.1.0, we must set the named curve ASN1 flag + // directly on the EC_KEY, as the EVP_PKEY wrapper doesn't exist (this is a + // half truth, as it exists on 1.0.2 stable, but not necessarily on all 1.0.2 + // versions, and certainly not on 1.0.1). + EcKeyPtr ec_key(::EVP_PKEY_get1_EC_KEY(generated_key.get())); // Bumps ref count, needs free + if (!ec_key) { + throw CryptoException("EVP_PKEY_get1_EC_KEY"); + } + ::EC_KEY_set_asn1_flag(ec_key.get(), OPENSSL_EC_NAMED_CURVE); +#endif + return std::make_shared<PrivateKeyImpl>(std::move(generated_key), Type::EC); +} + +namespace { + void assign_random_positive_serial_number(::X509& cert) { /* * From RFC3280, section 4.1.2.2: @@ -165,88 +235,23 @@ void add_any_subject_alternate_names(::X509& subject, ::X509& issuer, } } -} // anon ns - -vespalib::string PrivateKey::private_to_pem() const { - BioPtr bio = new_memory_bio(); - // TODO this API is const-broken even on 1.1.1, revisit in the future... - auto* mutable_pkey = const_cast<::EVP_PKEY*>(_pkey.get()); - if (::PEM_write_bio_PrivateKey(bio.get(), mutable_pkey, nullptr, nullptr, - 0, nullptr, nullptr) != 1) { - throw CryptoException("PEM_write_bio_PrivateKey"); - } - return bio_to_string(*bio); -} - -std::shared_ptr<PrivateKey> PrivateKey::generate_p256_ec_key() { - // We first have to generate an EVP context for the keygen _parameters_... - EvpPkeyCtxPtr params_ctx(::EVP_PKEY_CTX_new_id(EVP_PKEY_EC, nullptr)); - if (!params_ctx) { - throw CryptoException("EVP_PKEY_CTX_new_id"); - } - if (::EVP_PKEY_paramgen_init(params_ctx.get()) != 1) { - throw CryptoException("EVP_PKEY_paramgen_init"); - } - // Set EC keygen parameters to use prime256v1, which is the same as P-256 - if (EVP_PKEY_CTX_set_ec_paramgen_curve_nid(params_ctx.get(), NID_X9_62_prime256v1) <= 0) { - throw CryptoException("EVP_PKEY_CTX_set_ec_paramgen_curve_nid"); - } -#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) - // Must tag _explicitly_ as a named curve or many won't accept our pretty keys. - // If we don't do this, explicit curve parameters are included with the key, - // and this is not widely supported nor needed since we're generating a key on - // a standardized curve. - if (EVP_PKEY_CTX_set_ec_param_enc(params_ctx.get(), OPENSSL_EC_NAMED_CURVE) <= 0) { - throw CryptoException("EVP_PKEY_CTX_set_ec_param_enc"); - } -#endif - // Note: despite being an EVP_PKEY this is not an actual key, just key parameters! - ::EVP_PKEY* params_raw = nullptr; - if (::EVP_PKEY_paramgen(params_ctx.get(), ¶ms_raw) != 1) { - throw CryptoException("EVP_PKEY_paramgen"); - } - EvpPkeyPtr params(params_raw); - // Now we can create a context for the proper key generation - EvpPkeyCtxPtr key_ctx(::EVP_PKEY_CTX_new(params.get(), nullptr)); - if (!params_ctx) { - throw CryptoException("EVP_PKEY_CTX_new"); - } - if (::EVP_PKEY_keygen_init(key_ctx.get()) != 1) { - throw CryptoException("EVP_PKEY_keygen_init"); - } - // Finally, it's time to generate the key pair itself. - ::EVP_PKEY* pkey_raw = nullptr; - if (::EVP_PKEY_keygen(key_ctx.get(), &pkey_raw) != 1) { - throw CryptoException("EVP_PKEY_keygen"); - } - EvpPkeyPtr generated_key(pkey_raw); -#if (OPENSSL_VERSION_NUMBER < 0x10100000L) - // On OpenSSL versions prior to 1.1.0, we must set the named curve ASN1 flag - // directly on the EC_KEY, as the EVP_PKEY wrapper doesn't exist (this is a - // half truth, as it exists on 1.0.2 stable, but not necessarily on all 1.0.2 - // versions, and certainly not on 1.0.1). - EcKeyPtr ec_key(::EVP_PKEY_get1_EC_KEY(generated_key.get())); // Bumps ref count, needs free - if (!ec_key) { - throw CryptoException("EVP_PKEY_get1_EC_KEY"); - } - ::EC_KEY_set_asn1_flag(ec_key.get(), OPENSSL_EC_NAMED_CURVE); -#endif - return std::make_shared<PrivateKey>(std::move(generated_key), Type::EC); -} +} // anonymous namespace // Some references: // https://stackoverflow.com/questions/256405/programmatically-create-x509-certificate-using-openssl // https://opensource.apple.com/source/OpenSSL/OpenSSL-22/openssl/demos/x509/mkcert.c -std::shared_ptr<X509Certificate> X509Certificate::generate_from(Params params) { +std::shared_ptr<X509CertificateImpl> X509CertificateImpl::generate_openssl_x509_from(Params params) { X509Ptr cert(::X509_new()); if (!cert) { throw CryptoException("X509_new"); } + // FIXME make this const, currently is not due to OpenSSL API const issues (ugh). + auto& subject_key_impl = dynamic_cast<PrivateKeyImpl&>(*params.subject_key); ::X509_set_version(cert.get(), 2); // 2 actually means v3 :) assign_random_positive_serial_number(*cert); set_certificate_expires_from_now(*cert, params.valid_for); // Internal key copy; does not take ownership - if (::X509_set_pubkey(cert.get(), params.subject_key->native_key()) != 1) { + if (::X509_set_pubkey(cert.get(), subject_key_impl.native_key()) != 1) { throw CryptoException("X509_set_pubkey"); } // The "subject" is the target entity the certificate is intended to, well, certify. @@ -259,13 +264,14 @@ std::shared_ptr<X509Certificate> X509Certificate::generate_from(Params params) { // If we _do_ have an issuer, we'll record its Subject name as our Issuer name. // Note that it's legal to have a self-signed non-CA certificate, though it obviously // cannot be used to sign any subordinate certificates. - ::X509_NAME* issuer_name = (params.issuer - ? ::X509_get_subject_name(params.issuer->native_cert()) + auto* issuer_cert_impl = dynamic_cast<X509CertificateImpl*>(params.issuer.get()); // May be nullptr. + ::X509_NAME* issuer_name = (issuer_cert_impl + ? ::X509_get_subject_name(issuer_cert_impl->native_cert()) : subj_name); if (::X509_set_issuer_name(cert.get(), issuer_name) != 1) { // Makes internal copy throw CryptoException("X509_set_issuer_name"); } - ::X509& issuer_cert = params.issuer ? *params.issuer->native_cert() : *cert; + ::X509& issuer_cert = issuer_cert_impl ? *issuer_cert_impl->native_cert() : *cert; const char* basic_constraints = params.is_ca ? "critical,CA:TRUE" : "critical,CA:FALSE"; const char* key_usage = params.is_ca ? "critical,keyCertSign,digitalSignature" @@ -278,13 +284,14 @@ std::shared_ptr<X509Certificate> X509Certificate::generate_from(Params params) { add_v3_ext(*cert, issuer_cert, NID_authority_key_identifier, "keyid:always"); add_any_subject_alternate_names(*cert, issuer_cert, params.subject_info.subject_alt_names); - if (::X509_sign(cert.get(), params.issuer_key->native_key(), ::EVP_sha256()) == 0) { + auto& issuer_key_impl = dynamic_cast<PrivateKeyImpl&>(*params.issuer_key); + if (::X509_sign(cert.get(), issuer_key_impl.native_key(), ::EVP_sha256()) == 0) { throw CryptoException("X509_sign"); } - return std::make_shared<X509Certificate>(std::move(cert)); + return std::make_shared<X509CertificateImpl>(std::move(cert)); } -vespalib::string X509Certificate::to_pem() const { +vespalib::string X509CertificateImpl::to_pem() const { BioPtr bio = new_memory_bio(); // TODO this API is const-broken, revisit in the future... auto* mutable_cert = const_cast<::X509*>(_cert.get()); @@ -295,47 +302,4 @@ vespalib::string X509Certificate::to_pem() const { } -X509Certificate::DistinguishedName::DistinguishedName() = default; -X509Certificate::DistinguishedName::DistinguishedName(const DistinguishedName&) = default; -X509Certificate::DistinguishedName& X509Certificate::DistinguishedName::operator=(const DistinguishedName&) = default; -X509Certificate::DistinguishedName::DistinguishedName(DistinguishedName&&) = default; -X509Certificate::DistinguishedName& X509Certificate::DistinguishedName::operator=(DistinguishedName&&) = default; -X509Certificate::DistinguishedName::~DistinguishedName() = default; - -X509Certificate::Params::Params() = default; -X509Certificate::Params::~Params() = default; - -X509Certificate::Params -X509Certificate::Params::self_signed(SubjectInfo subject, - std::shared_ptr<PrivateKey> key) { - Params params; - params.subject_info = std::move(subject); - params.subject_key = key; - params.issuer_key = std::move(key); // self-signed, subject == issuer - params.is_ca = true; - return params; -} - -X509Certificate::Params -X509Certificate::Params::issued_by(SubjectInfo subject, - std::shared_ptr<PrivateKey> subject_key, - std::shared_ptr<X509Certificate> issuer, - std::shared_ptr<PrivateKey> issuer_key) { - Params params; - params.subject_info = std::move(subject); - params.issuer = std::move(issuer); - params.subject_key = std::move(subject_key); - params.issuer_key = std::move(issuer_key); - params.is_ca = false; // By default, caller can change for intermediate CAs - return params; -} - -CertKeyWrapper::CertKeyWrapper(std::shared_ptr<X509Certificate> cert_, - std::shared_ptr<PrivateKey> key_) - : cert(std::move(cert_)), - key(std::move(key_)) -{} - -CertKeyWrapper::~CertKeyWrapper() = default; - } diff --git a/vespalib/src/vespa/vespalib/crypto/openssl_crypto_impl.h b/vespalib/src/vespa/vespalib/crypto/openssl_crypto_impl.h new file mode 100644 index 00000000000..87e4ee14e65 --- /dev/null +++ b/vespalib/src/vespa/vespalib/crypto/openssl_crypto_impl.h @@ -0,0 +1,44 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/crypto/openssl_typedefs.h> +#include "private_key.h" +#include "x509_certificate.h" + +namespace vespalib::crypto::openssl_impl { + +class PrivateKeyImpl : public PrivateKey { + EvpPkeyPtr _pkey; + Type _type; +public: + PrivateKeyImpl(EvpPkeyPtr pkey, Type type) + : _pkey(std::move(pkey)), + _type(type) + {} + ~PrivateKeyImpl() override = default; + + ::EVP_PKEY* native_key() noexcept { return _pkey.get(); } + const ::EVP_PKEY* native_key() const noexcept { return _pkey.get(); } + + Type type() const noexcept override { return _type; } + vespalib::string private_to_pem() const override; + + static std::shared_ptr<PrivateKeyImpl> generate_openssl_p256_ec_key(); +}; + +class X509CertificateImpl : public X509Certificate { + X509Ptr _cert; +public: + explicit X509CertificateImpl(X509Ptr cert) : _cert(std::move(cert)) {} + ~X509CertificateImpl() = default; + + ::X509* native_cert() noexcept { return _cert.get(); } + const ::X509* native_cert() const noexcept { return _cert.get(); } + + vespalib::string to_pem() const override; + + // Generates an X509 certificate using a SHA-256 digest + static std::shared_ptr<X509CertificateImpl> generate_openssl_x509_from(Params params); +}; + +} diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_typedefs.h b/vespalib/src/vespa/vespalib/crypto/openssl_typedefs.h index afafe556338..2986a4515f7 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_typedefs.h +++ b/vespalib/src/vespa/vespalib/crypto/openssl_typedefs.h @@ -6,7 +6,7 @@ #include <openssl/crypto.h> #include <openssl/x509.h> -namespace vespalib::net::tls::impl { +namespace vespalib::crypto { struct BioDeleter { void operator()(::BIO* bio) const noexcept { diff --git a/vespalib/src/vespa/vespalib/crypto/private_key.cpp b/vespalib/src/vespa/vespalib/crypto/private_key.cpp new file mode 100644 index 00000000000..7ece9418bef --- /dev/null +++ b/vespalib/src/vespa/vespalib/crypto/private_key.cpp @@ -0,0 +1,11 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "private_key.h" +#include "openssl_crypto_impl.h" + +namespace vespalib::crypto { + +std::shared_ptr<PrivateKey> PrivateKey::generate_p256_ec_key() { + return openssl_impl::PrivateKeyImpl::generate_openssl_p256_ec_key(); +} + +} diff --git a/vespalib/src/vespa/vespalib/crypto/private_key.h b/vespalib/src/vespa/vespalib/crypto/private_key.h new file mode 100644 index 00000000000..7ac5c31502c --- /dev/null +++ b/vespalib/src/vespa/vespalib/crypto/private_key.h @@ -0,0 +1,34 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <memory> + +namespace vespalib::crypto { + +/* + * Represents an asymmetric cryptographic private key. + * + * Can only be used for private/public key crypto, not for secret key (e.g. AES) crypto. + * Currently only supports generating EC keys on the standard P-256 curve. + */ +class PrivateKey { +public: + enum class Type { + EC, + RSA // TODO implement support..! + }; + + virtual ~PrivateKey() = default; + + virtual Type type() const noexcept = 0; + // TODO should have a wrapper for this that takes care to securely erase + // string memory on destruction. + virtual vespalib::string private_to_pem() const = 0; + + static std::shared_ptr<PrivateKey> generate_p256_ec_key(); +protected: + PrivateKey() = default; +}; + +} diff --git a/vespalib/src/vespa/vespalib/crypto/x509_certificate.cpp b/vespalib/src/vespa/vespalib/crypto/x509_certificate.cpp new file mode 100644 index 00000000000..ecd061e573a --- /dev/null +++ b/vespalib/src/vespa/vespalib/crypto/x509_certificate.cpp @@ -0,0 +1,62 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "x509_certificate.h" +#include "openssl_crypto_impl.h" + +namespace vespalib::crypto { + +X509Certificate::DistinguishedName::DistinguishedName() = default; +X509Certificate::DistinguishedName::DistinguishedName(const DistinguishedName&) = default; +X509Certificate::DistinguishedName& X509Certificate::DistinguishedName::operator=(const DistinguishedName&) = default; +X509Certificate::DistinguishedName::DistinguishedName(DistinguishedName&&) noexcept = default; +X509Certificate::DistinguishedName& X509Certificate::DistinguishedName::operator=(DistinguishedName&&) noexcept = default; +X509Certificate::DistinguishedName::~DistinguishedName() = default; + +X509Certificate::Params::Params() = default; +X509Certificate::Params::~Params() = default; + +X509Certificate::Params::Params(const Params&) = default; +X509Certificate::Params& X509Certificate::Params::operator=(const Params&) = default; +X509Certificate::Params::Params(Params&&) noexcept = default; +X509Certificate::Params& X509Certificate::Params::operator=(Params&&) noexcept = default; + +X509Certificate::Params +X509Certificate::Params::self_signed(SubjectInfo subject, + std::shared_ptr<PrivateKey> key) +{ + Params params; + params.subject_info = std::move(subject); + params.subject_key = key; + params.issuer_key = std::move(key); // self-signed, subject == issuer + params.is_ca = true; + return params; +} + +X509Certificate::Params +X509Certificate::Params::issued_by(SubjectInfo subject, + std::shared_ptr<PrivateKey> subject_key, + std::shared_ptr<X509Certificate> issuer, + std::shared_ptr<PrivateKey> issuer_key) +{ + Params params; + params.subject_info = std::move(subject); + params.issuer = std::move(issuer); + params.subject_key = std::move(subject_key); + params.issuer_key = std::move(issuer_key); + params.is_ca = false; // By default, caller can change for intermediate CAs + return params; +} + +std::shared_ptr<X509Certificate> X509Certificate::generate_from(Params params) { + return openssl_impl::X509CertificateImpl::generate_openssl_x509_from(std::move(params)); +} + +CertKeyWrapper::CertKeyWrapper(std::shared_ptr<X509Certificate> cert_, + std::shared_ptr<PrivateKey> key_) + : cert(std::move(cert_)), + key(std::move(key_)) +{} + +CertKeyWrapper::~CertKeyWrapper() = default; + +} diff --git a/vespalib/src/tests/net/tls/openssl_impl/crypto_utils.h b/vespalib/src/vespa/vespalib/crypto/x509_certificate.h index 017dfbdbc12..9eea423c5a0 100644 --- a/vespalib/src/tests/net/tls/openssl_impl/crypto_utils.h +++ b/vespalib/src/vespa/vespalib/crypto/x509_certificate.h @@ -1,52 +1,32 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "private_key.h" #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/net/tls/impl/openssl_typedefs.h> #include <chrono> #include <memory> #include <vector> -// TODOs -// - add unit testing -// - extend interfaces (separate PublicKey etc) -// - hide all OpenSSL details from header -// - move to appropriate new namespace/directory somewhere under vespalib - -namespace vespalib::net::tls::impl { - -class PrivateKey { -public: - enum class Type { - EC, - RSA // TODO implement support..! - }; -private: - EvpPkeyPtr _pkey; - Type _type; -public: - PrivateKey(EvpPkeyPtr pkey, Type type) - : _pkey(std::move(pkey)), - _type(type) - {} - - ::EVP_PKEY* native_key() noexcept { return _pkey.get(); } - const ::EVP_PKEY* native_key() const noexcept { return _pkey.get(); } - - Type type() const noexcept { return _type; } - vespalib::string private_to_pem() const; - - static std::shared_ptr<PrivateKey> generate_p256_ec_key(); -}; - - +namespace vespalib::crypto { + +/** + * Represents an X509 certificate instance and provides utility methods + * for generating new certificates on the fly. Certificates can be created + * for both Certificate Authorities and regular hosts (leaves). + * + * This implementation aims to ensure that best cryptographic practices are + * followed automatically. In particular: + * - The certificate digest is always SHA-256, never SHA-1 or MD5 + * - The certificate serial number is a 160-bit secure random sequence + * (technically 159 bits since the MSB is always zero) rather than a + * collision-prone or predictable sequence number. + * + */ class X509Certificate { - X509Ptr _cert; public: - explicit X509Certificate(X509Ptr cert) : _cert(std::move(cert)) {} + virtual ~X509Certificate() = default; - ::X509* native_cert() noexcept { return _cert.get(); } - const ::X509* native_cert() const noexcept { return _cert.get(); } + virtual vespalib::string to_pem() const = 0; struct DistinguishedName { vespalib::string _country; // "C" @@ -61,9 +41,8 @@ public: DistinguishedName(); DistinguishedName(const DistinguishedName&); DistinguishedName& operator=(const DistinguishedName&); - // TODO make these noexcept once vespalib::string has noexcept move.. or move at all! - DistinguishedName(DistinguishedName&&); - DistinguishedName& operator=(DistinguishedName&&); + DistinguishedName(DistinguishedName&&) noexcept; + DistinguishedName& operator=(DistinguishedName&&) noexcept; ~DistinguishedName(); // TODO could add rvalue overloads as well... @@ -101,8 +80,13 @@ public: Params(); ~Params(); + Params(const Params&); + Params& operator=(const Params&); + Params(Params&&) noexcept; + Params& operator=(Params&&) noexcept; + SubjectInfo subject_info; - // TODO make public key, but private key has both and this is currently just for testing. + // TODO make public key, but private key has both. std::shared_ptr<PrivateKey> subject_key; std::shared_ptr<X509Certificate> issuer; // May be nullptr for self-signed certs std::shared_ptr<PrivateKey> issuer_key; @@ -119,9 +103,14 @@ public: // Generates an X509 certificate using a SHA-256 digest static std::shared_ptr<X509Certificate> generate_from(Params params); - vespalib::string to_pem() const; +protected: + X509Certificate() = default; }; +/* + * Simple wrapper for storing both a X509 certificate and the private key + * that signed it. Useful for testing. + */ struct CertKeyWrapper { std::shared_ptr<X509Certificate> cert; std::shared_ptr<PrivateKey> key; diff --git a/vespalib/src/vespa/vespalib/data/slime/object_value.h b/vespalib/src/vespa/vespalib/data/slime/object_value.h index 377bf5bd37f..651f3a156d2 100644 --- a/vespalib/src/vespa/vespalib/data/slime/object_value.h +++ b/vespalib/src/vespa/vespalib/data/slime/object_value.h @@ -12,8 +12,7 @@ #include <vespa/vespalib/stllike/vector_map.h> #include <vespa/vespalib/util/stash.h> -namespace vespalib { -namespace slime { +namespace vespalib::slime { /** * Class representing a collection of unordered values that can be @@ -32,7 +31,7 @@ private: Cursor &setIfUnset(SymbolInserter &symbol, const ValueFactory &input) { Value *&pos = _fields[symbol.insert()]; - if (pos != 0) { + if (pos != nullptr) { return *NixValue::invalid(); } pos = input.create(_stash); @@ -40,7 +39,7 @@ private: } Value *lookup(const SymbolLookup &symbol) const { - SymbolValueMap::const_iterator pos = _fields.find(symbol.lookup()); + auto pos = _fields.find(symbol.lookup()); if (pos == _fields.end()) { return NixValue::invalid(); } @@ -81,9 +80,7 @@ public: Cursor &setObject(Memory name) override; Symbol resolve(Memory symbol_name) override; - ~ObjectValue() { } + ~ObjectValue() override = default; }; -} // namespace vespalib::slime -} // namespace vespalib - +} diff --git a/vespalib/src/vespa/vespalib/geo/zcurve.cpp b/vespalib/src/vespa/vespalib/geo/zcurve.cpp index b2dc02759dc..00ce7ee18c6 100644 --- a/vespalib/src/vespa/vespalib/geo/zcurve.cpp +++ b/vespalib/src/vespa/vespalib/geo/zcurve.cpp @@ -4,8 +4,7 @@ #include <vespa/vespalib/util/priority_queue.h> #include <vespa/vespalib/util/fiddle.h> -namespace vespalib { -namespace geo { +namespace vespalib::geo { namespace { @@ -182,4 +181,3 @@ ZCurve::decodeSlow(int64_t enc, int32_t *xp, int32_t *yp) } } -} diff --git a/vespalib/src/vespa/vespalib/geo/zcurve.h b/vespalib/src/vespa/vespalib/geo/zcurve.h index 2a05c4c7744..bd76b78ea23 100644 --- a/vespalib/src/vespa/vespalib/geo/zcurve.h +++ b/vespalib/src/vespa/vespalib/geo/zcurve.h @@ -6,8 +6,7 @@ #include <cassert> #include <vector> -namespace vespalib { -namespace geo { +namespace vespalib::geo { /** * @brief Utility methods for a Z-curve (Morton-order) encoder and decoder. @@ -31,7 +30,7 @@ public: public: BoundingBox(int32_t minx, int32_t maxx, int32_t miny, int32_t maxy); - ~BoundingBox() { } + ~BoundingBox() = default; int64_t getzMinx() const { return _zMinx; } int64_t getzMaxx() const { return _zMaxx; } @@ -221,5 +220,3 @@ public: }; } -} - diff --git a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp index a92b0e06bbe..7bbc4b7523c 100644 --- a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp @@ -2,9 +2,9 @@ #include "crypto_engine.h" #include <vespa/vespalib/data/smart_buffer.h> +#include <vespa/vespalib/crypto/crypto_exception.h> #include <vespa/vespalib/net/tls/authorization_mode.h> #include <vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h> -#include <vespa/vespalib/net/tls/crypto_exception.h> #include <vespa/vespalib/net/tls/maybe_tls_crypto_engine.h> #include <vespa/vespalib/net/tls/statistics.h> #include <vespa/vespalib/net/tls/tls_crypto_engine.h> @@ -232,7 +232,7 @@ CryptoEngine::SP create_default_crypto_engine() { CryptoEngine::SP try_create_default_crypto_engine() { try { return create_default_crypto_engine(); - } catch (net::tls::CryptoException &e) { + } catch (crypto::CryptoException &e) { LOG(error, "failed to create default crypto engine: %s", e.what()); std::_Exit(78); } diff --git a/vespalib/src/vespa/vespalib/net/crypto_engine.h b/vespalib/src/vespa/vespalib/net/crypto_engine.h index 4deacf9a6c7..71511b8a552 100644 --- a/vespalib/src/vespa/vespalib/net/crypto_engine.h +++ b/vespalib/src/vespa/vespalib/net/crypto_engine.h @@ -19,6 +19,8 @@ class SocketSpec; **/ struct CryptoEngine { using SP = std::shared_ptr<CryptoEngine>; + virtual bool use_tls_when_client() const = 0; + virtual bool always_use_tls_when_server() const = 0; virtual CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) = 0; virtual CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) = 0; virtual ~CryptoEngine(); @@ -29,6 +31,8 @@ struct CryptoEngine { * Crypto engine without encryption. **/ struct NullCryptoEngine : public CryptoEngine { + bool use_tls_when_client() const override { return false; } + bool always_use_tls_when_server() const override { return false; } CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override; }; @@ -39,6 +43,8 @@ struct NullCryptoEngine : public CryptoEngine { * from TLS. **/ struct XorCryptoEngine : public CryptoEngine { + bool use_tls_when_client() const override { return false; } + bool always_use_tls_when_server() const override { return false; } CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override; }; diff --git a/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt b/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt index 6dc48db68e4..b7801f40959 100644 --- a/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt @@ -5,7 +5,6 @@ vespa_add_library(vespalib_vespalib_net_tls OBJECT auto_reloading_tls_crypto_engine.cpp crypto_codec.cpp crypto_codec_adapter.cpp - crypto_exception.cpp maybe_tls_crypto_engine.cpp maybe_tls_crypto_socket.cpp peer_credentials.cpp diff --git a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp index c425ab75ce8..bdb2402adbc 100644 --- a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp @@ -99,6 +99,18 @@ CryptoSocket::UP AutoReloadingTlsCryptoEngine::create_server_crypto_socket(Socke return acquire_current_engine()->create_server_crypto_socket(std::move(socket)); } +bool +AutoReloadingTlsCryptoEngine::use_tls_when_client() const +{ + return acquire_current_engine()->use_tls_when_client(); +} + +bool +AutoReloadingTlsCryptoEngine::always_use_tls_when_server() const +{ + return acquire_current_engine()->always_use_tls_when_server(); +} + std::unique_ptr<TlsCryptoSocket> AutoReloadingTlsCryptoEngine::create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) { return acquire_current_engine()->create_tls_client_crypto_socket(std::move(socket), spec); diff --git a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h index e268cbc8f1a..1b80b782daf 100644 --- a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h +++ b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h @@ -47,6 +47,8 @@ public: CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override; + bool use_tls_when_client() const override; + bool always_use_tls_when_server() const override; std::unique_ptr<TlsCryptoSocket> create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; std::unique_ptr<TlsCryptoSocket> create_tls_server_crypto_socket(SocketHandle socket) override; }; diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.cpp b/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.cpp index 614722a9769..d7d02534242 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.cpp @@ -1,11 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "direct_buffer_bio.h" -#include <vespa/vespalib/net/tls/crypto_exception.h> +#include <vespa/vespalib/crypto/crypto_exception.h> +#include <vespa/vespalib/util/backtrace.h> #include <utility> #include <cassert> -#include <vespa/vespalib/util/backtrace.h> - #include <vespa/log/log.h> LOG_SETUP(".vespalib.net.tls.impl.direct_buffer_bio"); @@ -19,6 +18,8 @@ LOG_SETUP(".vespalib.net.tls.impl.direct_buffer_bio"); * - https://github.com/indutny/uv_ssl_t/blob/master/src/bio.c */ +using namespace vespalib::crypto; + namespace vespalib::net::tls::impl { namespace { diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.h b/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.h index 581d43d6f29..8492bf2c436 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.h +++ b/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.h @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "openssl_typedefs.h" +#include <vespa/vespalib/crypto/openssl_typedefs.h> #include <openssl/bio.h> /* @@ -22,8 +22,8 @@ namespace vespalib::net::tls::impl { -BioPtr new_mutable_direct_buffer_bio(); -BioPtr new_const_direct_buffer_bio(); +crypto::BioPtr new_mutable_direct_buffer_bio(); +crypto::BioPtr new_const_direct_buffer_bio(); struct MutableBufferView { // Could use a pointer pair instead (or just modify the ptr), but being explicit is good for readability. diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp index 6a79caa8264..1d87a50190e 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp @@ -3,8 +3,8 @@ #include "openssl_tls_context_impl.h" #include "direct_buffer_bio.h" +#include <vespa/vespalib/crypto/crypto_exception.h> #include <vespa/vespalib/net/tls/crypto_codec.h> -#include <vespa/vespalib/net/tls/crypto_exception.h> #include <vespa/vespalib/net/tls/statistics.h> #include <mutex> @@ -36,6 +36,8 @@ LOG_SETUP(".vespalib.net.tls.openssl_crypto_codec_impl"); * light fades and turns to all-enveloping darkness. */ +using namespace vespalib::crypto; + namespace vespalib::net::tls::impl { namespace { diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h index ec8df853c16..80f3e12786a 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h +++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "openssl_typedefs.h" +#include <vespa/vespalib/crypto/openssl_typedefs.h> #include <vespa/vespalib/net/socket_address.h> #include <vespa/vespalib/net/socket_spec.h> #include <vespa/vespalib/net/tls/transport_security_options.h> @@ -47,12 +47,12 @@ class OpenSslCryptoCodecImpl : public CryptoCodec { // The context maintains shared verification callback state, so it must be // kept alive explictly for at least as long as any codecs. std::shared_ptr<OpenSslTlsContextImpl> _ctx; - SocketSpec _peer_spec; - SocketAddress _peer_address; - SslPtr _ssl; - ::BIO* _input_bio; // Owned by _ssl - ::BIO* _output_bio; // Owned by _ssl - Mode _mode; + SocketSpec _peer_spec; + SocketAddress _peer_address; + crypto::SslPtr _ssl; + ::BIO* _input_bio; // Owned by _ssl + ::BIO* _output_bio; // Owned by _ssl + Mode _mode; std::optional<DeferredHandshakeParams> _deferred_handshake_params; std::optional<HandshakeResult> _deferred_handshake_result; public: diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp index 98675ec6b0b..e66baf87999 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp @@ -1,9 +1,9 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "iana_cipher_map.h" -#include "openssl_typedefs.h" #include "openssl_tls_context_impl.h" #include "openssl_crypto_codec_impl.h" -#include <vespa/vespalib/net/tls/crypto_exception.h> +#include <vespa/vespalib/crypto/crypto_exception.h> +#include <vespa/vespalib/crypto/openssl_typedefs.h> #include <vespa/vespalib/net/tls/statistics.h> #include <vespa/vespalib/net/tls/transport_security_options.h> #include <vespa/vespalib/util/stringfmt.h> @@ -26,6 +26,8 @@ LOG_SETUP(".vespalib.net.tls.openssl_tls_context_impl"); # error "Provided OpenSSL version is too darn old, need at least 1.0.0" #endif +using namespace vespalib::crypto; + namespace vespalib::net::tls::impl { namespace { diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h index c519b1ae874..badfe8306d1 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h +++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "openssl_typedefs.h" +#include <vespa/vespalib/crypto/openssl_typedefs.h> #include <vespa/vespalib/net/socket_address.h> #include <vespa/vespalib/net/tls/tls_context.h> #include <vespa/vespalib/net/tls/transport_security_options.h> @@ -13,7 +13,7 @@ namespace vespalib::net::tls::impl { class OpenSslTlsContextImpl : public TlsContext { - SslCtxPtr _ctx; + crypto::SslCtxPtr _ctx; AuthorizationMode _authorization_mode; std::shared_ptr<CertificateVerificationCallback> _cert_verify_callback; TransportSecurityOptions _redacted_transport_options; diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h index 147a770bc8f..ece7d094c54 100644 --- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h +++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h @@ -28,6 +28,8 @@ public: : _null_engine(std::make_shared<NullCryptoEngine>()), _tls_engine(std::move(tls_engine)), _use_tls_when_client(use_tls_when_client) {} + bool use_tls_when_client() const override { return _use_tls_when_client; } + bool always_use_tls_when_server() const override { return false; } CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override; }; diff --git a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h index 5e760cf5585..444a817b357 100644 --- a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h +++ b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h @@ -27,6 +27,8 @@ public: net::tls::AuthorizationMode authz_mode = net::tls::AuthorizationMode::Enforce); std::unique_ptr<TlsCryptoSocket> create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; std::unique_ptr<TlsCryptoSocket> create_tls_server_crypto_socket(SocketHandle socket) override; + bool use_tls_when_client() const override { return true; } + bool always_use_tls_when_server() const override { return true; } CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override { return create_tls_client_crypto_socket(std::move(socket), spec); } diff --git a/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp b/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp index b31f2830976..ad46e7272cb 100644 --- a/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp +++ b/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp @@ -1,82 +1,77 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "make_tls_options_for_testing.h" +#include <vespa/vespalib/crypto/private_key.h> +#include <vespa/vespalib/crypto/x509_certificate.h> -/* - * Generated with the following commands: - * - * openssl ecparam -name prime256v1 -genkey -noout -out ca.key - * - * openssl req -new -x509 -nodes -key ca.key \ - * -sha256 -out ca.pem \ - * -subj '/C=US/L=LooneyVille/O=ACME/OU=ACME test CA/CN=acme.example.com' \ - * -days 10000 - * - * openssl ecparam -name prime256v1 -genkey -noout -out host.key - * - * openssl req -new -key host.key -out host.csr \ - * -subj '/C=US/L=LooneyVille/O=Wile. E. Coyote, Ltd./CN=wile.example.com' \ - * -sha256 - * - * openssl x509 -req -in host.csr \ - * -CA ca.pem \ - * -CAkey ca.key \ - * -CAcreateserial \ - * -out host.pem \ - * -days 10000 \ - * -sha256 - * - * TODO generate keypairs and certs at test-time to avoid any hard-coding - * There certs are valid until 2046, so that buys us some time..! - */ - -// ca.pem -constexpr const char* ca_pem = R"(-----BEGIN CERTIFICATE----- -MIIBuDCCAV4CCQDpVjQIixTxvDAKBggqhkjOPQQDAjBkMQswCQYDVQQGEwJVUzEU -MBIGA1UEBwwLTG9vbmV5VmlsbGUxDTALBgNVBAoMBEFDTUUxFTATBgNVBAsMDEFD -TUUgdGVzdCBDQTEZMBcGA1UEAwwQYWNtZS5leGFtcGxlLmNvbTAeFw0xODA4MzEx -MDU3NDVaFw00NjAxMTYxMDU3NDVaMGQxCzAJBgNVBAYTAlVTMRQwEgYDVQQHDAtM -b29uZXlWaWxsZTENMAsGA1UECgwEQUNNRTEVMBMGA1UECwwMQUNNRSB0ZXN0IENB -MRkwFwYDVQQDDBBhY21lLmV4YW1wbGUuY29tMFkwEwYHKoZIzj0CAQYIKoZIzj0D -AQcDQgAE1L7IzCN5pbyVnBATIHieuxq+hf9kWyn5yfjkXMhD52T5ITz1huq4nbiN -YtRoRP7XmipI60R/uiCHzERcsVz4rDAKBggqhkjOPQQDAgNIADBFAiEA6wmZDBca -y0aJ6ABtjbjx/vlmVDxdkaSZSgO8h2CkvIECIFktCkbZhDFfSvbqUScPOGuwkdGQ -L/EW2Bxp+1BPcYoZ ------END CERTIFICATE-----)"; - -// host.pem -constexpr const char* cert_pem = R"(-----BEGIN CERTIFICATE----- -MIIBsTCCAVgCCQD6GfDh0ltpsjAKBggqhkjOPQQDAjBkMQswCQYDVQQGEwJVUzEU -MBIGA1UEBwwLTG9vbmV5VmlsbGUxDTALBgNVBAoMBEFDTUUxFTATBgNVBAsMDEFD -TUUgdGVzdCBDQTEZMBcGA1UEAwwQYWNtZS5leGFtcGxlLmNvbTAeFw0xODA4MzEx -MDU3NDVaFw00NjAxMTYxMDU3NDVaMF4xCzAJBgNVBAYTAlVTMRQwEgYDVQQHDAtM -b29uZXlWaWxsZTEeMBwGA1UECgwVV2lsZS4gRS4gQ295b3RlLCBMdGQuMRkwFwYD -VQQDDBB3aWxlLmV4YW1wbGUuY29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE -e+Y4hxt66em0STviGUj6ZDbxzoLoubXWRml8JDFrEc2S2433KWw2npxYSKVCyo3a -/Vo33V8/H0WgOXioKEZJxDAKBggqhkjOPQQDAgNHADBEAiAN+87hQuGv3z0Ja2BV -b8PHq2vp3BJHjeMuxWu4BFPn0QIgYlvIHikspgGatXRNMZ1gPC0oCccsJFcie+Cw -zL06UPI= ------END CERTIFICATE-----)"; - -// host.key -constexpr const char* key_pem = R"(-----BEGIN EC PRIVATE KEY----- -MHcCAQEEID6di2PFYn8hPrxPbkFDGkSqF+K8L520In7nx3g0jwzOoAoGCCqGSM49 -AwEHoUQDQgAEe+Y4hxt66em0STviGUj6ZDbxzoLoubXWRml8JDFrEc2S2433KWw2 -npxYSKVCyo3a/Vo33V8/H0WgOXioKEZJxA== ------END EC PRIVATE KEY-----)"; +namespace { + +using namespace vespalib::crypto; + +struct TransientCryptoCredentials { + CertKeyWrapper root_ca; + CertKeyWrapper host_creds; + vespalib::net::tls::TransportSecurityOptions cached_transport_options; + + TransientCryptoCredentials(); + ~TransientCryptoCredentials(); + + static CertKeyWrapper make_root_ca() { + auto dn = X509Certificate::DistinguishedName() + .country("US").state("CA").locality("Sunnyvale") + .organization("ACME, Inc.") + .organizational_unit("ACME Root CA") + .add_common_name("acme.example.com"); + auto subject = X509Certificate::SubjectInfo(std::move(dn)); + auto key = PrivateKey::generate_p256_ec_key(); + auto params = X509Certificate::Params::self_signed(std::move(subject), key); + auto cert = X509Certificate::generate_from(std::move(params)); + return {std::move(cert), std::move(key)}; + } + + static CertKeyWrapper make_host_creds(const CertKeyWrapper& root_ca_creds) { + auto dn = X509Certificate::DistinguishedName() + .country("US").state("CA").locality("Sunnyvale") + .organization("Wile E. Coyote, Ltd.") + .organizational_unit("Unit Testing and Anvil Dropping Division") + .add_common_name("localhost"); // Should technically not be needed, but including it anyway. + auto subject = X509Certificate::SubjectInfo(std::move(dn)); + subject.add_subject_alt_name("DNS:localhost"); + auto key = PrivateKey::generate_p256_ec_key(); + auto params = X509Certificate::Params::issued_by(std::move(subject), key, root_ca_creds.cert, root_ca_creds.key); + params.valid_for = std::chrono::hours(1); + auto cert = X509Certificate::generate_from(std::move(params)); + return {std::move(cert), std::move(key)}; + } + + static const TransientCryptoCredentials& instance(); +}; + +TransientCryptoCredentials::TransientCryptoCredentials() + : root_ca(make_root_ca()), + host_creds(make_host_creds(root_ca)), + cached_transport_options(vespalib::net::tls::TransportSecurityOptions::Params(). + ca_certs_pem(root_ca.cert->to_pem()). + cert_chain_pem(host_creds.cert->to_pem()). + private_key_pem(host_creds.key->private_to_pem()). + authorized_peers(vespalib::net::tls::AuthorizedPeers::allow_all_authenticated())) +{} + +TransientCryptoCredentials::~TransientCryptoCredentials() = default; + +const TransientCryptoCredentials& TransientCryptoCredentials::instance() { + static TransientCryptoCredentials test_creds; + return test_creds; +} + +} namespace vespalib::test { SocketSpec local_spec("tcp/localhost:123"); vespalib::net::tls::TransportSecurityOptions make_tls_options_for_testing() { - auto ts_builder = vespalib::net::tls::TransportSecurityOptions::Params(). - ca_certs_pem(ca_pem). - cert_chain_pem(cert_pem). - private_key_pem(key_pem). - authorized_peers(vespalib::net::tls::AuthorizedPeers::allow_all_authenticated()). - disable_hostname_validation(true); // FIXME this is to avoid mass breakage of TLS'd networking tests. - return vespalib::net::tls::TransportSecurityOptions(std::move(ts_builder)); + return TransientCryptoCredentials::instance().cached_transport_options; } } // namespace vespalib::test diff --git a/vespalib/src/vespa/vespalib/util/stash.cpp b/vespalib/src/vespa/vespalib/util/stash.cpp index 18ed0e56e27..836654c2fb2 100644 --- a/vespalib/src/vespa/vespalib/util/stash.cpp +++ b/vespalib/src/vespa/vespalib/util/stash.cpp @@ -59,14 +59,14 @@ Stash::do_alloc(size_t size) } } -Stash::Stash(size_t chunk_size) +Stash::Stash(size_t chunk_size) noexcept : _chunks(nullptr), _cleanup(nullptr), _chunk_size(std::max(size_t(4096), chunk_size)) { } -Stash::Stash(Stash &&rhs) +Stash::Stash(Stash &&rhs) noexcept : _chunks(rhs._chunks), _cleanup(rhs._cleanup), _chunk_size(rhs._chunk_size) @@ -76,7 +76,7 @@ Stash::Stash(Stash &&rhs) } Stash & -Stash::operator=(Stash &&rhs) +Stash::operator=(Stash &&rhs) noexcept { stash::run_cleanup(_cleanup); stash::free_chunks(_chunks); diff --git a/vespalib/src/vespa/vespalib/util/stash.h b/vespalib/src/vespa/vespalib/util/stash.h index aa1441aa0bb..c5e8631ca9e 100644 --- a/vespalib/src/vespa/vespalib/util/stash.h +++ b/vespalib/src/vespa/vespalib/util/stash.h @@ -14,19 +14,19 @@ struct Cleanup { explicit Cleanup(Cleanup *next_in) noexcept : next(next_in) {} virtual void cleanup() = 0; protected: - virtual ~Cleanup() {} + virtual ~Cleanup() = default; }; // used as header for memory allocated outside the stash struct DeleteMemory : public Cleanup { explicit DeleteMemory(Cleanup *next_in) noexcept : Cleanup(next_in) {} - virtual void cleanup() override { free((void*)this); } + void cleanup() override { free((void*)this); } }; // used as prefix for objects to be destructed template<typename T> struct DestructObject : public Cleanup { explicit DestructObject(Cleanup *next_in) noexcept : Cleanup(next_in) {} - virtual void cleanup() override { reinterpret_cast<T*>(this + 1)->~T(); } + void cleanup() override { reinterpret_cast<T*>(this + 1)->~T(); } }; // used as prefix for arrays to be destructed @@ -34,7 +34,7 @@ template<typename T> struct DestructArray : public Cleanup { size_t size; explicit DestructArray(Cleanup *next_in, size_t size_in) noexcept : Cleanup(next_in), size(size_in) {} - virtual void cleanup() override { + void cleanup() override { T *array = reinterpret_cast<T*>(this + 1); for (size_t i = size; i-- > 0;) { array[i].~T(); @@ -46,7 +46,7 @@ struct Chunk { Chunk *next; size_t used; Chunk(const Chunk &) = delete; - Chunk(Chunk *next_in) : next(next_in), used(sizeof(Chunk)) {} + explicit Chunk(Chunk *next_in) : next(next_in), used(sizeof(Chunk)) {} void clear() { used = sizeof(Chunk); } char *alloc(size_t size, size_t chunk_size) { size_t aligned_size = ((size + (sizeof(char *) - 1)) @@ -124,14 +124,14 @@ public: }; typedef std::unique_ptr<Stash> UP; - explicit Stash(size_t chunk_size); - Stash() : Stash(4096) {} - Stash(Stash &&rhs); + explicit Stash(size_t chunk_size) noexcept ; + Stash() noexcept : Stash(4096) {} + Stash(Stash &&rhs) noexcept; Stash(const Stash &) = delete; Stash & operator = (const Stash &) = delete; ~Stash(); - Stash &operator=(Stash &&rhs); + Stash &operator=(Stash &&rhs) noexcept; void clear(); |