diff options
360 files changed, 4020 insertions, 2148 deletions
diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java index 2208b470ed8..fdf2bbfccff 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java @@ -201,7 +201,8 @@ public class InstanceValidatorTest { ApplicationInfo::getApplicationId, Function.identity() ) - )); + ), + true); SuperModelProvider superModelProvider = mock(SuperModelProvider.class); when(superModelProvider.getSuperModel()).thenReturn(superModel); diff --git a/client/src/main/java/ai/vespa/client/dsl/EndQuery.java b/client/src/main/java/ai/vespa/client/dsl/EndQuery.java index 4facf6ce0fb..2af1e0bb49d 100644 --- a/client/src/main/java/ai/vespa/client/dsl/EndQuery.java +++ b/client/src/main/java/ai/vespa/client/dsl/EndQuery.java @@ -177,7 +177,7 @@ public class EndQuery { } else if (others.isEmpty()) { sb.append("order by ").append(orderStr); } else { - sb.append("order by ").append(orderStr).append(", ").append(others); + sb.append("order by ").append(orderStr).append(" ").append(others); } if (groupQueryStr != null) { diff --git a/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy b/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy index e363a9bcc13..19c87d6aecd 100644 --- a/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy +++ b/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy @@ -59,7 +59,7 @@ class QTest extends Specification { .build() expect: - q == """yql=select * from sd1 where f1 contains "v1" and f2 contains "v2" or f3 contains "v3" and !(f4 contains "v4") order by f1 desc, f2 asc, limit 2 offset 1 timeout 3;¶mk1=paramv1""" + q == """yql=select * from sd1 where f1 contains "v1" and f2 contains "v2" or f3 contains "v3" and !(f4 contains "v4") order by f1 desc, f2 asc limit 2 offset 1 timeout 3;¶mk1=paramv1""" } def "matches"() { diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 43527335802..90132d6924a 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -990,13 +990,16 @@ ], "methods": [ "public void <init>()", - "public void <init>(java.util.Map)", + "public void <init>(java.util.Map, boolean)", "public java.util.Map getModelsPerTenant()", "public java.util.Map getModels()", + "public boolean isComplete()", "public java.util.List getAllApplicationInfos()", "public java.util.Optional getApplicationInfo(com.yahoo.config.provision.ApplicationId)", - "public com.yahoo.config.model.api.SuperModel cloneAndSetApplication(com.yahoo.config.model.api.ApplicationInfo)", - "public com.yahoo.config.model.api.SuperModel cloneAndRemoveApplication(com.yahoo.config.provision.ApplicationId)" + "public com.yahoo.config.model.api.SuperModel cloneAndSetApplication(com.yahoo.config.model.api.ApplicationInfo, boolean)", + "public com.yahoo.config.model.api.SuperModel cloneAndRemoveApplication(com.yahoo.config.provision.ApplicationId)", + "public com.yahoo.config.model.api.SuperModel cloneAsComplete()", + "public java.util.Set getApplicationIds()" ], "fields": [] }, @@ -1010,7 +1013,8 @@ ], "methods": [ "public abstract void applicationActivated(com.yahoo.config.model.api.SuperModel, com.yahoo.config.model.api.ApplicationInfo)", - "public abstract void applicationRemoved(com.yahoo.config.model.api.SuperModel, com.yahoo.config.provision.ApplicationId)" + "public abstract void applicationRemoved(com.yahoo.config.model.api.SuperModel, com.yahoo.config.provision.ApplicationId)", + "public abstract void notifyOfCompleteness(com.yahoo.config.model.api.SuperModel)" ], "fields": [] }, diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java index 1735e08c930..15502dac1f1 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java @@ -20,13 +20,15 @@ import java.util.Set; public class SuperModel { private final Map<ApplicationId, ApplicationInfo> models; + private final boolean complete; public SuperModel() { - this.models = Collections.emptyMap(); + this(Collections.emptyMap(), false); } - public SuperModel(Map<ApplicationId, ApplicationInfo> models) { + public SuperModel(Map<ApplicationId, ApplicationInfo> models, boolean complete) { this.models = models; + this.complete = complete; } public Map<TenantName, Set<ApplicationInfo>> getModelsPerTenant() { @@ -45,6 +47,8 @@ public class SuperModel { return ImmutableMap.copyOf(models); } + public boolean isComplete() { return complete; } + public List<ApplicationInfo> getAllApplicationInfos() { return new ArrayList<>(models.values()); } @@ -54,20 +58,22 @@ public class SuperModel { return applicationInfo == null ? Optional.empty() : Optional.of(applicationInfo); } - public SuperModel cloneAndSetApplication(ApplicationInfo application) { + public SuperModel cloneAndSetApplication(ApplicationInfo application, boolean complete) { Map<ApplicationId, ApplicationInfo> newModels = cloneModels(models); newModels.put(application.getApplicationId(), application); - - return new SuperModel(newModels); + return new SuperModel(newModels, complete); } public SuperModel cloneAndRemoveApplication(ApplicationId applicationId) { Map<ApplicationId, ApplicationInfo> newModels = cloneModels(models); newModels.remove(applicationId); - - return new SuperModel(newModels); + return new SuperModel(newModels, complete); } + public SuperModel cloneAsComplete() { return new SuperModel(models, true); } + + public Set<ApplicationId> getApplicationIds() { return models.keySet(); } + private static Map<ApplicationId, ApplicationInfo> cloneModels(Map<ApplicationId, ApplicationInfo> models) { return new LinkedHashMap<>(models); } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java index 497c38af908..e66a7e1ef7e 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java @@ -17,4 +17,12 @@ public interface SuperModelListener { * Application has been removed. */ void applicationRemoved(SuperModel superModel, ApplicationId id); + + /** + * Invoked once all applications that were supposed to be deployed on bootstrap + * have been activated (and the respective {@link #applicationActivated(SuperModel, ApplicationInfo) + * applicationActivated} have been invoked). The SuperModel is then said to be "complete". + * @param superModel + */ + void notifyOfCompleteness(SuperModel superModel); } diff --git a/config-model/pom.xml b/config-model/pom.xml index 25b733985f5..33f6657561c 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -105,6 +105,10 @@ <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> </exclusion> + <exclusion> + <groupId>org.apache.velocity</groupId> + <artifactId>velocity</artifactId> + </exclusion> </exclusions> </dependency> <dependency> diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java index 9cd7fb24e42..2790f2ddf6e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java @@ -24,16 +24,18 @@ public class TensorFieldProcessor extends Processor { @Override public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - for (var field : search.allConcreteFields()) { if ( field.getDataType() instanceof TensorDataType ) { - validateIndexingScripsForTensorField(field); - validateAttributeSettingForTensorField(field); - processIndexSettingsForTensorField(field); + if (validate) { + validateIndexingScripsForTensorField(field); + validateAttributeSettingForTensorField(field); + } + processIndexSettingsForTensorField(field, validate); } else if (field.getDataType() instanceof CollectionDataType){ - validateDataTypeForCollectionField(field); + if (validate) { + validateDataTypeForCollectionField(field); + } } } } @@ -68,12 +70,12 @@ public class TensorFieldProcessor extends Processor { } } - private void processIndexSettingsForTensorField(SDField field) { + private void processIndexSettingsForTensorField(SDField field, boolean validate) { if (!field.doesIndexing()) { return; } if (isTensorTypeThatSupportsHnswIndex(field)) { - if (!field.doesAttributing()) { + if (validate && !field.doesAttributing()) { fail(search, field, "A tensor that has an index must also be an attribute."); } var index = field.getIndex(field.getName()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java index 5714d41ef67..38037c8a522 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.admin; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.handler.ThreadpoolConfig; +import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Handler; @@ -27,6 +28,12 @@ public class LogserverContainerCluster extends ContainerCluster<LogserverContain builder.maxthreads(10); } + @Override + public void getConfig(QrStartConfig.Builder builder) { + super.getConfig(builder); + builder.jvm.heapsize(384); + } + protected boolean messageBusEnabled() { return false; } private void addLogHandler() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 071666b5bc7..f81757ac568 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -20,6 +20,7 @@ import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; import ai.vespa.metricsproxy.rpc.RpcServer; import ai.vespa.metricsproxy.service.ConfigSentinelClient; import ai.vespa.metricsproxy.service.SystemPollerProvider; +import ai.vespa.metricsproxy.telegraf.TelegrafConfig; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; @@ -67,6 +68,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC ApplicationDimensionsConfig.Producer, ConsumersConfig.Producer, MonitoringConfig.Producer, + TelegrafConfig.Producer, ThreadpoolConfig.Producer, MetricsNodesConfig.Producer { @@ -161,6 +163,25 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC } @Override + public void getConfig(TelegrafConfig.Builder builder) { + var userConsumers = getUserMetricsConsumers(); + for (var consumer : userConsumers.values()) { + for (var cloudWatch : consumer.cloudWatches()) { + var cloudWatchBuilder = new TelegrafConfig.CloudWatch.Builder(); + cloudWatchBuilder + .region(cloudWatch.region()) + .namespace(cloudWatch.namespace()) + .consumer(cloudWatch.consumer()); + cloudWatch.hostedAuth().ifPresent(hostedAuth -> cloudWatchBuilder + .accessKeyName(hostedAuth.accessKeyName) + .secretKeyName(hostedAuth.secretKeyName)); + cloudWatch.profile().ifPresent(cloudWatchBuilder::profile); + builder.cloudWatch(cloudWatchBuilder); + } + } + } + + @Override public void getConfig(ThreadpoolConfig.Builder builder) { builder.maxthreads(10); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java new file mode 100644 index 00000000000..fd290409ea5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java @@ -0,0 +1,49 @@ +package com.yahoo.vespa.model.admin.monitoring; + +import java.util.Optional; + +/** + * Helper object for CloudWatch configuration. + * + * @author gjoranv + */ +public class CloudWatch { + private final String region; + private final String namespace; + private final MetricsConsumer consumer; + + private HostedAuth hostedAuth; + private String profile; + + public CloudWatch(String region, String namespace, MetricsConsumer consumer) { + this.region = region; + this.namespace = namespace; + this.consumer = consumer; + } + + public String region() { return region; } + public String namespace() { return namespace; } + public String consumer() { return consumer.getId(); } + + public Optional<HostedAuth> hostedAuth() {return Optional.ofNullable(hostedAuth); } + public Optional<String> profile() { return Optional.ofNullable(profile); } + + public void setHostedAuth(HostedAuth hostedAuth) { + this.hostedAuth = hostedAuth; + } + + public void setProfile(String profile) { + this.profile = profile; + } + + public static class HostedAuth { + public final String accessKeyName; + public final String secretKeyName; + + public HostedAuth(String accessKeyName, String secretKeyName) { + this.accessKeyName = accessKeyName; + this.secretKeyName = secretKeyName; + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java index 403788e3af6..a8fbcf50b02 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java @@ -2,9 +2,13 @@ package com.yahoo.vespa.model.admin.monitoring; import javax.annotation.concurrent.Immutable; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Objects; +import static java.util.Collections.unmodifiableList; + /** * Represents an arbitrary metric consumer * @@ -17,6 +21,8 @@ public class MetricsConsumer { private final String id; private final MetricSet metricSet; + private final List<CloudWatch> cloudWatches = new ArrayList<>(); + /** * @param id the consumer * @param metricSet the metrics for this consumer @@ -39,4 +45,12 @@ public class MetricsConsumer { return metricSet.getMetrics(); } + public void addCloudWatch(CloudWatch cloudWatch) { + cloudWatches.add(cloudWatch); + } + + public List<CloudWatch> cloudWatches() { + return unmodifiableList(cloudWatches); + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java new file mode 100644 index 00000000000..4b9d5542aa9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java @@ -0,0 +1,36 @@ +package com.yahoo.vespa.model.admin.monitoring.builder.xml; + +import com.yahoo.vespa.model.admin.monitoring.CloudWatch; +import com.yahoo.vespa.model.admin.monitoring.CloudWatch.HostedAuth; +import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; +import org.w3c.dom.Element; + +import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalChildValue; + +/** + * @author gjoranv + */ +public class CloudWatchBuilder { + + private static final String REGION_ATTRIBUTE = "region"; + private static final String NAMESPACE_ATTRIBUTE = "namespace"; + private static final String ACCESS_KEY_ELEMENT = "access-key-name"; + private static final String SECRET_KEY_ELEMENT = "secret-key-name"; + private static final String PROFILE_ELEMENT = "profile"; + + public static CloudWatch buildCloudWatch(Element cloudwatchElement, MetricsConsumer consumer) { + CloudWatch cloudWatch = new CloudWatch(cloudwatchElement.getAttribute(REGION_ATTRIBUTE), + cloudwatchElement.getAttribute(NAMESPACE_ATTRIBUTE), + consumer); + + getOptionalChildValue(cloudwatchElement, PROFILE_ELEMENT).ifPresent(cloudWatch::setProfile); + + getOptionalChildValue(cloudwatchElement, ACCESS_KEY_ELEMENT) + .ifPresent(accessKey -> cloudWatch.setHostedAuth( + new HostedAuth(accessKey, + getOptionalChildValue(cloudwatchElement, SECRET_KEY_ELEMENT) + .orElseThrow(() -> new IllegalArgumentException("Access key given without a secret key."))))); + return cloudWatch; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java index f029dad01a9..b686288868f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java @@ -42,7 +42,11 @@ public class MetricsBuilder { throwIfIllegalConsumerId(metrics, consumerId); MetricSet metricSet = buildMetricSet(consumerId, consumerElement); - metrics.addConsumer(new MetricsConsumer(consumerId, metricSet)); + var consumer = new MetricsConsumer(consumerId, metricSet); + for (Element cloudwatchElement : XML.getChildren(consumerElement, "cloudwatch")) { + consumer.addCloudWatch(CloudWatchBuilder.buildCloudWatch(cloudwatchElement, consumer)); + } + metrics.addConsumer(consumer); } return metrics; } @@ -58,7 +62,7 @@ public class MetricsBuilder { private MetricSet buildMetricSet(String consumerId, Element consumerElement) { List<Metric> metrics = XML.getChildren(consumerElement, "metric").stream() - .map(metricElement -> metricFromElement(metricElement)) + .map(MetricsBuilder::metricFromElement) .collect(Collectors.toCollection(LinkedList::new)); List<MetricSet> metricSets = XML.getChildren(consumerElement, "metric-set").stream() diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java index f00ad0f0dbb..4b8bbd4ff08 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java @@ -12,7 +12,7 @@ public class EndpointCertificateSecretsValidator extends Validator { @Override public void validate(VespaModel model, DeployState deployState) { if (deployState.endpointCertificateSecrets().isPresent() && deployState.endpointCertificateSecrets().get() == EndpointCertificateSecrets.MISSING) { - throw new CertificateNotReadyException("TLS enabled, but could not retrieve certificate yet"); + throw new CertificateNotReadyException("TLS enabled, but could not yet retrieve certificate for application " + deployState.getProperties().applicationId().serializedForm()); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 58679c63565..3632cb08da5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -215,6 +215,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat super.getConfig(builder); builder.jvm.verbosegc(true) .availableProcessors(0) + .compressedClassSpaceSize(0) //TODO Reduce, next step is 512m .minHeapsize(1536) .heapsize(1536); if (getMemoryPercentage().isPresent()) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 6fa446bf365..3b132ab2342 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -484,6 +484,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> builder.jvm .verbosegc(false) .availableProcessors(2) + .compressedClassSpaceSize(32) .minHeapsize(32) .heapsize(512) .heapSizeAsPercentageOfPhysicalMemory(0) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java index 6b1a94e16ae..67c7b67ad9e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.http; -import com.google.common.collect.ImmutableList; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.config.application.api.DeployLogger; @@ -30,7 +29,7 @@ public final class AccessControl { public static final ComponentId ACCESS_CONTROL_CHAIN_ID = ComponentId.fromString("access-control-chain"); - private static final List<String> UNPROTECTED_HANDLERS = ImmutableList.of( + public static final List<String> UNPROTECTED_HANDLERS = List.of( FileStatusHandlerComponent.CLASS, ContainerCluster.APPLICATION_STATUS_HANDLER_CLASS, ContainerCluster.BINDINGS_OVERVIEW_HANDLER_CLASS, diff --git a/config-model/src/main/perl/vespa-deploy b/config-model/src/main/perl/vespa-deploy index 59a84f5b0c0..a128e4a8d4c 100755 --- a/config-model/src/main/perl/vespa-deploy +++ b/config-model/src/main/perl/vespa-deploy @@ -154,7 +154,7 @@ my $command = shift; $command ||= "help"; # The '--insecure' parameter is sadly required as it is not possible to disable or alter hostname verification with curl -my $curl_command = $VESPA_HOME . '/libexec/vespa/vespa-curl-wrapper --insecure -A vespa-deploy --silent --show-error --connect-timeout 30 --max-time 1200'; +my $curl_command = $VESPA_HOME . '/libexec/vespa/vespa-curl-wrapper -A vespa-deploy --silent --show-error --connect-timeout 30 --max-time 1200'; my $CURL_PUT = $curl_command . ' --write-out \%{http_code} --request PUT'; my $CURL_GET = $curl_command . ' --request GET'; diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc index 7a3e2916f94..e3ba7dc500d 100644 --- a/config-model/src/main/resources/schema/admin.rnc +++ b/config-model/src/main/resources/schema/admin.rnc @@ -82,10 +82,25 @@ Metrics = element metrics { element metric { attribute id { xsd:Name } & attribute display-name { xsd:Name }? - }* + }* & + Cloudwatch? }+ } +Cloudwatch = element cloudwatch { + attribute region { xsd:Name } & + attribute namespace { xsd:string { pattern = "[\w_\-/#:\.]+" } } & + ( + ( + element access-key-name { xsd:Name } & + element secret-key-name { xsd:Name } + ) + | + element profile { xsd:Name } + )? + +} + ClusterControllers = element cluster-controllers { attribute standalone-zookeeper { xsd:string }? & element cluster-controller { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java new file mode 100644 index 00000000000..b441f1e1993 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java @@ -0,0 +1,244 @@ +package com.yahoo.vespa.model.admin.metricsproxy; + +import ai.vespa.metricsproxy.core.ConsumersConfig; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.admin.monitoring.Metric; +import com.yahoo.vespa.model.admin.monitoring.MetricSet; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.checkMetric; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromModel; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromXml; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getCustomConsumer; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.servicesWithAdminOnly; +import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicConsumer.DEFAULT_PUBLIC_CONSUMER_ID; +import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicMetrics.defaultPublicMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.VESPA_CONSUMER_ID; +import static java.util.Collections.singleton; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Tests for {@link MetricsProxyContainerCluster} related to metrics consumers. + * + * @author gjoranv + */ +public class MetricsConsumersTest { + + private static int numPublicDefaultMetrics = defaultPublicMetricSet.getMetrics().size(); + private static int numDefaultVespaMetrics = defaultVespaMetricSet.getMetrics().size(); + private static int numVespaMetrics = vespaMetricSet.getMetrics().size(); + private static int numSystemMetrics = systemMetricSet.getMetrics().size(); + private static int numNetworkMetrics = networkMetricSet.getMetrics().size(); + private static int numMetricsForVespaConsumer = numVespaMetrics + numSystemMetrics + numNetworkMetrics; + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Test + public void default_public_consumer_is_set_up_for_self_hosted() { + ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted); + assertEquals(2, config.consumer().size()); + assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID); + + int numMetricsForPublicDefaultConsumer = defaultPublicMetricSet.getMetrics().size() + numSystemMetrics; + assertEquals(numMetricsForPublicDefaultConsumer, config.consumer(1).metric().size()); + } + + @Test + public void vespa_consumer_and_default_public_consumer_is_set_up_for_hosted() { + ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), hosted); + assertEquals(2, config.consumer().size()); + assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID); + assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID); + } + + @Test + public void vespa_consumer_is_always_present_and_has_all_vespa_metrics_and_all_system_metrics() { + ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted); + assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID); + assertEquals(numMetricsForVespaConsumer, config.consumer(0).metric().size()); + } + + @Test + public void vespa_consumer_can_be_amended_via_admin_object() { + VespaModel model = getModel(servicesWithAdminOnly(), self_hosted); + var additionalMetric = new Metric("additional-metric"); + model.getAdmin().setAdditionalDefaultMetrics(new MetricSet("amender-metrics", singleton(additionalMetric))); + + ConsumersConfig config = consumersConfigFromModel(model); + assertEquals(numMetricsForVespaConsumer + 1, config.consumer(0).metric().size()); + + ConsumersConfig.Consumer vespaConsumer = config.consumer(0); + assertTrue("Did not contain additional metric", checkMetric(vespaConsumer, additionalMetric)); + } + + @Test + public void vespa_is_a_reserved_consumer_id() { + assertReservedConsumerId("Vespa"); + } + + @Test + public void default_is_a_reserved_consumer_id() { + assertReservedConsumerId("default"); + } + + private void assertReservedConsumerId(String consumerId) { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='" + consumerId + "'/>", + " </metrics>", + " </admin>", + "</services>" + ); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("'" + consumerId + "' is not allowed as metrics consumer id"); + consumersConfigFromXml(services, self_hosted); + } + + @Test + public void vespa_consumer_id_is_allowed_for_hosted_infrastructure_applications() { + String services = String.join("\n", + "<services application-type='hosted-infrastructure'>", + " <admin version='4.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='Vespa'>", + " <metric id='custom.metric1'/>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + VespaModel hostedModel = getModel(services, hosted); + ConsumersConfig config = consumersConfigFromModel(hostedModel); + assertEquals(2, config.consumer().size()); + + // All default metrics are retained + ConsumersConfig.Consumer vespaConsumer = config.consumer(0); + assertEquals(numMetricsForVespaConsumer + 1, vespaConsumer.metric().size()); + + Metric customMetric1 = new Metric("custom.metric1"); + assertTrue("Did not contain metric: " + customMetric1, checkMetric(vespaConsumer, customMetric1)); + } + + @Test + public void consumer_id_is_case_insensitive() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='A'/>", + " <consumer id='a'/>", + " </metrics>", + " </admin>", + "</services>" + ); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("'a' is used as id for two metrics consumers"); + consumersConfigFromXml(services, self_hosted); + } + + @Test + public void non_existent_metric_set_causes_exception() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='consumer-with-non-existent-default-set'>", + " <metric-set id='non-existent'/>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("No such metric-set: non-existent"); + consumersConfigFromXml(services, self_hosted); + } + + @Test + public void consumer_with_no_metric_set_has_its_own_metrics_plus_system_metrics_plus_default_vespa_metrics() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='consumer-with-metrics-only'>", + " <metric id='custom.metric1'/>", + " <metric id='custom.metric2'/>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + ConsumersConfig.Consumer consumer = getCustomConsumer(services); + + assertEquals(numSystemMetrics + numDefaultVespaMetrics + 2, consumer.metric().size()); + + Metric customMetric1 = new Metric("custom.metric1"); + Metric customMetric2 = new Metric("custom.metric2"); + assertTrue("Did not contain metric: " + customMetric1, checkMetric(consumer, customMetric1)); + assertTrue("Did not contain metric: " + customMetric2, checkMetric(consumer, customMetric2)); + } + + @Test + public void consumer_with_default_public_metric_set_has_all_public_metrics_plus_all_system_metrics_plus_its_own() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='consumer-with-public-default-set'>", + " <metric-set id='public'/>", + " <metric id='custom.metric'/>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + ConsumersConfig.Consumer consumer = getCustomConsumer(services); + + assertEquals(numPublicDefaultMetrics + numSystemMetrics + 1, consumer.metric().size()); + + Metric customMetric = new Metric("custom.metric"); + assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric)); + } + + @Test + public void consumer_with_vespa_metric_set_has_all_vespa_metrics_plus_all_system_metrics_plus_its_own() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='consumer-with-vespa-set'>", + " <metric-set id='vespa'/>", + " <metric id='my.extra.metric'/>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + ConsumersConfig.Consumer consumer = getCustomConsumer(services); + assertEquals(numVespaMetrics + numSystemMetrics + 1, consumer.metric().size()); + + Metric customMetric = new Metric("my.extra.metric"); + assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric)); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index 9265e4437f1..bed77bd5c77 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -5,10 +5,9 @@ package com.yahoo.vespa.model.admin.metricsproxy; -import ai.vespa.metricsproxy.core.ConsumersConfig; -import ai.vespa.metricsproxy.http.metrics.MetricsV1Handler; import ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler; import ai.vespa.metricsproxy.http.application.MetricsNodesConfig; +import ai.vespa.metricsproxy.http.metrics.MetricsV1Handler; import ai.vespa.metricsproxy.http.prometheus.PrometheusHandler; import ai.vespa.metricsproxy.http.yamas.YamasHandler; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; @@ -21,13 +20,9 @@ import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames; -import com.yahoo.vespa.model.admin.monitoring.Metric; -import com.yahoo.vespa.model.admin.monitoring.MetricSet; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import java.util.Collection; @@ -39,26 +34,17 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.M import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.MY_TENANT; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.checkMetric; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromModel; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromXml; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getApplicationDimensionsConfig; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getCustomConsumer; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getMetricsNodesConfig; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getQrStartConfig; -import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicConsumer.DEFAULT_PUBLIC_CONSUMER_ID; -import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicMetrics.defaultPublicMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.VESPA_CONSUMER_ID; -import static java.util.Collections.singleton; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.servicesWithAdminOnly; import static java.util.stream.Collectors.toList; import static org.hamcrest.CoreMatchers.endsWith; import static org.hamcrest.CoreMatchers.hasItem; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -67,16 +53,6 @@ import static org.junit.Assert.assertTrue; */ public class MetricsProxyContainerClusterTest { - private static int numPublicDefaultMetrics = defaultPublicMetricSet.getMetrics().size(); - private static int numDefaultVespaMetrics = defaultVespaMetricSet.getMetrics().size(); - private static int numVespaMetrics = vespaMetricSet.getMetrics().size(); - private static int numSystemMetrics = systemMetricSet.getMetrics().size(); - private static int numNetworkMetrics = networkMetricSet.getMetrics().size(); - private static int numMetricsForVespaConsumer = numVespaMetrics + numSystemMetrics + numNetworkMetrics; - - @Rule - public ExpectedException thrown = ExpectedException.none(); - @Test public void metrics_proxy_bundle_is_included_in_bundles_config() { VespaModel model = getModel(servicesWithAdminOnly(), self_hosted); @@ -105,10 +81,11 @@ public class MetricsProxyContainerClusterTest { assertEquals(512, qrStartConfig.jvm().heapsize()); assertEquals(0, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory()); assertEquals(2, qrStartConfig.jvm().availableProcessors()); - assertEquals(false, qrStartConfig.jvm().verbosegc()); + assertFalse(qrStartConfig.jvm().verbosegc()); assertEquals("-XX:+UseG1GC -XX:MaxTenuringThreshold=15", qrStartConfig.jvm().gcopts()); assertEquals(512, qrStartConfig.jvm().stacksize()); assertEquals(0, qrStartConfig.jvm().directMemorySizeCache()); + assertEquals(32, qrStartConfig.jvm().compressedClassSpaceSize()); assertEquals(75, qrStartConfig.jvm().baseMaxDirectMemorySize()); } @@ -131,203 +108,6 @@ public class MetricsProxyContainerClusterTest { } @Test - public void default_public_consumer_is_set_up_for_self_hosted() { - ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted); - assertEquals(2, config.consumer().size()); - assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID); - - int numMetricsForPublicDefaultConsumer = defaultPublicMetricSet.getMetrics().size() + numSystemMetrics; - assertEquals(numMetricsForPublicDefaultConsumer, config.consumer(1).metric().size()); - } - - @Test - public void vespa_consumer_and_default_public_consumer_is_set_up_for_hosted() { - ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), hosted); - assertEquals(2, config.consumer().size()); - assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID); - assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID); - } - - @Test - public void vespa_consumer_is_always_present_and_has_all_vespa_metrics_and_all_system_metrics() { - ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted); - assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID); - assertEquals(numMetricsForVespaConsumer, config.consumer(0).metric().size()); - } - - @Test - public void vespa_consumer_can_be_amended_via_admin_object() { - VespaModel model = getModel(servicesWithAdminOnly(), self_hosted); - var additionalMetric = new Metric("additional-metric"); - model.getAdmin().setAdditionalDefaultMetrics(new MetricSet("amender-metrics", singleton(additionalMetric))); - - ConsumersConfig config = consumersConfigFromModel(model); - assertEquals(numMetricsForVespaConsumer + 1, config.consumer(0).metric().size()); - - ConsumersConfig.Consumer vespaConsumer = config.consumer(0); - assertTrue("Did not contain additional metric", checkMetric(vespaConsumer, additionalMetric)); - } - - @Test - public void vespa_is_a_reserved_consumer_id() { - assertReservedConsumerId("Vespa"); - } - - @Test - public void default_is_a_reserved_consumer_id() { - assertReservedConsumerId("default"); - } - - private void assertReservedConsumerId(String consumerId) { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='" + consumerId + "'/>", - " </metrics>", - " </admin>", - "</services>" - ); - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("'" + consumerId + "' is not allowed as metrics consumer id"); - consumersConfigFromXml(services, self_hosted); - } - - @Test - public void vespa_consumer_id_is_allowed_for_hosted_infrastructure_applications() { - String services = String.join("\n", - "<services application-type='hosted-infrastructure'>", - " <admin version='4.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='Vespa'>", - " <metric id='custom.metric1'/>", - " </consumer>", - " </metrics>", - " </admin>", - "</services>" - ); - VespaModel hostedModel = getModel(services, hosted); - ConsumersConfig config = consumersConfigFromModel(hostedModel); - assertEquals(2, config.consumer().size()); - - // All default metrics are retained - ConsumersConfig.Consumer vespaConsumer = config.consumer(0); - assertEquals(numMetricsForVespaConsumer + 1, vespaConsumer.metric().size()); - - Metric customMetric1 = new Metric("custom.metric1"); - assertTrue("Did not contain metric: " + customMetric1, checkMetric(vespaConsumer, customMetric1)); - } - - @Test - public void consumer_id_is_case_insensitive() { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='A'/>", - " <consumer id='a'/>", - " </metrics>", - " </admin>", - "</services>" - ); - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("'a' is used as id for two metrics consumers"); - consumersConfigFromXml(services, self_hosted); - } - - @Test - public void non_existent_metric_set_causes_exception() { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='consumer-with-non-existent-default-set'>", - " <metric-set id='non-existent'/>", - " </consumer>", - " </metrics>", - " </admin>", - "</services>" - ); - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("No such metric-set: non-existent"); - consumersConfigFromXml(services, self_hosted); - } - - @Test - public void consumer_with_no_metric_set_has_its_own_metrics_plus_system_metrics_plus_default_vespa_metrics() { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='consumer-with-metrics-only'>", - " <metric id='custom.metric1'/>", - " <metric id='custom.metric2'/>", - " </consumer>", - " </metrics>", - " </admin>", - "</services>" - ); - ConsumersConfig.Consumer consumer = getCustomConsumer(services); - - assertEquals(numSystemMetrics + numDefaultVespaMetrics + 2, consumer.metric().size()); - - Metric customMetric1 = new Metric("custom.metric1"); - Metric customMetric2 = new Metric("custom.metric2"); - assertTrue("Did not contain metric: " + customMetric1, checkMetric(consumer, customMetric1)); - assertTrue("Did not contain metric: " + customMetric2, checkMetric(consumer, customMetric2)); - } - - @Test - public void consumer_with_default_public_metric_set_has_all_public_metrics_plus_all_system_metrics_plus_its_own() { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='consumer-with-public-default-set'>", - " <metric-set id='public'/>", - " <metric id='custom.metric'/>", - " </consumer>", - " </metrics>", - " </admin>", - "</services>" - ); - ConsumersConfig.Consumer consumer = getCustomConsumer(services); - - assertEquals(numPublicDefaultMetrics + numSystemMetrics + 1, consumer.metric().size()); - - Metric customMetric = new Metric("custom.metric"); - assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric)); - } - - @Test - public void consumer_with_vespa_metric_set_has_all_vespa_metrics_plus_all_system_metrics_plus_its_own() { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='consumer-with-vespa-set'>", - " <metric-set id='vespa'/>", - " <metric id='my.extra.metric'/>", - " </consumer>", - " </metrics>", - " </admin>", - "</services>" - ); - ConsumersConfig.Consumer consumer = getCustomConsumer(services); - assertEquals(numVespaMetrics + numSystemMetrics + 1, consumer.metric().size()); - - Metric customMetric = new Metric("my.extra.metric"); - assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric)); - } - - @Test public void hosted_application_propagates_application_dimensions() { VespaModel hostedModel = getModel(servicesWithAdminOnly(), hosted); ApplicationDimensionsConfig config = getApplicationDimensionsConfig(hostedModel); @@ -357,16 +137,6 @@ public class MetricsProxyContainerClusterTest { assertEquals(MetricsV1Handler.VALUES_PATH, node.metricsPath()); } - private static String servicesWithAdminOnly() { - return String.join("\n", - "<services>", - " <admin version='4.0'>", - " <adminserver hostalias='node1'/>", - " </admin>", - "</services>" - ); - } - private static String servicesWithTwoNodes() { return String.join("\n", "<services>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java index 7cbc9db5eb2..8ecb13d7ae5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java @@ -55,6 +55,16 @@ class MetricsProxyModelTester { : CONTAINER_CONFIG_ID; } + static String servicesWithAdminOnly() { + return String.join("\n", + "<services>", + " <admin version='4.0'>", + " <adminserver hostalias='node1'/>", + " </admin>", + "</services>" + ); + } + static boolean checkMetric(ConsumersConfig.Consumer consumer, Metric metric) { for (ConsumersConfig.Consumer.Metric m : consumer.metric()) { if (metric.name.equals(m.name()) && metric.outputName.equals(m.outputname())) @@ -77,32 +87,32 @@ class MetricsProxyModelTester { } static ConsumersConfig consumersConfigFromModel(VespaModel model) { - return new ConsumersConfig((ConsumersConfig.Builder) model.getConfig(new ConsumersConfig.Builder(), CLUSTER_CONFIG_ID)); + return model.getConfig(ConsumersConfig.class, CLUSTER_CONFIG_ID); } static MetricsNodesConfig getMetricsNodesConfig(VespaModel model) { - return new MetricsNodesConfig((MetricsNodesConfig.Builder) model.getConfig(new MetricsNodesConfig.Builder(), CLUSTER_CONFIG_ID)); + return model.getConfig(MetricsNodesConfig.class, CLUSTER_CONFIG_ID); } static ApplicationDimensionsConfig getApplicationDimensionsConfig(VespaModel model) { - return new ApplicationDimensionsConfig((ApplicationDimensionsConfig.Builder) model.getConfig(new ApplicationDimensionsConfig.Builder(), CLUSTER_CONFIG_ID)); + return model.getConfig(ApplicationDimensionsConfig.class, CLUSTER_CONFIG_ID); } static QrStartConfig getQrStartConfig(VespaModel model) { - return new QrStartConfig((QrStartConfig.Builder) model.getConfig(new QrStartConfig.Builder(), CLUSTER_CONFIG_ID)); + return model.getConfig(QrStartConfig.class, CLUSTER_CONFIG_ID); } static NodeDimensionsConfig getNodeDimensionsConfig(VespaModel model, String configId) { - return new NodeDimensionsConfig((NodeDimensionsConfig.Builder) model.getConfig(new NodeDimensionsConfig.Builder(), configId)); + return model.getConfig(NodeDimensionsConfig.class, configId); } static VespaServicesConfig getVespaServicesConfig(String servicesXml) { VespaModel model = getModel(servicesXml, self_hosted); - return new VespaServicesConfig((VespaServicesConfig.Builder) model.getConfig(new VespaServicesConfig.Builder(), CONTAINER_CONFIG_ID)); + return model.getConfig(VespaServicesConfig.class, CONTAINER_CONFIG_ID); } static RpcConnectorConfig getRpcConnectorConfig(VespaModel model) { - return new RpcConnectorConfig((RpcConnectorConfig.Builder) model.getConfig(new RpcConnectorConfig.Builder(), CONTAINER_CONFIG_ID)); + return model.getConfig(RpcConnectorConfig.class, CONTAINER_CONFIG_ID); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java new file mode 100644 index 00000000000..144c45a7dd2 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java @@ -0,0 +1,87 @@ +package com.yahoo.vespa.model.admin.metricsproxy; + +import ai.vespa.metricsproxy.telegraf.TelegrafConfig; +import com.yahoo.vespa.model.VespaModel; +import org.junit.Test; + +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CLUSTER_CONFIG_ID; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; +import static org.junit.Assert.assertEquals; + +/** + * @author gjoranv + */ +public class TelegrafTest { + + @Test + public void telegraf_config_is_generated_for_cloudwatch_in_services() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='cloudwatch-consumer'>", + " <metric id='my-metric'/>", + " <cloudwatch region='us-east-1' namespace='my-namespace' >", + " <access-key-name>my-access-key</access-key-name>", + " <secret-key-name>my-secret-key</secret-key-name>", + " </cloudwatch>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + VespaModel hostedModel = getModel(services, hosted); + TelegrafConfig config = hostedModel.getConfig(TelegrafConfig.class, CLUSTER_CONFIG_ID); + var cloudWatch0 = config.cloudWatch(0); + assertEquals("cloudwatch-consumer", cloudWatch0.consumer()); + assertEquals("us-east-1", cloudWatch0.region()); + assertEquals("my-namespace", cloudWatch0.namespace()); + assertEquals("my-access-key", cloudWatch0.accessKeyName()); + assertEquals("my-secret-key", cloudWatch0.secretKeyName()); + assertEquals("", cloudWatch0.profile()); + } + + @Test + public void multiple_cloudwatches_are_allowed_for_the_same_consumer() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='cloudwatch-consumer'>", + " <metric id='my-metric'/>", + " <cloudwatch region='us-east-1' namespace='namespace-1' >", + " <access-key-name>access-key-1</access-key-name>", + " <secret-key-name>secret-key-1</secret-key-name>", + " </cloudwatch>", + " <cloudwatch region='us-east-1' namespace='namespace-2' >", + " <profile>profile-2</profile>", + " </cloudwatch>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + VespaModel hostedModel = getModel(services, hosted); + TelegrafConfig config = hostedModel.getConfig(TelegrafConfig.class, CLUSTER_CONFIG_ID); + + var cloudWatch0 = config.cloudWatch(0); + assertEquals("cloudwatch-consumer", cloudWatch0.consumer()); + assertEquals("us-east-1", cloudWatch0.region()); + assertEquals("namespace-1", cloudWatch0.namespace()); + assertEquals("access-key-1", cloudWatch0.accessKeyName()); + assertEquals("secret-key-1", cloudWatch0.secretKeyName()); + assertEquals("", cloudWatch0.profile()); + + var cloudWatch1 = config.cloudWatch(1); + assertEquals("cloudwatch-consumer", cloudWatch1.consumer()); + assertEquals("us-east-1", cloudWatch1.region()); + assertEquals("namespace-2", cloudWatch1.namespace()); + assertEquals("", cloudWatch1.accessKeyName()); + assertEquals("", cloudWatch1.secretKeyName()); + assertEquals("profile-2", cloudWatch1.profile()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java index 21df39ebde8..318a0630c4d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java @@ -47,7 +47,7 @@ public class EndpointCertificateSecretsValidatorTest { VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); exceptionRule.expect(CertificateNotReadyException.class); - exceptionRule.expectMessage("TLS enabled, but could not retrieve certificate yet"); + exceptionRule.expectMessage("TLS enabled, but could not yet retrieve certificate for application default:default:default"); new EndpointCertificateSecretsValidator().validate(model, deployState); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index d1c5e344c24..ce565989c18 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -97,6 +97,7 @@ public class ContainerClusterTest { cluster.getConfig(qsB); QrStartConfig qsC= new QrStartConfig(qsB); assertEquals(expectedMemoryPercentage, qsC.jvm().heapSizeAsPercentageOfPhysicalMemory()); + assertEquals(0, qsC.jvm().compressedClassSpaceSize()); } @Test @@ -156,6 +157,7 @@ public class ContainerClusterTest { QrStartConfig qrStartConfig = new QrStartConfig(qrBuilder); assertEquals(32, qrStartConfig.jvm().minHeapsize()); assertEquals(512, qrStartConfig.jvm().heapsize()); + assertEquals(32, qrStartConfig.jvm().compressedClassSpaceSize()); assertEquals(0, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory()); ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java index 9ecd33f4273..eda90b03147 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java @@ -28,7 +28,11 @@ import java.util.Map; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.core.IsNull.notNullValue; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertThat; + /** * @author einarmr @@ -204,24 +208,25 @@ public class DocprocBuilderTest extends DomBuilderTest { @Test public void testBundlesConfig() { - assertThat(bundlesConfig.bundle().size(), is(0)); + assertTrue(bundlesConfig.bundle().isEmpty()); } @Test public void testSchemaMappingConfig() { - assertThat(schemamappingConfig.fieldmapping().size(), is(0)); + assertTrue(schemamappingConfig.fieldmapping().isEmpty()); } @Test public void testQrStartConfig() { QrStartConfig.Jvm jvm = qrStartConfig.jvm(); - assertThat(jvm.server(), is(true)); - assertThat(jvm.verbosegc(), is(true)); - assertThat(jvm.gcopts(), is("-XX:+UseG1GC -XX:MaxTenuringThreshold=15")); - assertThat(jvm.minHeapsize(), is(1536)); - assertThat(jvm.heapsize(), is(1536)); - assertThat(jvm.stacksize(), is(512)); - assertThat(qrStartConfig.ulimitv(), is("")); + assertTrue(jvm.server()); + assertTrue(jvm.verbosegc()); + assertEquals("-XX:+UseG1GC -XX:MaxTenuringThreshold=15", jvm.gcopts()); + assertEquals(1536, jvm.minHeapsize()); + assertEquals(1536, jvm.heapsize()); + assertEquals(512, jvm.stacksize()); + assertTrue(qrStartConfig.ulimitv().isEmpty()); + assertEquals(0, jvm.compressedClassSpaceSize()); } } diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 1bf42650123..604e2abbbae 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -15,14 +15,25 @@ <slobrok hostalias="rtc-1" /> </slobroks> <metrics> - <consumer id="my-consumer"> + <consumer id="cloudwatch-hosted"> <metric-set id="my-set" /> <metric id="my-metric"/> <metric id="my-metric2" display-name="my-metric3"/> <metric display-name="my-metric4" id="my-metric4.avg"/> + <cloudwatch region="us-east1" namespace="my-namespace"> + <access-key-name>my-access-key</access-key-name> + <secret-key-name>my-secret-key</secret-key-name> + </cloudwatch> </consumer> - <consumer id="my-consumer2"> - <metric-set id="my-set2" /> + <consumer id="cloudwatch-self-hosted-with-default-auth"> + <metric-set id="public" /> + <cloudwatch region="us-east1" namespace="namespace_legal.chars:/#1" /> + </consumer> + <consumer id="cloudwatch-self-hosted-with-profile"> + <metric id="my-custom-metric" /> + <cloudwatch region="us-east1" namespace="another-namespace"> + <profile>profile-in-credentials-file</profile> + </cloudwatch> </consumer> </metrics> <logforwarding> diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index 92432d1cab4..9a091f1161c 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -507,7 +507,7 @@ ], "methods": [ "public abstract java.util.Optional getDeployment(com.yahoo.config.provision.ApplicationId)", - "public abstract java.util.Map getSupportedInfraDeployments()" + "public abstract void activateAllSupportedInfraApplications(boolean)" ], "fields": [] }, diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java index 6fbabfd0c95..363732ee8a7 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java @@ -1,7 +1,6 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import java.util.Map; import java.util.Optional; /** @@ -17,6 +16,6 @@ public interface InfraDeployer { */ Optional<Deployment> getDeployment(ApplicationId application); - /** Returns deployments by application id for the supported infrastructure applications in this zone */ - Map<ApplicationId, Deployment> getSupportedInfraDeployments(); + /** Deploys all supported infrastructure applications in this zone. */ + void activateAllSupportedInfraApplications(boolean propagateException); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java index 892ac639198..02da2ab98e2 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java @@ -14,4 +14,12 @@ public enum RoutingMethod { /** Routing happens through a dedicated layer 4 load balancer */ exclusive, + /** Routing happens through a shared layer 4 load balancer */ + sharedLayer4; + + /** Returns whether this method routes requests directly to the Vespa container cluster */ + public boolean isDirect() { + return this == exclusive || this == sharedLayer4; + } + } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java index 9d33824e0be..b73cf89d1b4 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java @@ -162,9 +162,9 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer dispatchRpcRequest(req, () -> { StringBuilder sb = new StringBuilder(); sb.append("\nDelayed responses queue size: "); - sb.append(proxyServer.delayedResponses.size()); + sb.append(proxyServer.delayedResponses().size()); sb.append("\nContents: "); - for (DelayedResponse delayed : proxyServer.delayedResponses.responses()) { + for (DelayedResponse delayed : proxyServer.delayedResponses().responses()) { sb.append(delayed.getRequest().toString()).append("\n"); } @@ -357,7 +357,7 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer @Override public void notifyTargetInvalid(Target target) { log.log(LogLevel.DEBUG, () -> "Target invalid " + target); - for (Iterator<DelayedResponse> it = proxyServer.delayedResponses.responses().iterator(); it.hasNext(); ) { + for (Iterator<DelayedResponse> it = proxyServer.delayedResponses().responses().iterator(); it.hasNext(); ) { DelayedResponse delayed = it.next(); JRTServerConfigRequest request = delayed.getRequest(); if (request.getRequest().target().equals(target)) { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java index 2f8a1b463e3..9fb78e7e812 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java @@ -26,4 +26,6 @@ interface ConfigSourceClient { void updateSubscribers(RawConfig config); + DelayedResponses delayedResponses(); + } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java index f9f5c475723..51446882025 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java @@ -18,6 +18,7 @@ class MemoryCacheConfigClient implements ConfigSourceClient { private final static Logger log = Logger.getLogger(MemoryCacheConfigClient.class.getName()); private final MemoryCache cache; + private final DelayedResponses delayedResponses = new DelayedResponses(); MemoryCacheConfigClient(MemoryCache cache) { this.cache = cache; @@ -61,4 +62,9 @@ class MemoryCacheConfigClient implements ConfigSourceClient { @Override public void updateSubscribers(RawConfig config) {} + @Override + public DelayedResponses delayedResponses() { + return delayedResponses; + } + } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java index d77206aee81..545b962f6ff 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.proxy; -import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Spec; import com.yahoo.jrt.Supervisor; @@ -10,20 +9,15 @@ import com.yahoo.log.LogLevel; import com.yahoo.log.LogSetup; import com.yahoo.log.event.Event; import com.yahoo.vespa.config.RawConfig; -import com.yahoo.vespa.config.TimingValues; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import com.yahoo.vespa.config.proxy.filedistribution.FileDistributionAndUrlDownload; import com.yahoo.yolean.system.CatchSignals; import java.util.List; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Logger; import static com.yahoo.vespa.config.proxy.Mode.ModeName.DEFAULT; -import static java.util.concurrent.TimeUnit.SECONDS; /** * A proxy server that handles RPC config requests. The proxy can run in two modes: @@ -40,59 +34,34 @@ public class ProxyServer implements Runnable { private final static Logger log = Logger.getLogger(ProxyServer.class.getName()); private final AtomicBoolean signalCaught = new AtomicBoolean(false); - - // Scheduled executor that periodically checks for requests that have timed out and response should be returned to clients - private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1, new DaemonThreadFactory()); private final Supervisor supervisor = new Supervisor(new Transport(JRT_TRANSPORT_THREADS)); - private ScheduledFuture<?> delayedResponseScheduler; private final ConfigProxyRpcServer rpcServer; - final DelayedResponses delayedResponses; private ConfigSourceSet configSource; private volatile ConfigSourceClient configClient; - private final TimingValues timingValues; private final MemoryCache memoryCache; - private static final double timingValuesRatio = 0.8; - private final static TimingValues defaultTimingValues; private final FileDistributionAndUrlDownload fileDistributionAndUrlDownload; private volatile Mode mode = new Mode(DEFAULT); - static { - // Proxy should time out before clients upon subscription. - TimingValues tv = new TimingValues(); - tv.setUnconfiguredDelay((long)(tv.getUnconfiguredDelay()* timingValuesRatio)). - setConfiguredErrorDelay((long)(tv.getConfiguredErrorDelay()* timingValuesRatio)). - setSubscribeTimeout((long)(tv.getSubscribeTimeout()* timingValuesRatio)). - setConfiguredErrorTimeout(-1); // Never cache errors - defaultTimingValues = tv; - } - - ProxyServer(Spec spec, ConfigSourceSet source, TimingValues timingValues, - MemoryCache memoryCache, ConfigSourceClient configClient) { - this.delayedResponses = new DelayedResponses(); + ProxyServer(Spec spec, ConfigSourceSet source, MemoryCache memoryCache, ConfigSourceClient configClient) { this.configSource = source; log.log(LogLevel.DEBUG, "Using config source '" + source); - this.timingValues = timingValues; this.memoryCache = memoryCache; this.rpcServer = createRpcServer(spec); - this.configClient = createClient(rpcServer, delayedResponses, source, timingValues, memoryCache, configClient); + this.configClient = (configClient == null) ? createRpcClient(rpcServer, source, memoryCache) : configClient; this.fileDistributionAndUrlDownload = new FileDistributionAndUrlDownload(supervisor, source); } + @Override public void run() { if (rpcServer != null) { Thread t = new Thread(rpcServer); t.setName("RpcServer"); t.start(); } - // Wait for 5 seconds initially, then run every second - delayedResponseScheduler = scheduler.scheduleAtFixedRate(new DelayedResponseHandler(delayedResponses, - memoryCache, - rpcServer), - 5, 1, SECONDS); } RawConfig resolveConfig(JRTServerConfigRequest req) { @@ -124,7 +93,7 @@ public class ProxyServer implements Runnable { break; case DEFAULT: flush(); - configClient = createRpcClient(); + configClient = createRpcClient(rpcServer, configSource, memoryCache); this.mode = new Mode(modeName); break; default: @@ -133,20 +102,12 @@ public class ProxyServer implements Runnable { log.log(LogLevel.INFO, "Switched from '" + oldMode.name().toLowerCase() + "' mode to '" + getMode().name().toLowerCase() + "' mode"); } - private ConfigSourceClient createClient(RpcServer rpcServer, DelayedResponses delayedResponses, - ConfigSourceSet source, TimingValues timingValues, - MemoryCache memoryCache, ConfigSourceClient client) { - return (client == null) - ? new RpcConfigSourceClient(rpcServer, source, memoryCache, timingValues, delayedResponses) - : client; - } - private ConfigProxyRpcServer createRpcServer(Spec spec) { return (spec == null) ? null : new ConfigProxyRpcServer(this, supervisor, spec); // TODO: Try to avoid first argument being 'this' } - private RpcConfigSourceClient createRpcClient() { - return new RpcConfigSourceClient(rpcServer, configSource, memoryCache, timingValues, delayedResponses); + private static RpcConfigSourceClient createRpcClient(RpcServer rpcServer, ConfigSourceSet source, MemoryCache memoryCache) { + return new RpcConfigSourceClient(rpcServer, source, memoryCache); } private void setupSignalHandler() { @@ -181,8 +142,7 @@ public class ProxyServer implements Runnable { Event.started("configproxy"); ConfigSourceSet configSources = new ConfigSourceSet(properties.configSources); - ProxyServer proxyServer = new ProxyServer(new Spec(null, port), configSources, - defaultTimingValues(), new MemoryCache(), null); + ProxyServer proxyServer = new ProxyServer(new Spec(null, port), configSources, new MemoryCache(), null); // catch termination and interrupt signal proxyServer.setupSignalHandler(); Thread proxyserverThread = new Thread(proxyServer); @@ -204,14 +164,6 @@ public class ProxyServer implements Runnable { } } - static TimingValues defaultTimingValues() { - return defaultTimingValues; - } - - TimingValues getTimingValues() { - return timingValues; - } - // Cancels all config instances and flushes the cache. When this method returns, // the cache will not be updated again before someone calls getConfig(). private synchronized void flush() { @@ -222,7 +174,7 @@ public class ProxyServer implements Runnable { void stop() { Event.stopping("configproxy", "shutdown"); if (rpcServer != null) rpcServer.shutdown(); - if (delayedResponseScheduler != null) delayedResponseScheduler.cancel(true); + if (configClient != null) configClient.cancel(); flush(); fileDistributionAndUrlDownload.close(); } @@ -242,7 +194,11 @@ public class ProxyServer implements Runnable { void updateSourceConnections(List<String> sources) { configSource = new ConfigSourceSet(sources); flush(); - configClient = createRpcClient(); + configClient = createRpcClient(rpcServer, configSource, memoryCache); + } + + DelayedResponses delayedResponses() { + return configClient.delayedResponses(); } } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java index ee843088086..2a33e8c6928 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java @@ -11,18 +11,23 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; import com.yahoo.log.LogLevel; -import com.yahoo.vespa.config.*; +import com.yahoo.vespa.config.ConfigCacheKey; +import com.yahoo.vespa.config.RawConfig; +import com.yahoo.vespa.config.TimingValues; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.concurrent.DelayQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.logging.Logger; +import static java.util.concurrent.TimeUnit.SECONDS; + /** * An Rpc client to a config source * @@ -31,6 +36,8 @@ import java.util.logging.Logger; class RpcConfigSourceClient implements ConfigSourceClient { private final static Logger log = Logger.getLogger(RpcConfigSourceClient.class.getName()); + private static final double timingValuesRatio = 0.8; + private final Supervisor supervisor = new Supervisor(new Transport()); private final RpcServer rpcServer; @@ -39,38 +46,37 @@ class RpcConfigSourceClient implements ConfigSourceClient { private final Object activeSubscribersLock = new Object(); private final MemoryCache memoryCache; private final DelayedResponses delayedResponses; - private final TimingValues timingValues; - + private final static TimingValues timingValues; private final ExecutorService exec; - private final Map<ConfigSourceSet, JRTConfigRequester> requesterPool; - + private final JRTConfigRequester requester; + // Scheduled executor that periodically checks for requests that have timed out and response should be returned to clients + private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1, new DaemonThreadFactory()); + private ScheduledFuture<?> delayedResponseScheduler; + + static { + // Proxy should time out before clients upon subscription. + TimingValues tv = new TimingValues(); + tv.setUnconfiguredDelay((long)(tv.getUnconfiguredDelay()* timingValuesRatio)). + setConfiguredErrorDelay((long)(tv.getConfiguredErrorDelay()* timingValuesRatio)). + setSubscribeTimeout((long)(tv.getSubscribeTimeout()* timingValuesRatio)). + setConfiguredErrorTimeout(-1); // Never cache errors + timingValues = tv; + } - RpcConfigSourceClient(RpcServer rpcServer, - ConfigSourceSet configSourceSet, - MemoryCache memoryCache, - TimingValues timingValues, - DelayedResponses delayedResponses) { + RpcConfigSourceClient(RpcServer rpcServer, ConfigSourceSet configSourceSet, MemoryCache memoryCache) { this.rpcServer = rpcServer; this.configSourceSet = configSourceSet; this.memoryCache = memoryCache; - this.delayedResponses = delayedResponses; - this.timingValues = timingValues; + this.delayedResponses = new DelayedResponses(); checkConfigSources(); exec = Executors.newCachedThreadPool(new DaemonThreadFactory("subscriber-")); - requesterPool = createRequesterPool(configSourceSet, timingValues); - } - - /** - * Creates a requester (connection) pool of one entry, to be used each time this {@link RpcConfigSourceClient} is used - * @param ccs a {@link ConfigSourceSet} - * @param timingValues a {@link TimingValues} - * @return requester map - */ - private Map<ConfigSourceSet, JRTConfigRequester> createRequesterPool(ConfigSourceSet ccs, TimingValues timingValues) { - Map<ConfigSourceSet, JRTConfigRequester> ret = new HashMap<>(); - if (ccs.getSources().isEmpty()) return ret; // unit test, just skip creating any requester - ret.put(ccs, new JRTConfigRequester(new JRTConnectionPool(ccs), timingValues)); - return ret; + requester = JRTConfigRequester.create(configSourceSet, timingValues); + // Wait for 5 seconds initially, then run every second + delayedResponseScheduler = scheduler.scheduleAtFixedRate( + new DelayedResponseHandler(delayedResponses, memoryCache, rpcServer), + 5, + 1, + SECONDS); } /** @@ -152,8 +158,8 @@ class RpcConfigSourceClient implements ConfigSourceClient { log.log(LogLevel.DEBUG, () -> "Already a subscriber running for: " + configCacheKey); } else { log.log(LogLevel.DEBUG, () -> "Could not find good config in cache, creating subscriber for: " + configCacheKey); - UpstreamConfigSubscriber subscriber = new UpstreamConfigSubscriber(input, this, configSourceSet, - timingValues, requesterPool, memoryCache); + UpstreamConfigSubscriber subscriber = + new UpstreamConfigSubscriber(input, this, configSourceSet, timingValues, requester, memoryCache); try { subscriber.subscribe(); activeSubscribers.put(configCacheKey, subscriber); @@ -169,6 +175,8 @@ class RpcConfigSourceClient implements ConfigSourceClient { @Override public void cancel() { shutdownSourceConnections(); + delayedResponseScheduler.cancel(true); + scheduler.shutdown(); } /** @@ -183,25 +191,18 @@ class RpcConfigSourceClient implements ConfigSourceClient { activeSubscribers.clear(); } exec.shutdown(); - for (JRTConfigRequester requester : requesterPool.values()) { - requester.close(); - } + requester.close(); } @Override public String getActiveSourceConnection() { - if (requesterPool.get(configSourceSet) != null) { - return requesterPool.get(configSourceSet).getConnectionPool().getCurrent().getAddress(); - } else { - return ""; - } + return requester.getConnectionPool().getCurrent().getAddress(); } @Override public List<String> getSourceConnections() { ArrayList<String> ret = new ArrayList<>(); - final JRTConfigRequester jrtConfigRequester = requesterPool.get(configSourceSet); - if (jrtConfigRequester != null) { + if (configSourceSet != null) { ret.addAll(configSourceSet.getSources()); } return ret; @@ -244,4 +245,9 @@ class RpcConfigSourceClient implements ConfigSourceClient { log.log(LogLevel.DEBUG, () -> "Finished updating config for " + config.getKey() + "," + config.getGeneration()); } + @Override + public DelayedResponses delayedResponses() { + return delayedResponses; + } + } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java index f8df16cb3d2..d8a8c5ce941 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java @@ -1,16 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.proxy; -import com.yahoo.config.subscription.ConfigSource; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.config.subscription.impl.GenericConfigHandle; import com.yahoo.config.subscription.impl.GenericConfigSubscriber; import com.yahoo.config.subscription.impl.JRTConfigRequester; import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.yolean.Exceptions; import com.yahoo.vespa.config.RawConfig; import com.yahoo.vespa.config.TimingValues; +import com.yahoo.yolean.Exceptions; import java.util.Map; import java.util.logging.Logger; @@ -24,26 +23,26 @@ public class UpstreamConfigSubscriber implements Subscriber { private final RawConfig config; private final ConfigSourceClient configSourceClient; - private final ConfigSource configSourceSet; + private final ConfigSourceSet configSourceSet; private final TimingValues timingValues; - private final Map<ConfigSourceSet, JRTConfigRequester> requesterPool; + private final JRTConfigRequester requester; private final MemoryCache memoryCache; private GenericConfigSubscriber subscriber; private GenericConfigHandle handle; - UpstreamConfigSubscriber(RawConfig config, ConfigSourceClient configSourceClient, ConfigSource configSourceSet, - TimingValues timingValues, Map<ConfigSourceSet, JRTConfigRequester> requesterPool, + UpstreamConfigSubscriber(RawConfig config, ConfigSourceClient configSourceClient, ConfigSourceSet configSourceSet, + TimingValues timingValues, JRTConfigRequester requester, MemoryCache memoryCache) { this.config = config; this.configSourceClient = configSourceClient; this.configSourceSet = configSourceSet; this.timingValues = timingValues; - this.requesterPool = requesterPool; + this.requester = requester; this.memoryCache = memoryCache; } void subscribe() { - subscriber = new GenericConfigSubscriber(requesterPool); + subscriber = new GenericConfigSubscriber(Map.of(configSourceSet, requester)); ConfigKey<?> key = config.getKey(); handle = subscriber.subscribe(new ConfigKey<>(key.getName(), key.getConfigId(), key.getNamespace()), config.getDefContent(), configSourceSet, timingValues); diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java index dc1c995fbb5..29bd38ea891 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java @@ -264,7 +264,7 @@ public class ConfigProxyRpcServerTest { } private static ProxyServer createTestServer(ConfigSourceSet source) { - return new ProxyServer(null, source, ProxyServer.defaultTimingValues(), new MemoryCache(), null); + return new ProxyServer(null, source, new MemoryCache(), null); } private static class TestServer implements AutoCloseable { diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java index 963c922d5b5..06e55eef4fa 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java @@ -16,6 +16,7 @@ import java.util.List; public class MockConfigSourceClient implements ConfigSourceClient{ private final MockConfigSource configSource; private final MemoryCache memoryCache; + private final DelayedResponses delayedResponses = new DelayedResponses(); MockConfigSourceClient(MockConfigSource configSource, MemoryCache memoryCache) { this.configSource = configSource; @@ -53,7 +54,9 @@ public class MockConfigSourceClient implements ConfigSourceClient{ } @Override - public void updateSubscribers(RawConfig config) { + public void updateSubscribers(RawConfig config) { } + + @Override + public DelayedResponses delayedResponses() { return delayedResponses; } - } } diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java index 712567774f1..f52598b3ee5 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java @@ -14,7 +14,11 @@ import org.junit.rules.TemporaryFolder; import java.util.Optional; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author hmusum @@ -23,7 +27,7 @@ public class ProxyServerTest { private final MemoryCache memoryCache = new MemoryCache(); private final MockConfigSource source = new MockConfigSource(); - private MockConfigSourceClient client = new MockConfigSourceClient(source, memoryCache); + private final MockConfigSourceClient client = new MockConfigSourceClient(source, memoryCache); private ProxyServer proxy; static final RawConfig fooConfig = ConfigTester.fooConfig; @@ -54,7 +58,6 @@ public class ProxyServerTest { public void basic() { assertTrue(proxy.getMode().isDefault()); assertThat(proxy.getMemoryCache().size(), is(0)); - assertThat(proxy.getTimingValues(), is(ProxyServer.defaultTimingValues())); ConfigTester tester = new ConfigTester(); final MemoryCache memoryCache = proxy.getMemoryCache(); @@ -222,7 +225,7 @@ public class ProxyServerTest { private static ProxyServer createTestServer(ConfigSourceSet source, ConfigSourceClient configSourceClient, MemoryCache memoryCache) { - return new ProxyServer(null, source, ProxyServer.defaultTimingValues(), memoryCache, configSourceClient); + return new ProxyServer(null, source, memoryCache, configSourceClient); } static RawConfig createConfigWithNextConfigGeneration(RawConfig config, int errorCode) { diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClientTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClientTest.java index 35f1dd8fcd8..8510b23bbd2 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClientTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClientTest.java @@ -18,7 +18,6 @@ import static org.junit.Assert.assertEquals; public class RpcConfigSourceClientTest { private MockRpcServer rpcServer; - private DelayedResponses delayedResponses; private RpcConfigSourceClient rpcConfigSourceClient; @Rule @@ -28,10 +27,7 @@ public class RpcConfigSourceClientTest { @Before public void setup() { rpcServer = new MockRpcServer(); - delayedResponses = new DelayedResponses(); - rpcConfigSourceClient = - new RpcConfigSourceClient(rpcServer, new MockConfigSource(), - new MemoryCache(), ProxyServer.defaultTimingValues(), delayedResponses); + rpcConfigSourceClient = new RpcConfigSourceClient(rpcServer, new MockConfigSource(), new MemoryCache()); } @Test @@ -98,7 +94,7 @@ public class RpcConfigSourceClientTest { } private void simulateClientRequestingConfig(RawConfig config) { - delayedResponses.add(new DelayedResponse(JRTServerConfigRequestV3.createFromRequest(JRTConfigRequestFactory.createFromRaw(config, -10L).getRequest()))); + rpcConfigSourceClient.delayedResponses().add(new DelayedResponse(JRTServerConfigRequestV3.createFromRequest(JRTConfigRequestFactory.createFromRaw(config, -10L).getRequest()))); } private void configUpdatedSendResponse(RawConfig config) { diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java index c799186435c..7472439d6a4 100755 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java @@ -3,7 +3,11 @@ package com.yahoo.config.subscription; import com.yahoo.log.LogLevel; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; import java.util.logging.Logger; diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java index 2873855d1c2..3891d710fa3 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java @@ -319,6 +319,7 @@ public class ConfigSubscriber implements AutoCloseable { @Override public void close() { synchronized (monitor) { + if (state == State.CLOSED) return; state = State.CLOSED; } for (ConfigHandle<? extends ConfigInstance> h : subscriptionHandles) { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java index 6d08976b61c..49c5dcd343c 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java @@ -20,7 +20,6 @@ import com.yahoo.yolean.Exceptions; import java.time.Duration; import java.time.Instant; import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; @@ -37,15 +36,17 @@ public class JRTConfigRequester implements RequestWaiter { private static final Logger log = Logger.getLogger(JRTConfigRequester.class.getName()); public static final ConfigSourceSet defaultSourceSet = ConfigSourceSet.createDefault(); + private static final JRTManagedConnectionPools managedPool = new JRTManagedConnectionPools(); private static final int TRACELEVEL = 6; private final TimingValues timingValues; private int fatalFailures = 0; // independent of transientFailures private int transientFailures = 0; // independent of fatalFailures - private final ScheduledThreadPoolExecutor scheduler = new ScheduledThreadPoolExecutor(1, new JRTSourceThreadFactory()); + private final ScheduledThreadPoolExecutor scheduler; private Instant suspendWarningLogged = Instant.MIN; private Instant noApplicationWarningLogged = Instant.MIN; private static final Duration delayBetweenWarnings = Duration.ofSeconds(60); private final ConnectionPool connectionPool; + private final ConfigSourceSet configSourceSet; static final float randomFraction = 0.2f; /* Time to be added to server timeout to create client timeout. This is the time allowed for the server to respond after serverTimeout has elapsed. */ private static final Double additionalTimeForClientTimeout = 10.0; @@ -56,11 +57,23 @@ public class JRTConfigRequester implements RequestWaiter { * @param connectionPool the connectionPool this requester should use * @param timingValues timeouts and delays used when sending JRT config requests */ - public JRTConfigRequester(ConnectionPool connectionPool, TimingValues timingValues) { + JRTConfigRequester(ConfigSourceSet configSourceSet, ScheduledThreadPoolExecutor scheduler, + ConnectionPool connectionPool, TimingValues timingValues) { + this.configSourceSet = configSourceSet; + this.scheduler = scheduler; this.connectionPool = connectionPool; this.timingValues = timingValues; } + /** Only for testing */ + public JRTConfigRequester(ConnectionPool connectionPool, TimingValues timingValues) { + this(null, new ScheduledThreadPoolExecutor(1), connectionPool, timingValues); + } + + public static JRTConfigRequester create(ConfigSourceSet sourceSet, TimingValues timingValues) { + return managedPool.acquire(sourceSet, timingValues); + } + /** * Requests the config for the {@link com.yahoo.config.ConfigInstance} on the given {@link ConfigSubscription} * @@ -273,18 +286,8 @@ public class JRTConfigRequester implements RequestWaiter { // Fake that we have logged to avoid printing warnings after this suspendWarningLogged = Instant.now(); noApplicationWarningLogged = Instant.now(); - - connectionPool.close(); - scheduler.shutdown(); - } - - private static class JRTSourceThreadFactory implements ThreadFactory { - @Override - public Thread newThread(Runnable runnable) { - Thread t = new Thread(runnable, String.format("jrt-config-requester-%d", System.currentTimeMillis())); - // We want a daemon thread to avoid hanging threads in case something goes wrong in the config system - t.setDaemon(true); - return t; + if (configSourceSet != null) { + managedPool.release(configSourceSet); } } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java index 39e6c69f539..a94a135f9d8 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java @@ -28,7 +28,7 @@ import com.yahoo.vespa.config.protocol.Payload; public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { private JRTConfigRequester requester; - private TimingValues timingValues; + private final TimingValues timingValues; // Last time we got an OK JRT callback private Instant lastOK = Instant.MIN; @@ -156,7 +156,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc private JRTConfigRequester getRequester() { JRTConfigRequester requester = subscriber.requesters().get(sources); if (requester == null) { - requester = new JRTConfigRequester(new JRTConnectionPool(sources), timingValues); + requester = JRTConfigRequester.create(sources, timingValues); subscriber.requesters().put(sources, requester); } return requester; diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTManagedConnectionPools.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTManagedConnectionPools.java new file mode 100644 index 00000000000..32d2d962e4d --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTManagedConnectionPools.java @@ -0,0 +1,66 @@ +package com.yahoo.config.subscription.impl; + +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.vespa.config.JRTConnectionPool; +import com.yahoo.vespa.config.TimingValues; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; + +public class JRTManagedConnectionPools { + private static class JRTSourceThreadFactory implements ThreadFactory { + @Override + public Thread newThread(Runnable runnable) { + Thread t = new Thread(runnable, String.format("jrt-config-requester-%d", System.currentTimeMillis())); + // We want a daemon thread to avoid hanging threads in case something goes wrong in the config system + t.setDaemon(true); + return t; + } + } + private static class CountedPool { + long count; + final JRTConnectionPool pool; + final ScheduledThreadPoolExecutor scheduler = new ScheduledThreadPoolExecutor(1, new JRTSourceThreadFactory()); + CountedPool(JRTConnectionPool requester) { + this.pool = requester; + count = 0; + } + } + + private final Map<ConfigSourceSet, CountedPool> pools = new HashMap<>(); + + public JRTConfigRequester acquire(ConfigSourceSet sourceSet, TimingValues timingValues) { + CountedPool countedPool; + synchronized (pools) { + countedPool = pools.get(sourceSet); + if (countedPool == null) { + countedPool = new CountedPool(new JRTConnectionPool(sourceSet)); + pools.put(sourceSet, countedPool); + } + countedPool.count++; + } + return new JRTConfigRequester(sourceSet, countedPool.scheduler, countedPool.pool, timingValues); + } + + public synchronized void release(ConfigSourceSet sourceSet) { + CountedPool countedPool; + synchronized (pools) { + countedPool = pools.get(sourceSet); + if (countedPool != null) + countedPool.count--; + if (countedPool == null || countedPool.count > 0) return; + pools.remove(sourceSet); + } + + countedPool.pool.close(); + countedPool.scheduler.shutdown(); + try { + countedPool.scheduler.awaitTermination(30, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw new RuntimeException("Failed shutting down scheduler:", e); + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java index a4da996effd..326c1287468 100644 --- a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java +++ b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java @@ -112,19 +112,6 @@ public class JRTConnectionPool implements ConnectionPool { return this; } - public String getAllSourceAddresses() { - StringBuilder sb = new StringBuilder(); - synchronized (connections) { - for (JRTConnection conn : connections.values()) { - sb.append(conn.getAddress()); - sb.append(","); - } - } - // Remove trailing "," - sb.deleteCharAt(sb.length() - 1); - return sb.toString(); - } - public String toString() { StringBuilder sb = new StringBuilder(); synchronized (connections) { diff --git a/config/src/main/java/com/yahoo/vespa/config/TimingValues.java b/config/src/main/java/com/yahoo/vespa/config/TimingValues.java index 780cf657009..5d5967e56c4 100644 --- a/config/src/main/java/com/yahoo/vespa/config/TimingValues.java +++ b/config/src/main/java/com/yahoo/vespa/config/TimingValues.java @@ -12,12 +12,11 @@ public class TimingValues { public static final long defaultNextConfigTimeout = 1000; // See getters below for an explanation of how these values are used and interpreted // All time values in milliseconds. - private long successTimeout = 600000; - private long errorTimeout = 20000; - private long initialTimeout = 15000; + private final long successTimeout; + private final long errorTimeout; + private final long initialTimeout; private long subscribeTimeout = 55000; private long configuredErrorTimeout = -1; // Don't ever timeout (and do not use error response) when we are already configured - private long nextConfigTimeout = defaultNextConfigTimeout; private long fixedDelay = 5000; private long unconfiguredDelay = 1000; @@ -26,6 +25,9 @@ public class TimingValues { private final Random rand; public TimingValues() { + successTimeout = 600000; + errorTimeout = 20000; + initialTimeout = 15000; this.rand = new Random(System.currentTimeMillis()); } @@ -100,20 +102,6 @@ public class TimingValues { } /** - * Returns initial timeout to use as server timeout when a config is requested for the first time. - * - * @return timeout in milliseconds. - */ - public long getInitialTimeout() { - return initialTimeout; - } - - public TimingValues setInitialTimeout(long t) { - initialTimeout = t; - return this; - } - - /** * Returns timeout to use as server timeout when subscribing for the first time. * * @return timeout in milliseconds. @@ -127,38 +115,12 @@ public class TimingValues { return this; } - /** - * Returns the time to retry getting config from the remote sources, until the next error response will - * be set as config. Counted from the last ok request was received. A negative value means that - * we will always retry getting config and never set an error response as config. - * - * @return timeout in milliseconds. - */ - public long getConfiguredErrorTimeout() { - return configuredErrorTimeout; - } - public TimingValues setConfiguredErrorTimeout(long t) { configuredErrorTimeout = t; return this; } /** - * Returns timeout used when calling {@link com.yahoo.config.subscription.ConfigSubscriber#nextConfig()} or - * {@link com.yahoo.config.subscription.ConfigSubscriber#nextGeneration()} - * - * @return timeout in milliseconds. - */ - public long getNextConfigTimeout() { - return nextConfigTimeout; - } - - public TimingValues setNextConfigTimeout(long t) { - nextConfigTimeout = t; - return this; - } - - /** * Returns time to wait until next attempt to get config after a failed request when the client has not * gotten a successful response to a config subscription (i.e, the client has not been configured). * A negative value means that there will never be a next attempt. If a negative value is set, the @@ -201,12 +163,6 @@ public class TimingValues { return maxDelayMultiplier; } - - public TimingValues setSuccessTimeout(long successTimeout) { - this.successTimeout = successTimeout; - return this; - } - /** * Returns fixed delay that is used when retrying getting config no matter if it was a success or an error * and independent of number of retries. @@ -228,10 +184,6 @@ public class TimingValues { return Math.round(val - (val * fraction) + (rand.nextFloat() * 2L * val * fraction)); } - Random getRandom() { - return rand; - } - @Override public String toString() { return "TimingValues [successTimeout=" + successTimeout diff --git a/config/src/test/java/com/yahoo/config/subscription/BasicTest.java b/config/src/test/java/com/yahoo/config/subscription/BasicTest.java index 3b8b7db6487..5b145d40b7f 100644 --- a/config/src/test/java/com/yahoo/config/subscription/BasicTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/BasicTest.java @@ -1,13 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription; -import static org.junit.Assert.*; -import static org.hamcrest.CoreMatchers.is; import com.yahoo.foo.AppConfig; import org.junit.Test; +import static org.junit.Assert.assertEquals; + public class BasicTest { @@ -17,7 +17,8 @@ public class BasicTest { ConfigHandle<AppConfig> h = s.subscribe(AppConfig.class, "raw:times 0"); s.nextConfig(0); AppConfig c = h.getConfig(); - assertThat(c.times(), is(0)); + assertEquals(0, c.times()); + s.close(); } @Test @@ -26,6 +27,7 @@ public class BasicTest { ConfigHandle<AppConfig> h = s.subscribe(AppConfig.class, "raw:times 2"); s.nextGeneration(0); AppConfig c = h.getConfig(); - assertThat(c.times(), is(2)); + assertEquals(2, c.times()); + s.close(); } } diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java index 21cdfbe7d30..db30e7b7389 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java @@ -128,6 +128,7 @@ public class ConfigSetSubscriptionTest { assertEquals(hA0.getConfig().times(), 8800); assertEquals(hA1.getConfig().times(), 890); assertEquals(hS.getConfig().stringVal(), "new StringVal"); + subscriber.close(); } @Test diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java index 933a9fd130a..c8d4c081fc9 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java @@ -60,6 +60,7 @@ public class ConfigSubscriptionTest { assertEquals(c1, c1); assertNotEquals(c1, c2); + sub.close(); } @Test @@ -70,6 +71,7 @@ public class ConfigSubscriptionTest { sub.nextConfig(); assertTrue(handle.getConfig().boolval()); //assertTrue(sub.getSource() instanceof RawSource); + sub.close(); } // Test that subscription is closed and subscriptionHandles is empty if we get an exception diff --git a/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java b/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java index e9dc9cf7b98..9c83f2f3c9a 100644 --- a/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java @@ -49,14 +49,17 @@ public class GenericConfigSubscriberTest { public void testGenericRequesterPooling() { ConfigSourceSet source1 = new ConfigSourceSet("tcp/foo:78"); ConfigSourceSet source2 = new ConfigSourceSet("tcp/bar:79"); - JRTConfigRequester req1 = new JRTConfigRequester(new JRTConnectionPool(source1), JRTConfigRequesterTest.getTestTimingValues()); - JRTConfigRequester req2 = new JRTConfigRequester(new JRTConnectionPool(source2), JRTConfigRequesterTest.getTestTimingValues()); + JRTConfigRequester req1 = JRTConfigRequester.create(source1, JRTConfigRequesterTest.getTestTimingValues()); + JRTConfigRequester req2 = JRTConfigRequester.create(source2, JRTConfigRequesterTest.getTestTimingValues()); Map<ConfigSourceSet, JRTConfigRequester> requesters = new LinkedHashMap<>(); requesters.put(source1, req1); requesters.put(source2, req2); GenericConfigSubscriber sub = new GenericConfigSubscriber(requesters); assertEquals(sub.requesters().get(source1).getConnectionPool().getCurrent().getAddress(), "tcp/foo:78"); assertEquals(sub.requesters().get(source2).getConnectionPool().getCurrent().getAddress(), "tcp/bar:79"); + for (JRTConfigRequester requester : requesters.values()) { + requester.close(); + } } @Test(expected=UnsupportedOperationException.class) diff --git a/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java b/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java index 757dd99f43b..4211345dff7 100644 --- a/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java @@ -1,10 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription.impl; +import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.foo.SimpletypesConfig; import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.jrt.Request; import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConnectionPool; import com.yahoo.vespa.config.ErrorCode; import com.yahoo.vespa.config.ErrorType; import com.yahoo.vespa.config.TimingValues; @@ -17,6 +19,8 @@ import java.util.Random; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -349,4 +353,23 @@ public class JRTConfigRequesterTest { } } + @Test + public void testManagedPool() { + ConfigSourceSet sourceSet = ConfigSourceSet.createDefault(); + TimingValues timingValues = new TimingValues(); + JRTConfigRequester requester1 = JRTConfigRequester.create(sourceSet, timingValues); + JRTConfigRequester requester2 = JRTConfigRequester.create(sourceSet, timingValues); + assertNotSame(requester1, requester2); + assertSame(requester1.getConnectionPool(), requester2.getConnectionPool()); + ConnectionPool firstPool = requester1.getConnectionPool(); + requester1.close(); + requester2.close(); + requester1 = JRTConfigRequester.create(sourceSet, timingValues); + assertNotSame(firstPool, requester1.getConnectionPool()); + requester2 = JRTConfigRequester.create(new ConfigSourceSet("test-managed-pool-2"), timingValues); + assertNotSame(requester1.getConnectionPool(), requester2.getConnectionPool()); + requester1.close(); + requester2.close(); + } + } diff --git a/config/src/tests/trace/trace.cpp b/config/src/tests/trace/trace.cpp index 41c874eb1d4..9a355f39ecc 100644 --- a/config/src/tests/trace/trace.cpp +++ b/config/src/tests/trace/trace.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/common/trace.h> +#include <vespa/vespalib/data/slime/slime.h> using namespace config; diff --git a/config/src/vespa/config/common/configdefinition.cpp b/config/src/vespa/config/common/configdefinition.cpp index 861fc224867..92af068cff5 100644 --- a/config/src/vespa/config/common/configdefinition.cpp +++ b/config/src/vespa/config/common/configdefinition.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "configdefinition.h" #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/data/slime/slime.h> using namespace vespalib; using namespace vespalib::slime; diff --git a/config/src/vespa/config/common/configdefinition.h b/config/src/vespa/config/common/configdefinition.h index 3d86e5d2567..9154f3ad88a 100644 --- a/config/src/vespa/config/common/configdefinition.h +++ b/config/src/vespa/config/common/configdefinition.h @@ -3,8 +3,11 @@ #include <vespa/vespalib/stllike/string.h> #include <vector> -#include <vespa/vespalib/data/slime/slime.h> +namespace vespalib::slime { + class Cursor; + class Inspector; +} namespace config { /** diff --git a/config/src/vespa/config/common/trace.cpp b/config/src/vespa/config/common/trace.cpp index d1bb154eda9..76310d08c7d 100644 --- a/config/src/vespa/config/common/trace.cpp +++ b/config/src/vespa/config/common/trace.cpp @@ -2,6 +2,7 @@ #include "trace.h" #include <vespa/vespalib/trace/slime_trace_serializer.h> #include <vespa/vespalib/trace/slime_trace_deserializer.h> +#include <vespa/vespalib/data/slime/slime.h> using namespace vespalib; using namespace vespalib::slime; diff --git a/config/src/vespa/config/common/trace.h b/config/src/vespa/config/common/trace.h index 772cdb6f31e..c120fe30d12 100644 --- a/config/src/vespa/config/common/trace.h +++ b/config/src/vespa/config/common/trace.h @@ -1,11 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/trace/tracenode.h> #include <vespa/vespalib/stllike/string.h> -#include <memory> +#include <vespa/vespalib/data/memory.h> +namespace vespalib::slime { + class Cursor; + class Inspector; +} namespace config { /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index b2a10f4bb21..4e1006213c6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -62,6 +62,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable private final StateMonitor stateMonitor; private final VipStatus vipStatus; private final ConfigserverConfig configserverConfig; + private final SuperModelManager superModelManager; private final Duration maxDurationOfRedeployment; private final Duration sleepTimeWhenRedeployingFails; private final RedeployingApplicationsFails exitIfRedeployingApplicationsFails; @@ -70,29 +71,32 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable @SuppressWarnings("unused") @Inject public ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, - VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus) { + VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus, + SuperModelManager superModelManager) { this(applicationRepository, server, versionState, stateMonitor, vipStatus, BOOTSTRAP_IN_CONSTRUCTOR, EXIT_JVM, applicationRepository.configserverConfig().hostedVespa() ? VipStatusMode.VIP_STATUS_FILE - : VipStatusMode.VIP_STATUS_PROGRAMMATICALLY); + : VipStatusMode.VIP_STATUS_PROGRAMMATICALLY, + superModelManager); } // For testing only ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus, Mode mode, VipStatusMode vipStatusMode) { - this(applicationRepository, server, versionState, stateMonitor, vipStatus, mode, CONTINUE, vipStatusMode); + this(applicationRepository, server, versionState, stateMonitor, vipStatus, mode, CONTINUE, vipStatusMode, null); } private ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus, Mode mode, RedeployingApplicationsFails exitIfRedeployingApplicationsFails, - VipStatusMode vipStatusMode) { + VipStatusMode vipStatusMode, SuperModelManager superModelManager) { this.applicationRepository = applicationRepository; this.server = server; this.versionState = versionState; this.stateMonitor = stateMonitor; this.vipStatus = vipStatus; this.configserverConfig = applicationRepository.configserverConfig(); + this.superModelManager = superModelManager; this.maxDurationOfRedeployment = Duration.ofSeconds(configserverConfig.maxDurationOfBootstrap()); this.sleepTimeWhenRedeployingFails = Duration.ofSeconds(configserverConfig.sleepTimeWhenRedeployingFails()); this.exitIfRedeployingApplicationsFails = exitIfRedeployingApplicationsFails; @@ -208,6 +212,9 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable private boolean redeployAllApplications() throws InterruptedException { Instant end = Instant.now().plus(maxDurationOfRedeployment); Set<ApplicationId> applicationsNotRedeployed = applicationRepository.listApplications(); + if (superModelManager != null) { + superModelManager.setBootstrapApplicationSet(applicationsNotRedeployed); + } do { applicationsNotRedeployed = redeployApplications(applicationsNotRedeployed); if ( ! applicationsNotRedeployed.isEmpty()) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java index ea835206b7c..ccd92d4a195 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java @@ -20,6 +20,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Set; /** * Provides a SuperModel - a model of all application instances, and makes it stays @@ -38,6 +39,9 @@ public class SuperModelManager implements SuperModelProvider { private final long masterGeneration; // ConfigserverConfig's generation private final GenerationCounter generationCounter; + // The initial set of applications to be deployed on bootstrap. + private Optional<Set<ApplicationId>> bootstrapApplicationSet = Optional.empty(); + @Inject public SuperModelManager(ConfigserverConfig configserverConfig, NodeFlavors nodeFlavors, @@ -75,6 +79,10 @@ public class SuperModelManager implements SuperModelProvider { listeners.add(listener); SuperModel superModel = superModelConfigProvider.getSuperModel(); superModel.getAllApplicationInfos().forEach(application -> listener.applicationActivated(superModel, application)); + + if (superModel.isComplete()) { + listener.notifyOfCompleteness(superModel); + } } } @@ -89,13 +97,19 @@ public class SuperModelManager implements SuperModelProvider { .getForVersionOrLatest(Optional.empty(), Instant.now()) .toApplicationInfo(); - SuperModel newSuperModel = this.superModelConfigProvider - .getSuperModel() - .cloneAndSetApplication(applicationInfo); + SuperModel oldSuperModel = superModelConfigProvider.getSuperModel(); + SuperModel newSuperModel = oldSuperModel + .cloneAndSetApplication(applicationInfo, isComplete(oldSuperModel)); + generationCounter.increment(); makeNewSuperModelConfigProvider(newSuperModel); - listeners.stream().forEach(listener -> - listener.applicationActivated(newSuperModel, applicationInfo)); + listeners.forEach(listener -> listener.applicationActivated(newSuperModel, applicationInfo)); + + if (!oldSuperModel.isComplete() && newSuperModel.isComplete()) { + for (var listener : listeners) { + listener.notifyOfCompleteness(newSuperModel); + } + } } } @@ -106,11 +120,36 @@ public class SuperModelManager implements SuperModelProvider { .cloneAndRemoveApplication(applicationId); generationCounter.increment(); makeNewSuperModelConfigProvider(newSuperModel); - listeners.stream().forEach(listener -> - listener.applicationRemoved(newSuperModel, applicationId)); + listeners.forEach(listener -> listener.applicationRemoved(newSuperModel, applicationId)); } } + public void setBootstrapApplicationSet(Set<ApplicationId> bootstrapApplicationSet) { + synchronized (monitor) { + this.bootstrapApplicationSet = Optional.of(bootstrapApplicationSet); + + SuperModel superModel = superModelConfigProvider.getSuperModel(); + if (!superModel.isComplete() && isComplete(superModel)) { + // We do NOT increment the generation since completeness is not part of the config: + // generationCounter.increment() + + SuperModel newSuperModel = superModel.cloneAsComplete(); + makeNewSuperModelConfigProvider(newSuperModel); + listeners.forEach(listener -> listener.notifyOfCompleteness(newSuperModel)); + } + } + } + + /** Returns freshly calculated value of isComplete. */ + private boolean isComplete(SuperModel currentSuperModel) { + if (currentSuperModel.isComplete()) return true; + if (bootstrapApplicationSet.isEmpty()) return false; + + Set<ApplicationId> currentApplicationIds = superModelConfigProvider.getSuperModel().getApplicationIds(); + if (currentApplicationIds.size() < bootstrapApplicationSet.get().size()) return false; + return currentApplicationIds.containsAll(bootstrapApplicationSet.get()); + } + private void makeNewSuperModelConfigProvider(SuperModel newSuperModel) { generation = masterGeneration + generationCounter.get(); superModelConfigProvider = new SuperModelConfigProvider(newSuperModel, zone, flagSource); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java index b7486dc7951..561422c1cf8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java @@ -55,7 +55,7 @@ public class SuperModelControllerTest { ApplicationId app = ApplicationId.from(TenantName.from("a"), ApplicationName.from("foo"), InstanceName.defaultName()); models.put(app, new ApplicationInfo(app, 4L, new VespaModel(FilesApplicationPackage.fromFile(testApp)))); - SuperModel superModel = new SuperModel(models); + SuperModel superModel = new SuperModel(models, true); handler = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); } @@ -98,7 +98,7 @@ public class SuperModelControllerTest { models.put(advanced, createApplicationInfo(testApp2, advanced, 4L)); models.put(tooAdvanced, createApplicationInfo(testApp3, tooAdvanced, 4L)); - SuperModel superModel = new SuperModel(models); + SuperModel superModel = new SuperModel(models, true); SuperModelController han = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); LbServicesConfig.Builder lb = new LbServicesConfig.Builder(); han.getSuperModel().getConfig(lb); @@ -126,7 +126,7 @@ public class SuperModelControllerTest { models.put(advanced, createApplicationInfo(testApp2, advanced, 4L)); models.put(tooAdvanced, createApplicationInfo(testApp3, tooAdvanced, 4L)); - SuperModel superModel = new SuperModel(models); + SuperModel superModel = new SuperModel(models, true); SuperModelController han = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); LbServicesConfig.Builder lb = new LbServicesConfig.Builder(); han.getSuperModel().getConfig(lb); diff --git a/container-accesslogging/pom.xml b/container-accesslogging/pom.xml index 53c17257992..59dabba9efe 100644 --- a/container-accesslogging/pom.xml +++ b/container-accesslogging/pom.xml @@ -65,6 +65,11 @@ <artifactId>jackson-databind</artifactId> <scope>provided</scope> </dependency> + <dependency> + <groupId>org.assertj</groupId> + <artifactId>assertj-core</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> @@ -73,6 +78,10 @@ <artifactId>bundle-plugin</artifactId> <extensions>true</extensions> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> </plugins> <outputDirectory>${buildOutputDirectory}</outputDirectory> </build> diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java index 82c89276319..a3d34ae6a2c 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java @@ -11,7 +11,6 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; -import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.concurrent.ArrayBlockingQueue; @@ -41,7 +40,7 @@ public class LogFileHandler extends StreamHandler { private String filePattern = "./log.%T"; // default to current directory, ms time stamp private long nextRotationTime = 0; private FileOutputStream currentOutputStream = null; - private String fileName; + private volatile String fileName; private String symlinkName = null; private ArrayBlockingQueue<LogRecord> logQueue = new ArrayBlockingQueue<>(100000); private LogRecord rotateCmd = new LogRecord(Level.SEVERE, "rotateNow"); diff --git a/container-accesslogging/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java b/container-accesslogging/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java index 999f430aa7a..d7361eec488 100644 --- a/container-accesslogging/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java +++ b/container-accesslogging/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java @@ -2,7 +2,9 @@ package com.yahoo.container.logging; import com.yahoo.io.IOUtils; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.FileInputStream; @@ -17,21 +19,23 @@ import java.util.logging.LogRecord; import java.util.logging.SimpleFormatter; import java.util.zip.GZIPInputStream; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.assertj.core.api.Assertions.assertThat; /** * @author Bob Travis + * @author bjorncs */ -// TODO: Make these tests wait until the right things happen rather than waiting for a predetermined time -// These tests take too long, and are not cleaning up properly. See how this should be done in YApacheLogTestCase public class LogFileHandlerTestCase { + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Test - public void testIt() { + public void testIt() throws IOException { + File root = temporaryFolder.newFolder("logfilehandlertest"); + LogFileHandler h = new LogFileHandler(); - h.setFilePattern("./logfilehandlertest.%Y%m%d%H%M%S"); + h.setFilePattern(root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S"); h.setFormatter(new Formatter() { public String format(LogRecord r) { DateFormat df = new SimpleDateFormat("yyyy.MM.dd:HH:mm:ss.SSS"); @@ -43,42 +47,27 @@ public class LogFileHandlerTestCase { long millisPerDay = 60*60*24*1000; long tomorrowDays = (now / millisPerDay) +1; long tomorrowMillis = tomorrowDays * millisPerDay; - assertEquals (tomorrowMillis, h.getNextRotationTime(now)); + assertThat(tomorrowMillis).isEqualTo(h.getNextRotationTime(now)); long[] rTimes = {1000, 2000, 10000}; h.setRotationTimes(rTimes); - assertEquals (tomorrowMillis+1000, h.getNextRotationTime(tomorrowMillis)); - assertEquals (tomorrowMillis+10000, h.getNextRotationTime(tomorrowMillis+3000)); - boolean okToWrite = false; // don't want regular unit tests to create tiles.... - if (okToWrite) { - LogRecord lr = new LogRecord(Level.INFO, "test"); - h.publish(lr); - h.publish(new LogRecord(Level.INFO, "another test")); - h.rotateNow(); - h.publish(lr); - h.flush(); - } - } - - private boolean delete(String fileOrDir) { - File file = new File(fileOrDir); - return file.delete(); - } - - private void deleteOnExit(String fileOrDir) { - new File(fileOrDir).deleteOnExit(); - } - - private static void deleteRecursive(String directory) { - IOUtils.recursiveDeleteDir(new File(directory)); + assertThat(tomorrowMillis+1000).isEqualTo(h.getNextRotationTime(tomorrowMillis)); + assertThat(tomorrowMillis+10000).isEqualTo(h.getNextRotationTime(tomorrowMillis+3000)); + LogRecord lr = new LogRecord(Level.INFO, "test"); + h.publish(lr); + h.publish(new LogRecord(Level.INFO, "another test")); + h.rotateNow(); + h.publish(lr); + h.flush(); + h.shutdown(); } @Test - public void testSimpleLogging() { - String logFilePattern = "./testLogFileG1.txt"; + public void testSimpleLogging() throws IOException { + File logFile = temporaryFolder.newFile("testLogFileG1.txt"); //create logfilehandler LogFileHandler h = new LogFileHandler(); - h.setFilePattern(logFilePattern); + h.setFilePattern(logFile.getAbsolutePath()); h.setFormatter(new SimpleFormatter()); h.setRotationTimes("0 5 ..."); @@ -86,17 +75,16 @@ public class LogFileHandlerTestCase { LogRecord lr = new LogRecord(Level.INFO, "testDeleteFileFirst1"); h.publish(lr); h.flush(); - - new File(logFilePattern).deleteOnExit(); + h.shutdown(); } @Test - public void testDeleteFileDuringLogging() { - String logFilePattern = "./testLogFileG2.txt"; + public void testDeleteFileDuringLogging() throws IOException { + File logFile = temporaryFolder.newFile("testLogFileG2.txt"); //create logfilehandler LogFileHandler h = new LogFileHandler(); - h.setFilePattern(logFilePattern); + h.setFilePattern(logFile.getAbsolutePath()); h.setFormatter(new SimpleFormatter()); h.setRotationTimes("0 5 ..."); @@ -106,20 +94,20 @@ public class LogFileHandlerTestCase { h.flush(); //delete log file - delete(logFilePattern); + logFile.delete(); //write log again lr = new LogRecord(Level.INFO, "testDeleteFileDuringLogging2"); h.publish(lr); h.flush(); - - new File(logFilePattern).deleteOnExit(); + h.shutdown(); } @Test - public void testSymlink() { + public void testSymlink() throws IOException, InterruptedException { + File root = temporaryFolder.newFolder("testlogforsymlinkchecking"); LogFileHandler h = new LogFileHandler(); - h.setFilePattern("./testlogforsymlinkchecking/logfilehandlertest.%Y%m%d%H%M%S%s"); + h.setFilePattern(root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S%s"); h.setFormatter(new Formatter() { public String format(LogRecord r) { DateFormat df = new SimpleDateFormat("yyyy.MM.dd:HH:mm:ss.SSS"); @@ -132,50 +120,43 @@ public class LogFileHandlerTestCase { h.publish(lr); String f1 = h.getFileName(); String f2 = null; - try { - while (f1 == null) { - Thread.sleep(1); - f1 = h.getFileName(); - } - h.rotateNow(); + while (f1 == null) { + Thread.sleep(1); + f1 = h.getFileName(); + } + h.rotateNow(); + Thread.sleep(1); + f2 = h.getFileName(); + while (f1.equals(f2)) { Thread.sleep(1); f2 = h.getFileName(); - while (f1.equals(f2)) { - Thread.sleep(1); - f2 = h.getFileName(); - } - lr = new LogRecord(Level.INFO, "string which is way longer than the word test"); - h.publish(lr); - Thread.sleep(1000); - File f = new File(f1); - long first = f.length(); - f = new File(f2); - long second = f.length(); - final long secondLength = 72; - for (int n = 0; n < 20 && second != secondLength; ++n) { - Thread.sleep(1000); - second = f.length(); - } - f = new File("./testlogforsymlinkchecking", "symlink"); - long link = f.length(); - assertEquals(secondLength, link); - assertEquals(31, first); - assertEquals(secondLength, second); - } catch (InterruptedException e) { - // just let the test pass } - deleteOnExit("./testlogforsymlinkchecking"); - deleteOnExit("./testlogforsymlinkchecking/symlink"); - deleteOnExit(f1); - if (f2 != null) - deleteOnExit(f2); + lr = new LogRecord(Level.INFO, "string which is way longer than the word test"); + h.publish(lr); + h.waitDrained(); + File f = new File(f1); + long first = f.length(); + f = new File(f2); + long second = f.length(); + final long secondLength = 72; + for (int n = 0; n < 20 && second != secondLength; ++n) { + Thread.sleep(1); + second = f.length(); + } + f = new File(root, "symlink"); + long link = f.length(); + assertThat(secondLength).isEqualTo(link); + assertThat(31).isEqualTo(first); + assertThat(secondLength).isEqualTo(second); + h.shutdown(); } @Test public void testcompression() throws InterruptedException, IOException { - IOUtils.recursiveDeleteDir(new File("./testcompression")); + File root = temporaryFolder.newFolder("testcompression"); + LogFileHandler h = new LogFileHandler(true); - h.setFilePattern("./testcompression/logfilehandlertest.%Y%m%d%H%M%S%s"); + h.setFilePattern(root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S%s"); h.setFormatter(new Formatter() { public String format(LogRecord r) { DateFormat df = new SimpleDateFormat("yyyy.MM.dd:HH:mm:ss.SSS"); @@ -183,28 +164,28 @@ public class LogFileHandlerTestCase { return ("["+timeStamp+"]" + " " + formatMessage(r) + "\n"); } } ); - for (int i=0; i < 10000; i++) { + int logEntries = 10000; + for (int i = 0; i < logEntries; i++) { LogRecord lr = new LogRecord(Level.INFO, "test"); h.publish(lr); } h.waitDrained(); String f1 = h.getFileName(); - assertTrue(f1.startsWith("./testcompression/logfilehandlertest.")); + assertThat(f1).startsWith(root.getAbsolutePath() + "/logfilehandlertest."); File uncompressed = new File(f1); File compressed = new File(f1 + ".gz"); - assertTrue(uncompressed.exists()); - assertFalse(compressed.exists()); + assertThat(uncompressed).exists(); + assertThat(compressed).doesNotExist(); String content = IOUtils.readFile(uncompressed); - assertEquals(310000, content.length()); + assertThat(content).hasLineCount(logEntries); h.rotateNow(); while (uncompressed.exists()) { - Thread.sleep(10); + Thread.sleep(1); } - assertTrue(compressed.exists()); + assertThat(compressed).exists(); String unzipped = IOUtils.readAll(new InputStreamReader(new GZIPInputStream(new FileInputStream(compressed)))); - assertEquals(content, unzipped); - - IOUtils.recursiveDeleteDir(new File("./testcompression")); + assertThat(content).isEqualTo(unzipped); + h.shutdown(); } } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java index bfcecd61fa4..6c8cee8433c 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java @@ -17,6 +17,7 @@ import com.yahoo.jdisc.handler.ResponseDispatch; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.log.LogLevel; +import java.net.URI; import java.time.Duration; import java.util.HashMap; import java.util.Map; @@ -91,6 +92,11 @@ public abstract class ThreadedRequestHandler extends AbstractRequestHandler { if (endpoint != null) { dimensions.put("endpoint", endpoint); } + URI uri = request.getUri(); + dimensions.put("scheme", uri.getScheme()); + dimensions.put("port", Integer.toString(uri.getPort())); + String handlerClassName = getClass().getName(); + dimensions.put("handler-name", handlerClassName); return this.metric.createContext(dimensions); } diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh index 382843b5688..af429d56a75 100755 --- a/container-disc/src/main/sh/vespa-start-container-daemon.sh +++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh @@ -64,6 +64,7 @@ configure_memory() { consider_fallback jvm_heapsize 1536 consider_fallback jvm_stacksize 512 consider_fallback jvm_baseMaxDirectMemorySize 75 + consider_fallback jvm_compressedClassSpaceSize 32 consider_fallback jvm_directMemorySizeCache 0 # Update jvm_heapsize only if percentage is explicitly set (default is 0). @@ -80,16 +81,20 @@ configure_memory() { fi # Safety measure against bad min vs max heapsize. - if ((jvm_minHeapsize > jvm_heapsize)); then + if ((jvm_minHeapsize > jvm_heapsize)); then jvm_minHeapsize=${jvm_heapsize} echo "Misconfigured heap size, jvm_minHeapsize(${jvm_minHeapsize} is larger than jvm_heapsize(${jvm_heapsize}). It has been capped." - fi + fi maxDirectMemorySize=$(( jvm_baseMaxDirectMemorySize + jvm_heapsize / 8 + jvm_directMemorySizeCache )) memory_options="-Xms${jvm_minHeapsize}m -Xmx${jvm_heapsize}m" memory_options="${memory_options} -XX:ThreadStackSize=${jvm_stacksize}" - memory_options="${memory_options} -XX:MaxDirectMemorySize=${maxDirectMemorySize}m" + memory_options="${memory_options} -XX:MaxDirectMemorySize=${maxDirectMemorySize}m" + + if ((jvm_compressedClassSpaceSize != 0)); then + memory_options="${memory_options} -XX:CompressedClassSpaceSize=${jvm_compressedClassSpaceSize}m" + fi if [ "${VESPA_USE_HUGEPAGES}" ]; then memory_options="${memory_options} -XX:+UseLargePages" diff --git a/container-search/src/main/resources/configdefinitions/qr-start.def b/container-search/src/main/resources/configdefinitions/qr-start.def index 031877ada81..95e9d4575dd 100644 --- a/container-search/src/main/resources/configdefinitions/qr-start.def +++ b/container-search/src/main/resources/configdefinitions/qr-start.def @@ -20,6 +20,9 @@ jvm.minHeapsize int default=1536 restart ## Stack size (in kilobytes) jvm.stacksize int default=512 restart +## CompressedOOps size in megabytes +jvm.compressedClassSpaceSize int default=32 restart + ## Base value of maximum direct memory size (in megabytes) jvm.baseMaxDirectMemorySize int default=75 restart diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java index 0aa0df8ae2b..171c5caa756 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java @@ -18,25 +18,23 @@ public class EndpointCertificateMetadata { private final int version; private final Optional<String> request_id; private final Optional<List<String>> requestedDnsSans; + private final Optional<String> issuer; public EndpointCertificateMetadata(String keyName, String certName, int version) { - this.keyName = keyName; - this.certName = certName; - this.version = version; - this.request_id = Optional.empty(); - this.requestedDnsSans = Optional.empty(); + this(keyName, certName, version, Optional.empty(), Optional.empty(), Optional.empty()); + } + + public EndpointCertificateMetadata(String keyName, String certName, int version, String request_id, List<String> requestedDnsSans) { + this(keyName, certName, version, Optional.of(request_id), Optional.of(requestedDnsSans), Optional.empty()); } - public EndpointCertificateMetadata(String keyName, String certName, int version, Optional<String> request_id, Optional<List<String>> requestedDnsSans) { + public EndpointCertificateMetadata(String keyName, String certName, int version, Optional<String> request_id, Optional<List<String>> requestedDnsSans, Optional<String> issuer) { this.keyName = keyName; this.certName = certName; this.version = version; this.request_id = request_id; this.requestedDnsSans = requestedDnsSans; - } - - public EndpointCertificateMetadata(String keyName, String certName, int version, String request_id, List<String> requestedDnsSans) { - this(keyName, certName, version, Optional.of(request_id), Optional.of(requestedDnsSans)); + this.issuer = issuer; } public String keyName() { @@ -59,6 +57,10 @@ public class EndpointCertificateMetadata { return requestedDnsSans; } + public Optional<String> issuer() { + return issuer; + } + @Override public String toString() { return "EndpointCertificateMetadata{" + @@ -67,6 +69,7 @@ public class EndpointCertificateMetadata { ", version=" + version + ", request_id=" + request_id + ", requestedDnsSans=" + requestedDnsSans + + ", issuer=" + issuer + '}'; } @@ -79,11 +82,12 @@ public class EndpointCertificateMetadata { keyName.equals(that.keyName) && certName.equals(that.certName) && request_id.equals(that.request_id) && - requestedDnsSans.equals(that.requestedDnsSans); + requestedDnsSans.equals(that.requestedDnsSans) && + issuer.equals(that.issuer); } @Override public int hashCode() { - return Objects.hash(keyName, certName, version, request_id, requestedDnsSans); + return Objects.hash(keyName, certName, version, request_id, requestedDnsSans, issuer); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java index 8e81400f3c8..c38ea158507 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java @@ -7,6 +7,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; /** @@ -21,7 +22,7 @@ public class EndpointCertificateMock implements EndpointCertificateProvider { } @Override - public EndpointCertificateMetadata requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames) { + public EndpointCertificateMetadata requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames, Optional<EndpointCertificateMetadata> currentMetadata) { this.dnsNames.put(applicationId, dnsNames); String endpointCertificatePrefix = String.format("vespa.tls.%s.%s@%s", applicationId.tenant(), applicationId.application(), diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java index 97d2bdb3343..9c5c25c1c71 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.certificates; import com.yahoo.config.provision.ApplicationId; import java.util.List; +import java.util.Optional; /** * Generates an endpoint certificate for an application instance. @@ -12,7 +13,7 @@ import java.util.List; */ public interface EndpointCertificateProvider { - EndpointCertificateMetadata requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames); + EndpointCertificateMetadata requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames, Optional<EndpointCertificateMetadata> currentMetadata); List<EndpointCertificateMetadata> listCertificates(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 67a6faac606..4c91238453a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -179,8 +179,7 @@ enum PathGroup { "/application/v4/tenant/"), /** Paths which contain (not very strictly) classified information about, e.g., customers. */ - classifiedInfo("/cost/v1/{*}", - "/", + classifiedInfo("/", "/d/{*}"), /** Same as classifiedInfo, but with optional /api prefix */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 17c9e852bd9..e3d41873a8e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -49,7 +49,7 @@ import com.yahoo.vespa.hosted.controller.concurrent.Once; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Versions; -import com.yahoo.vespa.hosted.controller.endpointcertificates.EndpointCertificateManager; +import com.yahoo.vespa.hosted.controller.certificate.EndpointCertificateManager; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.security.Credentials; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 74304f2e49d..cf272d94dcd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -237,7 +237,8 @@ public class RoutingController { } } - private boolean supportsRoutingMethod(RoutingMethod routingMethod, ZoneId zone) { + /** Returns whether given routingMethod is supported by zone */ + public boolean supportsRoutingMethod(RoutingMethod routingMethod, ZoneId zone) { return controller.zoneRegistry().zones().routingMethod(routingMethod).ids().contains(zone); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index c39255fd7a8..5fa463233fd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -1,15 +1,13 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; -import com.google.common.hash.Hashing; -import com.google.common.io.BaseEncoding; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import java.net.URI; -import java.nio.charset.Charset; import java.util.Objects; /** @@ -20,28 +18,29 @@ import java.util.Objects; */ public class Endpoint { - public static final String YAHOO_DNS_SUFFIX = ".vespa.yahooapis.com"; - public static final String OATH_DNS_SUFFIX = ".vespa.oath.cloud"; - public static final String PUBLIC_DNS_SUFFIX = ".public.vespa.oath.cloud"; - public static final String PUBLIC_CD_DNS_SUFFIX = ".public-cd.vespa.oath.cloud"; + private static final String YAHOO_DNS_SUFFIX = ".vespa.yahooapis.com"; + private static final String OATH_DNS_SUFFIX = ".vespa.oath.cloud"; + private static final String PUBLIC_DNS_SUFFIX = ".public.vespa.oath.cloud"; + private static final String PUBLIC_CD_DNS_SUFFIX = ".public-cd.vespa.oath.cloud"; private final URI url; private final Scope scope; private final boolean legacy; - private final boolean directRouting; + private final RoutingMethod routingMethod; private final boolean tls; private final boolean wildcard; private Endpoint(String name, ApplicationId application, ZoneId zone, SystemName system, Port port, boolean legacy, - boolean directRouting, boolean wildcard) { + RoutingMethod routingMethod, boolean wildcard) { Objects.requireNonNull(name, "name must be non-null"); Objects.requireNonNull(application, "application must be non-null"); Objects.requireNonNull(system, "system must be non-null"); Objects.requireNonNull(port, "port must be non-null"); - this.url = createUrl(name, application, zone, system, port, legacy, directRouting); + Objects.requireNonNull(routingMethod, "routingMethod must be non-null"); + this.url = createUrl(name, application, zone, system, port, legacy, routingMethod); this.scope = zone == null ? Scope.global : Scope.zone; this.legacy = legacy; - this.directRouting = directRouting; + this.routingMethod = routingMethod; this.tls = port.tls; this.wildcard = wildcard; } @@ -67,12 +66,9 @@ public class Endpoint { return legacy; } - /** - * Returns whether this endpoint supports direct routing. Direct routing means that this endpoint is served by an - * exclusive load balancer instead of a shared routing layer. - */ - public boolean directRouting() { - return directRouting; + /** Returns the routing used for this */ + public RoutingMethod routingMethod() { + return routingMethod; } /** Returns whether this endpoint supports TLS connections */ @@ -100,13 +96,18 @@ public class Endpoint { @Override public String toString() { - return String.format("endpoint %s [scope=%s, legacy=%s, directRouting=%s]", url, scope, legacy, directRouting); + return String.format("endpoint %s [scope=%s, legacy=%s, routingMethod=%s]", url, scope, legacy, routingMethod); + } + + /** Returns the DNS suffix used for endpoints in given system */ + public static String dnsSuffix(SystemName system) { + return dnsSuffix(system, false); } private static URI createUrl(String name, ApplicationId application, ZoneId zone, SystemName system, - Port port, boolean legacy, boolean directRouting) { + Port port, boolean legacy, RoutingMethod routingMethod) { String scheme = port.tls ? "https" : "http"; - String separator = separator(system, directRouting, port.tls); + String separator = separator(system, routingMethod, port.tls); String portPart = port.isDefault() ? "" : ":" + port.port; return URI.create(scheme + "://" + sanitize(namePart(name, separator)) + @@ -126,9 +127,9 @@ public class Endpoint { return part.replace('_', '-'); } - private static String separator(SystemName system, boolean directRouting, boolean tls) { + private static String separator(SystemName system, RoutingMethod routingMethod, boolean tls) { if (!tls) return "."; - if (directRouting) return "."; + if (routingMethod.isDirect()) return "."; if (system.isPublic()) return "."; return "--"; } @@ -214,13 +215,6 @@ public class Endpoint { } - /** Create a DNS name based on a hash of the ApplicationId. This should always be less than 64 characters long. */ - public static String createHashedCn(ApplicationId application, SystemName system) { - var hashCode = Hashing.sha1().hashString(application.serializedForm(), Charset.defaultCharset()); - var base32encoded = BaseEncoding.base32().omitPadding().lowerCase().encode(hashCode.asBytes()); - return 'v' + base32encoded + dnsSuffix(system, false); - } - /** Build an endpoint for given application */ public static EndpointBuilder of(ApplicationId application) { return new EndpointBuilder(application); @@ -234,8 +228,8 @@ public class Endpoint { private ClusterSpec.Id cluster; private EndpointId endpointId; private Port port; + private RoutingMethod routingMethod = RoutingMethod.shared; private boolean legacy = false; - private boolean directRouting = false; private boolean wildcard = false; private EndpointBuilder(ApplicationId application) { @@ -292,9 +286,9 @@ public class Endpoint { return this; } - /** Enables direct routing support for this */ - public EndpointBuilder directRouting() { - this.directRouting = true; + /** Sets the routing method for this */ + public EndpointBuilder routingMethod(RoutingMethod method) { + this.routingMethod = method; return this; } @@ -310,13 +304,13 @@ public class Endpoint { } else { throw new IllegalArgumentException("Must set either cluster, rotation or wildcard target"); } - if (system.isPublic() && !directRouting) { - throw new IllegalArgumentException("Public system only supports direct routing endpoints"); + if (system.isPublic() && routingMethod != RoutingMethod.exclusive) { + throw new IllegalArgumentException("Public system only supports routing method " + RoutingMethod.exclusive); } - if (directRouting && !port.isDefault()) { - throw new IllegalArgumentException("Direct routing endpoints only support default port"); + if (routingMethod.isDirect() && !port.isDefault()) { + throw new IllegalArgumentException("Routing method " + routingMethod + " can only use default port"); } - return new Endpoint(name, application, zone, system, port, legacy, directRouting, wildcard); + return new Endpoint(name, application, zone, system, port, legacy, routingMethod, wildcard); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java index d915da21603..e22d381e615 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java @@ -1,8 +1,12 @@ -package com.yahoo.vespa.hosted.controller.endpointcertificates; +package com.yahoo.vespa.hosted.controller.certificate; import com.google.common.collect.Sets; +import com.google.common.hash.Hashing; +import com.google.common.io.BaseEncoding; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; @@ -23,6 +27,7 @@ import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import java.nio.charset.Charset; import java.security.cert.X509Certificate; import java.time.Clock; import java.time.Instant; @@ -142,7 +147,7 @@ public class EndpointCertificateManager { if (storedMetaData.requestedDnsSans().isPresent() && storedMetaData.request_id().isPresent()) return; - var hashedCn = Endpoint.createHashedCn(applicationId, zoneRegistry.system()); // use as join key + var hashedCn = commonNameHashOf(applicationId, zoneRegistry.system()); // use as join key EndpointCertificateMetadata providerMetadata = sanToEndpointCertificate.get(hashedCn); if(providerMetadata == null) { @@ -156,7 +161,8 @@ public class EndpointCertificateManager { storedMetaData.certName(), storedMetaData.version(), providerMetadata.request_id(), - providerMetadata.requestedDnsSans()); + providerMetadata.requestedDnsSans(), + Optional.empty()); if (mode == BackfillMode.DRYRUN) { log.log(LogLevel.INFO, "Would update stored metadata " + storedMetaData + " with data from provider: " + backfilledMetadata); @@ -176,7 +182,7 @@ public class EndpointCertificateManager { private EndpointCertificateMetadata provisionEndpointCertificate(Instance instance) { List<ZoneId> zones = zoneRegistry.zones().controllerUpgraded().zones().stream().map(ZoneApi::getId).collect(Collectors.toUnmodifiableList()); EndpointCertificateMetadata provisionedCertificateMetadata = endpointCertificateProvider - .requestCaSignedCertificate(instance.id(), dnsNamesOf(instance.id(), zones)); + .requestCaSignedCertificate(instance.id(), dnsNamesOf(instance.id(), zones), Optional.empty()); curator.writeEndpointCertificateMetadata(instance.id(), provisionedCertificateMetadata); return provisionedCertificateMetadata; } @@ -232,7 +238,7 @@ public class EndpointCertificateManager { // We add first an endpoint name based on a hash of the applicationId, // as the certificate provider requires the first CN to be < 64 characters long. - endpointDnsNames.add(Endpoint.createHashedCn(applicationId, zoneRegistry.system())); + endpointDnsNames.add(commonNameHashOf(applicationId, zoneRegistry.system())); var globalDefaultEndpoint = Endpoint.of(applicationId).named(EndpointId.defaultId()); var rotationEndpoints = Endpoint.of(applicationId).wildcard(); @@ -243,7 +249,7 @@ public class EndpointCertificateManager { )); Stream.concat(Stream.of(globalDefaultEndpoint, rotationEndpoints), zoneLocalEndpoints) - .map(Endpoint.EndpointBuilder::directRouting) + .map(endpoint -> endpoint.routingMethod(RoutingMethod.exclusive)) .map(endpoint -> endpoint.on(Endpoint.Port.tls())) .map(endpointBuilder -> endpointBuilder.in(zoneRegistry.system())) .map(Endpoint::dnsName).forEach(endpointDnsNames::add); @@ -251,4 +257,11 @@ public class EndpointCertificateManager { return Collections.unmodifiableList(endpointDnsNames); } + /** Create a common name based on a hash of the ApplicationId. This should always be less than 64 characters long. */ + private static String commonNameHashOf(ApplicationId application, SystemName system) { + var hashCode = Hashing.sha1().hashString(application.serializedForm(), Charset.defaultCharset()); + var base32encoded = BaseEncoding.base32().omitPadding().lowerCase().encode(hashCode.asBytes()); + return 'v' + base32encoded + Endpoint.dnsSuffix(system); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 9d666c6f7b5..4d09765f78b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -28,7 +28,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; @@ -38,6 +37,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentFailureMails; +import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; @@ -76,7 +76,6 @@ import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Nod import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.reserved; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.endpointCertificateTimeout; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.outOfCapacity; @@ -672,22 +671,36 @@ public class InternalStepRunner implements StepRunner { return; try { - if (run.status() == outOfCapacity && run.id().type().isProduction()) - controller.serviceRegistry().mailer().send(mails.outOfCapacity(run.id(), recipients)); - if (run.status() == deploymentFailed) - controller.serviceRegistry().mailer().send(mails.deploymentFailure(run.id(), recipients)); - if (run.status() == installationFailed) - controller.serviceRegistry().mailer().send(mails.installationFailure(run.id(), recipients)); - if (run.status() == testFailure) - controller.serviceRegistry().mailer().send(mails.testFailure(run.id(), recipients)); - if (run.status() == error) - controller.serviceRegistry().mailer().send(mails.systemError(run.id(), recipients)); + mailOf(run, recipients).ifPresent(controller.serviceRegistry().mailer()::send); } catch (RuntimeException e) { logger.log(INFO, "Exception trying to send mail for " + run.id(), e); } } + private Optional<Mail> mailOf(Run run, List<String> recipients) { + switch (run.status()) { + case running: + case aborted: + case success: + return Optional.empty(); + case outOfCapacity: + return run.id().type().isProduction() ? Optional.of(mails.outOfCapacity(run.id(), recipients)) : Optional.empty(); + case deploymentFailed: + return Optional.of(mails.deploymentFailure(run.id(), recipients)); + case installationFailed: + return Optional.of(mails.installationFailure(run.id(), recipients)); + case testFailure: + return Optional.of(mails.testFailure(run.id(), recipients)); + case error: + case endpointCertificateTimeout: + return Optional.of(mails.systemError(run.id(), recipients)); + default: + logger.log(WARNING, "Don't know what mail to send for run status '" + run.status() + "'"); + return Optional.of(mails.systemError(run.id(), recipients)); + } + } + /** Returns the deployment of the real application in the zone of the given job, if it exists. */ private Optional<Deployment> deployment(ApplicationId id, JobType type) { return Optional.ofNullable(application(id).deployments().get(type.zone(controller.system()))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java index ff88805f957..5b792f384e5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java @@ -1,12 +1,11 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer; -import com.yahoo.vespa.hosted.controller.restapi.cost.CostCalculator; +import com.yahoo.vespa.hosted.controller.metric.CostCalculator; import java.time.Clock; import java.time.Duration; @@ -34,7 +33,8 @@ public class CostReportMaintainer extends Maintainer { @Override protected void maintain() { - consumer.consume(CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller(), clock, consumer.fixedAllocations(), CloudName.from("yahoo"))); + var csv = CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller(), clock, consumer.fixedAllocations()); + consumer.consume(csv); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java index ec01b3817a7..4cc4ee0386c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java @@ -1,5 +1,5 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.cost; +package com.yahoo.vespa.hosted.controller.metric; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; @@ -28,12 +28,12 @@ import static com.yahoo.yolean.Exceptions.uncheck; public class CostCalculator { private static final double SELF_HOSTED_DISCOUNT = .5; + private static final CloudName cloudName = CloudName.from("yahoo"); public static String resourceShareByPropertyToCsv(NodeRepository nodeRepository, Controller controller, Clock clock, - Map<Property, ResourceAllocation> fixedAllocations, - CloudName cloudName) { + Map<Property, ResourceAllocation> fixedAllocations) { var date = DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("UTC")).format(clock.instant()); @@ -61,14 +61,12 @@ public class CostCalculator { } // Add fixed allocations from config - if (cloudName.equals(CloudName.from("yahoo"))) { - for (var kv : fixedAllocations.entrySet()) { - var property = kv.getKey(); - var allocation = allocationByProperty.getOrDefault(property, ResourceAllocation.ZERO); - var discountedFixedAllocation = kv.getValue().multiply(SELF_HOSTED_DISCOUNT); - allocationByProperty.put(property, allocation.plus(discountedFixedAllocation)); - totalAllocation = totalAllocation.plus(discountedFixedAllocation); - } + for (var kv : fixedAllocations.entrySet()) { + var property = kv.getKey(); + var allocation = allocationByProperty.getOrDefault(property, ResourceAllocation.ZERO); + var discountedFixedAllocation = kv.getValue().multiply(SELF_HOSTED_DISCOUNT); + allocationByProperty.put(property, allocation.plus(discountedFixedAllocation)); + totalAllocation = totalAllocation.plus(discountedFixedAllocation); } return toCsv(allocationByProperty, date, totalAllocation); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index ad2835e301f..eb86b1028e2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -42,7 +42,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.NavigableMap; @@ -521,8 +520,7 @@ public class CuratorDb { } public Optional<EndpointCertificateMetadata> readEndpointCertificateMetadata(ApplicationId applicationId) { - Optional<String> zkData = curator.getData(endpointCertificatePath(applicationId)).map(String::new); - return zkData.map(EndpointCertificateMetadataSerializer::fromJsonOrTlsSecretsKeysString); + return curator.getData(endpointCertificatePath(applicationId)).map(String::new).map(EndpointCertificateMetadataSerializer::fromJsonString); } public Map<ApplicationId, EndpointCertificateMetadata> readAllEndpointCertificateMetadata() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java index 653f224a02b..501d3a06d42 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java @@ -4,6 +4,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; +import com.yahoo.slime.Type; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import java.util.List; @@ -33,6 +34,7 @@ public class EndpointCertificateMetadataSerializer { private final static String versionField = "version"; private final static String requestIdField = "requestId"; private final static String requestedDnsSansField = "requestedDnsSans"; + private final static String issuerField = "issuer"; public static Slime toSlime(EndpointCertificateMetadata metadata) { Slime slime = new Slime(); @@ -51,46 +53,31 @@ public class EndpointCertificateMetadataSerializer { } public static EndpointCertificateMetadata fromSlime(Inspector inspector) { - switch (inspector.type()) { - case STRING: // TODO: Remove once all are transmitted and stored as JSON - return new EndpointCertificateMetadata( - inspector.asString() + "-key", - inspector.asString() + "-cert", - 0 - ); - case OBJECT: { - Optional<String> request_id = inspector.field(requestIdField).valid() ? - Optional.of(inspector.field(requestIdField).asString()) : - Optional.empty(); + if (inspector.type() != Type.OBJECT) + throw new IllegalArgumentException("Unknown format encountered for endpoint certificate metadata!"); + Optional<String> request_id = inspector.field(requestIdField).valid() ? + Optional.of(inspector.field(requestIdField).asString()) : + Optional.empty(); - Optional<List<String>> requestedDnsSans = inspector.field(requestedDnsSansField).valid() ? - Optional.of(IntStream.range(0, inspector.field(requestedDnsSansField).entries()) - .mapToObj(i -> inspector.field(requestedDnsSansField).entry(i).asString()).collect(Collectors.toList())) : - Optional.empty(); + Optional<List<String>> requestedDnsSans = inspector.field(requestedDnsSansField).valid() ? + Optional.of(IntStream.range(0, inspector.field(requestedDnsSansField).entries()) + .mapToObj(i -> inspector.field(requestedDnsSansField).entry(i).asString()).collect(Collectors.toList())) : + Optional.empty(); - return new EndpointCertificateMetadata( - inspector.field(keyNameField).asString(), - inspector.field(certNameField).asString(), - Math.toIntExact(inspector.field(versionField).asLong()), - request_id, - requestedDnsSans - ); - } + Optional<String> issuer = inspector.field(issuerField).valid() ? + Optional.of(inspector.field(issuerField).asString()) : + Optional.empty(); - default: - throw new IllegalArgumentException("Unknown format encountered for endpoint certificate metadata!"); - } + return new EndpointCertificateMetadata( + inspector.field(keyNameField).asString(), + inspector.field(certNameField).asString(), + Math.toIntExact(inspector.field(versionField).asLong()), + request_id, + requestedDnsSans, + issuer); } - public static EndpointCertificateMetadata fromTlsSecretsKeysString(String tlsSecretsKeys) { - return fromSlime(new Slime().setString(tlsSecretsKeys)); - } - - public static EndpointCertificateMetadata fromJsonOrTlsSecretsKeysString(String zkdata) { - if (zkdata.strip().startsWith("{")) { - return fromSlime(SlimeUtils.jsonToSlime(zkdata).get()); - } else { - return fromTlsSecretsKeysString(zkdata); - } + public static EndpointCertificateMetadata fromJsonString(String zkdata) { + return fromSlime(SlimeUtils.jsonToSlime(zkdata).get()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 073306719f3..94d7b120406 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -1061,6 +1061,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // Add zone endpoints defined by routing policies var endpointArray = response.setArray("endpoints"); for (var policy : controller.routingController().policies().get(deploymentId).values()) { + // TODO(mpolden): Always add endpoints from all policies, independent of routing method. This allows removal + // of RoutingGenerator and eliminates the external call to the routing layer below. + if (!controller.routingController().supportsRoutingMethod(RoutingMethod.exclusive, deployment.zone())) continue; if (!policy.status().isActive()) continue; { var endpointObject = endpointArray.addObject(); @@ -1157,6 +1160,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { deployment.activity().lastWritesPerSecond().ifPresent(value -> activity.setDouble("lastWritesPerSecond", value)); // Cost + // TODO(mpolden): Unused, remove this field and related code. DeploymentCost appCost = new DeploymentCost(Map.of()); Cursor costObject = response.setObject("cost"); toSlime(appCost, costObject); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 32c2d6ec3d1..0d1dca391bd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -554,15 +554,16 @@ class JobControllerApiHandlerHelper { private static String nameOf(RunStatus status) { switch (status) { - case running: return "running"; - case aborted: return "aborted"; - case error: return "error"; - case testFailure: return "testFailure"; - case outOfCapacity: return "outOfCapacity"; - case installationFailed: return "installationFailed"; - case deploymentFailed: return "deploymentFailed"; - case success: return "success"; - default: throw new IllegalArgumentException("Unexpected status '" + status + "'"); + case running: return "running"; + case aborted: return "aborted"; + case error: return "error"; + case testFailure: return "testFailure"; + case endpointCertificateTimeout: return "endpointCertificateTimeout"; + case outOfCapacity: return "outOfCapacity"; + case installationFailed: return "installationFailed"; + case deploymentFailed: return "deploymentFailed"; + case success: return "success"; + default: throw new IllegalArgumentException("Unexpected status '" + status + "'"); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java deleted file mode 100644 index 8bf3f4a6fd0..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.cost; - -import com.yahoo.config.provision.CloudName; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; -import com.yahoo.restapi.Path; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; -import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.StringResponse; - -import java.time.Clock; -import java.util.Optional; - -import static com.yahoo.jdisc.http.HttpRequest.Method.GET; - -/** - * @author ldalves - */ -public class CostApiHandler extends LoggingRequestHandler { - - private final Controller controller; - private final NodeRepository nodeRepository; - private final CostReportConsumer costReportConsumer; - - public CostApiHandler(Context ctx, Controller controller) { - super(ctx); - this.controller = controller; - this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); - this.costReportConsumer = controller.serviceRegistry().costReportConsumer(); - } - - @Override - public HttpResponse handle(HttpRequest request) { - if (request.getMethod() != GET) { - return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); - } - - Path path = new Path(request.getUri()); - - if (path.matches("/cost/v1/csv")) { - Optional<String> cloudProperty = Optional.ofNullable(request.getProperty("cloud")); - CloudName cloud = cloudProperty.map(CloudName::from).orElse(CloudName.defaultName()); - return new StringResponse(CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller, Clock.systemUTC(), costReportConsumer.fixedAllocations(), cloud)); - } - - return ErrorResponse.notFoundError("Nothing at " + path); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/package-info.java deleted file mode 100644 index a96ae5488fa..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.controller.restapi.cost; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java index 9ef2d519c05..32ec9f359a9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableSortedSet; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; import com.yahoo.vespa.hosted.controller.application.EndpointId; @@ -70,7 +71,10 @@ public class RoutingPolicy { /** Returns the endpoint of this */ public Endpoint endpointIn(SystemName system) { - return Endpoint.of(id.owner()).target(id.cluster(), id.zone()).on(Port.tls()).directRouting().in(system); + return Endpoint.of(id.owner()).target(id.cluster(), id.zone()) + .routingMethod(RoutingMethod.exclusive) + .on(Port.tls()) + .in(system); } /** Returns global endpoints which this is a member of */ @@ -100,7 +104,10 @@ public class RoutingPolicy { /** Creates a global endpoint for given application */ public static Endpoint globalEndpointOf(ApplicationId application, EndpointId endpointId, SystemName system) { - return Endpoint.of(application).named(endpointId).on(Port.tls()).directRouting().in(system); + return Endpoint.of(application).named(endpointId) + .on(Port.tls()) + .routingMethod(RoutingMethod.exclusive) + .in(system); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java index ea97e3e6c71..81bbbb479b2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; import org.junit.Test; @@ -47,23 +48,23 @@ public class EndpointTest { // Main endpoint with direct routing and default TLS port "https://a1.t1.global.vespa.oath.cloud/", - Endpoint.of(app1).named(endpointId).on(Port.tls()).directRouting().in(SystemName.main), + Endpoint.of(app1).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint with custom rotation name "https://r1.a1.t1.global.vespa.oath.cloud/", - Endpoint.of(app1).named(EndpointId.of("r1")).on(Port.tls()).directRouting().in(SystemName.main), + Endpoint.of(app1).named(EndpointId.of("r1")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint for custom instance in default rotation "https://i2.a2.t2.global.vespa.oath.cloud/", - Endpoint.of(app2).named(endpointId).on(Port.tls()).directRouting().in(SystemName.main), + Endpoint.of(app2).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint for custom instance with custom rotation name "https://r2.i2.a2.t2.global.vespa.oath.cloud/", - Endpoint.of(app2).named(EndpointId.of("r2")).on(Port.tls()).directRouting().in(SystemName.main), + Endpoint.of(app2).named(EndpointId.of("r2")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint in public system "https://a1.t1.global.public.vespa.oath.cloud/", - Endpoint.of(app1).named(endpointId).on(Port.tls()).directRouting().in(SystemName.Public) + Endpoint.of(app1).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public) ); tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); } @@ -95,23 +96,23 @@ public class EndpointTest { // Main endpoint with direct routing and default TLS port "https://a1.t1.global.vespa.oath.cloud/", - Endpoint.of(app1).named(endpointId).on(Port.tls()).directRouting().in(SystemName.main), + Endpoint.of(app1).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint with custom rotation name "https://r1.a1.t1.global.vespa.oath.cloud/", - Endpoint.of(app1).named(EndpointId.of("r1")).on(Port.tls()).directRouting().in(SystemName.main), + Endpoint.of(app1).named(EndpointId.of("r1")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint for custom instance in default rotation "https://i2.a2.t2.global.vespa.oath.cloud/", - Endpoint.of(app2).named(endpointId).on(Port.tls()).directRouting().in(SystemName.main), + Endpoint.of(app2).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint for custom instance with custom rotation name "https://r2.i2.a2.t2.global.vespa.oath.cloud/", - Endpoint.of(app2).named(EndpointId.of("r2")).on(Port.tls()).directRouting().in(SystemName.main), + Endpoint.of(app2).named(EndpointId.of("r2")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint in public system "https://a1.t1.global.public.vespa.oath.cloud/", - Endpoint.of(app1).named(endpointId).on(Port.tls()).directRouting().in(SystemName.Public) + Endpoint.of(app1).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public) ); tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); } @@ -153,11 +154,15 @@ public class EndpointTest { // Non-default cluster in public "https://c1.a1.t1.us-north-1.public.vespa.oath.cloud/", - Endpoint.of(app1).target(ClusterSpec.Id.from("c1"), prodZone).on(Port.tls()).directRouting().in(SystemName.Public), + Endpoint.of(app1).target(ClusterSpec.Id.from("c1"), prodZone).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public), // Non-default cluster and instance in public "https://c2.i2.a2.t2.us-north-1.public.vespa.oath.cloud/", - Endpoint.of(app2).target(ClusterSpec.Id.from("c2"), prodZone).on(Port.tls()).directRouting().in(SystemName.Public) + Endpoint.of(app2).target(ClusterSpec.Id.from("c2"), prodZone).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public), + + // Endpoint in main using shared layer 4 + "https://a1.t1.us-north-1.vespa.oath.cloud/", + Endpoint.of(app1).target(cluster, prodZone).on(Port.tls()).routingMethod(RoutingMethod.sharedLayer4).in(SystemName.main) ); tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); } @@ -173,7 +178,7 @@ public class EndpointTest { "https://a1.t1.global.public.vespa.oath.cloud/", Endpoint.of(app1) .named(EndpointId.defaultId()) - .directRouting() + .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) .in(SystemName.Public), @@ -181,7 +186,7 @@ public class EndpointTest { "https://*.a1.t1.global.public.vespa.oath.cloud/", Endpoint.of(app1) .wildcard() - .directRouting() + .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) .in(SystemName.Public), @@ -189,7 +194,7 @@ public class EndpointTest { "https://a1.t1.us-north-1.public.vespa.oath.cloud/", Endpoint.of(app1) .target(defaultCluster, prodZone) - .directRouting() + .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) .in(SystemName.Public), @@ -197,7 +202,7 @@ public class EndpointTest { "https://*.a1.t1.us-north-1.public.vespa.oath.cloud/", Endpoint.of(app1) .wildcard(prodZone) - .directRouting() + .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) .in(SystemName.Public), @@ -205,7 +210,7 @@ public class EndpointTest { "https://a1.t1.us-north-2.test.public.vespa.oath.cloud/", Endpoint.of(app1) .target(defaultCluster, testZone) - .directRouting() + .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) .in(SystemName.Public), @@ -213,7 +218,7 @@ public class EndpointTest { "https://*.a1.t1.us-north-2.test.public.vespa.oath.cloud/", Endpoint.of(app1) .wildcard(testZone) - .directRouting() + .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) .in(SystemName.Public) ); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java index 3f8e91dec58..93f081be63e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java @@ -1,4 +1,4 @@ -package com.yahoo.vespa.hosted.controller.endpointcertificates; +package com.yahoo.vespa.hosted.controller.certificate; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.SystemName; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java index 7428b9901a2..5f8a3eaa98a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java @@ -29,17 +29,10 @@ public class EndpointCertificateMetadataSerializerTest { } @Test - public void deserializeFromString() { - assertEquals( - new EndpointCertificateMetadata("foo-key", "foo-cert", 0), - EndpointCertificateMetadataSerializer.fromJsonOrTlsSecretsKeysString("foo")); - } - - @Test public void deserializeFromJson() { assertEquals( sample, - EndpointCertificateMetadataSerializer.fromJsonOrTlsSecretsKeysString( + EndpointCertificateMetadataSerializer.fromJsonString( "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1}")); } @@ -47,7 +40,7 @@ public class EndpointCertificateMetadataSerializerTest { public void deserializeFromJsonWithRequestMetadata() { assertEquals( sampleWithRequestMetadata, - EndpointCertificateMetadataSerializer.fromJsonOrTlsSecretsKeysString( + EndpointCertificateMetadataSerializer.fromJsonString( "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"]}")); } }
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index da770c9c023..9e03a236f4a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -92,9 +92,6 @@ public class ControllerContainerTest { " <handler id='com.yahoo.vespa.hosted.controller.restapi.os.OsApiHandler'>\n" + " <binding>http://*/os/v1/*</binding>\n" + " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.cost.CostApiHandler'>\n" + - " <binding>http://*/cost/v1/*</binding>\n" + - " </handler>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v2.ZoneApiHandler'>\n" + " <binding>http://*/zone/v2</binding>\n" + " <binding>http://*/zone/v2/*</binding>\n" + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 88faf54ea28..442770ba23e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -13,6 +13,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; @@ -58,6 +59,7 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.RotationStatusUpdater; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; @@ -82,6 +84,7 @@ import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Base64; import java.util.Collections; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -1475,14 +1478,17 @@ public class ApplicationApiTest extends ControllerContainerTest { @Test public void applicationWithRoutingPolicy() { var app = deploymentTester.newDeploymentContext(createTenantAndApplication()); + var zone = ZoneId.from(Environment.prod, RegionName.from("us-west-1")); + deploymentTester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(zone), + EnumSet.of(RoutingMethod.exclusive, RoutingMethod.shared)); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) .instances("instance1") - .region("us-west-1") + .region(zone.region().value()) .build(); app.submit(applicationPackage).deploy(); - app.addRoutingPolicy(ZoneId.from(Environment.prod, RegionName.from("us-west-1")), true); - app.addRoutingPolicy(ZoneId.from(Environment.prod, RegionName.from("us-west-1")), false); + app.addRoutingPolicy(zone, true); + app.addRoutingPolicy(zone, false); // GET application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", GET) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java deleted file mode 100644 index f992a54a114..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.cost; - -import com.yahoo.application.container.handler.Request; -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; -import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; -import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; -import org.junit.Before; -import org.junit.Test; - -/** - * @author andreer - */ -public class CostApiTest extends ControllerContainerTest { - - private static final String responses = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/responses/"; - private static final AthenzIdentity operator = AthenzUser.fromUserId("operatorUser"); - private static final CloudName cloud1 = CloudName.from("yahoo"); - private static final CloudName cloud2 = CloudName.from("cloud2"); - private static final ZoneApi zone1 = ZoneApiMock.newBuilder().withId("prod.us-east-3").with(cloud1).build(); - private static final ZoneApi zone2 = ZoneApiMock.newBuilder().withId("prod.us-west-1").with(cloud1).build(); - private static final ZoneApi zone3 = ZoneApiMock.newBuilder().withId("prod.eu-west-1").with(cloud2).build(); - - private ContainerTester tester; - - @Before - public void before() { - tester = new ContainerTester(container, responses); - tester.serviceRegistry().zoneRegistry().setSystemName(SystemName.cd) - .setZones(zone1, zone2, zone3); - } - - @Test - public void test_api() { - assertResponse(new Request("http://localhost:8080/cost/v1/csv"), - "Date,Property,Reserved Cpu Cores,Reserved Memory GB,Reserved Disk Space GB,Usage Fraction\n", 200); - } - - private void assertResponse(Request request, String body, int statusCode) { - addIdentityToRequest(request, operator); - tester.assertResponse(request, body, statusCode); - } - -} diff --git a/default_build_settings.cmake b/default_build_settings.cmake index e29e4c32017..a23a8f1489a 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -80,11 +80,17 @@ function(setup_vespa_default_build_settings_fedora_32) set(DEFAULT_VESPA_LLVM_VERSION "10" PARENT_SCOPE) endfunction() -function(setup_vespa_default_build_settings_ubuntu_18_10) - message("-- Setting up default build settings for ubuntu 18.10") - set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" "/usr/lib/llvm-6.0/lib" PARENT_SCOPE) - set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/lib/llvm-6.0/include" PARENT_SCOPE) - set(DEFAULT_VESPA_LLVM_VERSION "6.0" PARENT_SCOPE) +function(setup_vespa_default_build_settings_fedora_33) + message("-- Setting up default build settings for fedora 33") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "10" PARENT_SCOPE) +endfunction() + +function(setup_vespa_default_build_settings_ubuntu_19_10) + message("-- Setting up default build settings for ubuntu 19.10") + set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" "/usr/lib/llvm-9/lib" PARENT_SCOPE) + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/lib/llvm-9/include" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "9" PARENT_SCOPE) endfunction() function(vespa_use_default_vespa_unprivileged) @@ -173,8 +179,10 @@ function(vespa_use_default_build_settings) setup_vespa_default_build_settings_fedora_31() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 32") setup_vespa_default_build_settings_fedora_32() - elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "ubuntu 18.10") - setup_vespa_default_build_settings_ubuntu_18_10() + elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 33") + setup_vespa_default_build_settings_fedora_33() + elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "ubuntu 19.10") + setup_vespa_default_build_settings_ubuntu_19_10() else() message(FATAL_ERROR "-- Unkonwn vespa build platform ${VESPA_OS_DISTRO_COMBINED}") endif() diff --git a/dist/vespa.spec b/dist/vespa.spec index c54e4442167..8ce494fd7cb 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -60,27 +60,37 @@ BuildRequires: vespa-protobuf-devel >= 3.7.0-4 %if 0%{?fedora} BuildRequires: cmake >= 3.9.1 BuildRequires: maven -BuildRequires: vespa-protobuf-devel >= 3.7.0-4 BuildRequires: openssl-devel %if 0%{?fc29} +BuildRequires: vespa-protobuf-devel >= 3.7.0-4 BuildRequires: llvm-devel >= 7.0.0 BuildRequires: boost-devel >= 1.66 BuildRequires: gtest-devel BuildRequires: gmock-devel %endif %if 0%{?fc30} +BuildRequires: vespa-protobuf-devel >= 3.7.0-4 BuildRequires: llvm-devel >= 8.0.0 BuildRequires: boost-devel >= 1.69 BuildRequires: gtest-devel BuildRequires: gmock-devel %endif %if 0%{?fc31} +BuildRequires: vespa-protobuf-devel >= 3.7.0-4 BuildRequires: llvm-devel >= 9.0.0 BuildRequires: boost-devel >= 1.69 BuildRequires: gtest-devel BuildRequires: gmock-devel %endif %if 0%{?fc32} +BuildRequires: protobuf-devel +BuildRequires: llvm-devel >= 10.0.0 +BuildRequires: boost-devel >= 1.69 +BuildRequires: gtest-devel +BuildRequires: gmock-devel +%endif +%if 0%{?fc33} +BuildRequires: protobuf-devel BuildRequires: llvm-devel >= 10.0.0 BuildRequires: boost-devel >= 1.69 BuildRequires: gtest-devel @@ -159,21 +169,29 @@ Requires: openssl-libs %define _extra_include_directory %{_vespa_deps_prefix}/include;/usr/include/openblas %endif %if 0%{?fedora} -Requires: vespa-protobuf >= 3.7.0-4 Requires: openssl-libs %if 0%{?fc29} +Requires: vespa-protobuf >= 3.7.0-4 Requires: llvm-libs >= 7.0.0 %define _vespa_llvm_version 7 %endif %if 0%{?fc30} +Requires: vespa-protobuf >= 3.7.0-4 Requires: llvm-libs >= 8.0.0 %define _vespa_llvm_version 8 %endif %if 0%{?fc31} +Requires: vespa-protobuf >= 3.7.0-4 Requires: llvm-libs >= 9.0.0 %define _vespa_llvm_version 9 %endif %if 0%{?fc32} +Requires: protobuf +Requires: llvm-libs >= 10.0.0 +%define _vespa_llvm_version 10 +%endif +%if 0%{?fc33} +Requires: protobuf Requires: llvm-libs >= 10.0.0 %define _vespa_llvm_version 10 %endif diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java index 32302a98757..ecfe9b2468a 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java @@ -34,6 +34,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { private final Map<String, String> labels = new HashMap<>(); private final List<String> environmentAssignments = new ArrayList<>(); private final List<String> volumeBindSpecs = new ArrayList<>(); + private final List<String> dnsOptions = new ArrayList<>(); private final List<Ulimit> ulimits = new ArrayList<>(); private final Set<Capability> addCapabilities = new HashSet<>(); private final Set<Capability> dropCapabilities = new HashSet<>(); @@ -96,6 +97,12 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { } @Override + public Docker.CreateContainerCommand withDnsOption(String dnsOption) { + dnsOptions.add(dnsOption); + return this; + } + + @Override public Docker.CreateContainerCommand withPrivileged(boolean privileged) { this.privileged = privileged; return this; @@ -171,6 +178,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { .withPidsLimit(-1L) .withCapAdd(addCapabilities.toArray(new Capability[0])) .withCapDrop(dropCapabilities.toArray(new Capability[0])) + .withDnsOptions(dnsOptions) .withPrivileged(privileged); containerResources.ifPresent(cr -> hostConfig @@ -241,6 +249,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { toRepeatedOption("--cap-add", addCapabilitiesList), toRepeatedOption("--cap-drop", dropCapabilitiesList), toRepeatedOption("--security-opt", securityOpts), + toRepeatedOption("--dns-option", dnsOptions), toOptionalOption("--net", networkMode), toOptionalOption("--ip", ipv4Address), toOptionalOption("--ip6", ipv6Address), diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java index 4e7ef5a1ff6..648c94d71ab 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java @@ -53,6 +53,7 @@ public interface Docker { CreateContainerCommand withAddCapability(String capabilityName); CreateContainerCommand withDropCapability(String capabilityName); CreateContainerCommand withSecurityOpts(String securityOpt); + CreateContainerCommand withDnsOption(String dnsOption); CreateContainerCommand withPrivileged(boolean privileged); void create(); diff --git a/eval/src/vespa/eval/eval/interpreted_function.cpp b/eval/src/vespa/eval/eval/interpreted_function.cpp index 0a630a3e20a..ec28604fd87 100644 --- a/eval/src/vespa/eval/eval/interpreted_function.cpp +++ b/eval/src/vespa/eval/eval/interpreted_function.cpp @@ -3,21 +3,16 @@ #include "interpreted_function.h" #include "node_visitor.h" #include "node_traverser.h" -#include "check_type.h" -#include "tensor_spec.h" -#include "operation.h" #include "tensor_nodes.h" #include "tensor_engine.h" +#include "make_tensor_function.h" +#include "compile_tensor_function.h" #include <vespa/vespalib/util/classname.h> #include <vespa/eval/eval/llvm/compile_cache.h> #include <vespa/vespalib/util/benchmark_timer.h> #include <set> -#include "make_tensor_function.h" -#include "compile_tensor_function.h" - -namespace vespalib { -namespace eval { +namespace vespalib::eval { namespace { @@ -42,11 +37,12 @@ InterpretedFunction::State::State(const TensorEngine &engine_in) params(nullptr), stash(), stack(), - program_offset(0) + program_offset(0), + if_cnt(0) { } -InterpretedFunction::State::~State() {} +InterpretedFunction::State::~State() = default; void InterpretedFunction::State::init(const LazyParams ¶ms_in) { @@ -82,7 +78,7 @@ InterpretedFunction::InterpretedFunction(const TensorEngine &engine, const nodes _program = compile_tensor_function(optimized, _stash); } -InterpretedFunction::~InterpretedFunction() {} +InterpretedFunction::~InterpretedFunction() = default; const Value & InterpretedFunction::eval(Context &ctx, const LazyParams ¶ms) const @@ -123,5 +119,4 @@ InterpretedFunction::detect_issues(const Function &function) return Function::Issues(std::move(checker.issues)); } -} // namespace vespalib::eval -} // namespace vespalib +} diff --git a/eval/src/vespa/eval/eval/interpreted_function.h b/eval/src/vespa/eval/eval/interpreted_function.h index e3e8d18b44f..e638ccffcea 100644 --- a/eval/src/vespa/eval/eval/interpreted_function.h +++ b/eval/src/vespa/eval/eval/interpreted_function.h @@ -7,8 +7,7 @@ #include "lazy_params.h" #include <vespa/vespalib/util/stash.h> -namespace vespalib { -namespace eval { +namespace vespalib::eval { namespace nodes { struct Node; } struct TensorEngine; @@ -107,5 +106,4 @@ public: static Function::Issues detect_issues(const Function &function); }; -} // namespace vespalib::eval -} // namespace vespalib +} diff --git a/eval/src/vespa/eval/eval/lazy_params.cpp b/eval/src/vespa/eval/eval/lazy_params.cpp index aec8cf78059..2c00c4c312b 100644 --- a/eval/src/vespa/eval/eval/lazy_params.cpp +++ b/eval/src/vespa/eval/eval/lazy_params.cpp @@ -1,19 +1,16 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "lazy_params.h" -#include <assert.h> +#include <vespa/vespalib/util/stash.h> +#include <cassert> namespace vespalib::eval { -LazyParams::~LazyParams() -{ -} +LazyParams::~LazyParams() = default; //----------------------------------------------------------------------------- -SimpleObjectParams::~SimpleObjectParams() -{ -} +SimpleObjectParams::~SimpleObjectParams() = default; const Value & SimpleObjectParams::resolve(size_t idx, Stash &) const @@ -24,9 +21,7 @@ SimpleObjectParams::resolve(size_t idx, Stash &) const //----------------------------------------------------------------------------- -SimpleParams::~SimpleParams() -{ -} +SimpleParams::~SimpleParams() = default; const Value & SimpleParams::resolve(size_t idx, Stash &stash) const diff --git a/eval/src/vespa/eval/eval/tensor_function.h b/eval/src/vespa/eval/eval/tensor_function.h index 4b862e9ec6a..55d27fb74ea 100644 --- a/eval/src/vespa/eval/eval/tensor_function.h +++ b/eval/src/vespa/eval/eval/tensor_function.h @@ -2,20 +2,17 @@ #pragma once -#include <memory> -#include <vector> -#include <variant> -#include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/util/arrayref.h> -#include <vespa/vespalib/util/overload.h> #include "tensor_spec.h" #include "lazy_params.h" #include "value_type.h" #include "value.h" #include "aggr.h" - #include "interpreted_function.h" +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/arrayref.h> +#include <vespa/vespalib/util/overload.h> +#include <variant> namespace vespalib { diff --git a/eval/src/vespa/eval/eval/value.h b/eval/src/vespa/eval/eval/value.h index 6701173bcd3..cad76c93c5c 100644 --- a/eval/src/vespa/eval/eval/value.h +++ b/eval/src/vespa/eval/eval/value.h @@ -2,13 +2,11 @@ #pragma once -#include <vespa/vespalib/stllike/string.h> -#include <memory> -#include <vespa/vespalib/util/stash.h> #include "value_type.h" +#include <vespa/vespalib/util/traits.h> +#include <memory> -namespace vespalib { -namespace eval { +namespace vespalib::eval { class Tensor; @@ -40,7 +38,6 @@ public: static const ValueType &double_type() { return _type; } }; -} // namespace vespalib::eval -} // namespace vespalib +} VESPA_CAN_SKIP_DESTRUCTION(::vespalib::eval::DoubleValue); diff --git a/fbench/CMakeLists.txt b/fbench/CMakeLists.txt index 5cc56786227..3da632d98a6 100644 --- a/fbench/CMakeLists.txt +++ b/fbench/CMakeLists.txt @@ -15,6 +15,7 @@ vespa_define_module( TESTS src/test + src/test/authority ) vespa_install_script(util/resultfilter.pl vespa-fbench-result-filter.pl bin) diff --git a/fbench/src/fbench/fbench.cpp b/fbench/src/fbench/fbench.cpp index 91475ce2125..593ae30a0e5 100644 --- a/fbench/src/fbench/fbench.cpp +++ b/fbench/src/fbench/fbench.cpp @@ -86,10 +86,13 @@ FBench::init_crypto_engine(const std::string &ca_certs_file_name, return false; } bool load_failed = false; - vespalib::net::tls::TransportSecurityOptions - tls_opts(maybe_load(ca_certs_file_name, load_failed), - maybe_load(cert_chain_file_name, load_failed), - maybe_load(private_key_file_name, load_failed)); + auto ts_builder = vespalib::net::tls::TransportSecurityOptions::Params(). + ca_certs_pem(maybe_load(ca_certs_file_name, load_failed)). + cert_chain_pem(maybe_load(cert_chain_file_name, load_failed)). + private_key_pem(maybe_load(private_key_file_name, load_failed)). + authorized_peers(vespalib::net::tls::AuthorizedPeers::allow_all_authenticated()). + disable_hostname_validation(true); // TODO configurable or default false! + vespalib::net::tls::TransportSecurityOptions tls_opts(std::move(ts_builder)); if (load_failed) { fprintf(stderr, "failed to load transport security options\n"); return false; diff --git a/fbench/src/httpclient/CMakeLists.txt b/fbench/src/httpclient/CMakeLists.txt index 5f3333128b3..a28f3666383 100644 --- a/fbench/src/httpclient/CMakeLists.txt +++ b/fbench/src/httpclient/CMakeLists.txt @@ -3,5 +3,6 @@ vespa_add_library(fbench_httpclient STATIC SOURCES httpclient.cpp DEPENDS + fbench_util fastos ) diff --git a/fbench/src/httpclient/httpclient.cpp b/fbench/src/httpclient/httpclient.cpp index 99134a6e297..9615a6e6df7 100644 --- a/fbench/src/httpclient/httpclient.cpp +++ b/fbench/src/httpclient/httpclient.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "httpclient.h" #include <vespa/vespalib/net/socket_spec.h> +#include <util/authority.h> #include <cassert> #include <cstring> @@ -29,7 +30,8 @@ HTTPClient::HTTPClient(vespalib::CryptoEngine::SP engine, const char *hostname, _keepAlive(keepAlive), _headerBenchmarkdataCoverage(headerBenchmarkdataCoverage), _extraHeaders(extraHeaders), - _authority(authority), + _sni_spec(make_sni_spec(authority, hostname, port, _engine->use_tls_when_client())), + _host_header_value(make_host_header_value(_sni_spec, _engine->use_tls_when_client())), _reuseCount(0), _bufsize(10240), _buf(new char[_bufsize]), @@ -51,11 +53,6 @@ HTTPClient::HTTPClient(vespalib::CryptoEngine::SP engine, const char *hostname, _dataDone(false), _reader(NULL) { - if (_authority == "") { - char tmp[1024]; - snprintf(tmp, 1024, "%s:%d", hostname, port); - _authority = tmp; - } } bool @@ -70,8 +67,7 @@ HTTPClient::connect_socket() if (!handle.valid()) { return false; } - _socket = vespalib::SyncCryptoSocket::create_client(*_engine, std::move(handle), - vespalib::SocketSpec::from_host_port(_hostname, _port)); + _socket = vespalib::SyncCryptoSocket::create_client(*_engine, std::move(handle), _sni_spec); return bool(_socket); } @@ -153,14 +149,14 @@ HTTPClient::Connect(const char *url, bool usePost, const char *content, int cLen "Content-Length: %d\r\n" "%s" "\r\n", - url, _authority.c_str(), cLen, headers.c_str()); + url, _host_header_value.c_str(), cLen, headers.c_str()); } else { snprintf(req, req_max, "GET %s HTTP/1.1\r\n" "Host: %s\r\n" "%s" "\r\n", - url, _authority.c_str(), headers.c_str()); + url, _host_header_value.c_str(), headers.c_str()); } // try to reuse connection if keep-alive is enabled diff --git a/fbench/src/httpclient/httpclient.h b/fbench/src/httpclient/httpclient.h index 9c3ccd437d1..cad01826db7 100644 --- a/fbench/src/httpclient/httpclient.h +++ b/fbench/src/httpclient/httpclient.h @@ -6,6 +6,7 @@ #include <vespa/vespalib/net/sync_crypto_socket.h> #include <vespa/vespalib/net/crypto_engine.h> #include <vespa/vespalib/net/socket_address.h> +#include <vespa/vespalib/net/socket_spec.h> /** * This class implements a HTTP client that may be used to fetch @@ -99,7 +100,8 @@ protected: bool _keepAlive; bool _headerBenchmarkdataCoverage; std::string _extraHeaders; - std::string _authority; + vespalib::SocketSpec _sni_spec; + std::string _host_header_value; uint64_t _reuseCount; size_t _bufsize; diff --git a/fbench/src/test/authority/CMakeLists.txt b/fbench/src/test/authority/CMakeLists.txt new file mode 100644 index 00000000000..00f804f43f6 --- /dev/null +++ b/fbench/src/test/authority/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(fbench_authority_test_app TEST + SOURCES + authority_test.cpp + DEPENDS + fbench_util + vespalib + gtest +) +vespa_add_test(NAME fbench_authority_test_app COMMAND fbench_authority_test_app) diff --git a/fbench/src/test/authority/authority_test.cpp b/fbench/src/test/authority/authority_test.cpp new file mode 100644 index 00000000000..de723a8730f --- /dev/null +++ b/fbench/src/test/authority/authority_test.cpp @@ -0,0 +1,87 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <util/authority.h> +#include <vespa/vespalib/gtest/gtest.h> + +using vespalib::SocketSpec; + +//----------------------------------------------------------------------------- + +TEST(MakeSNISpecTest, host_port_is_parsed_as_expected) { + EXPECT_EQ(make_sni_spec("my_host:123", "fallback", 456, false).host(), "my_host"); + EXPECT_EQ(make_sni_spec("my_host:123", "fallback", 456, true).host(), "my_host"); + EXPECT_EQ(make_sni_spec("my_host:123", "fallback", 456, false).port(), 123); + EXPECT_EQ(make_sni_spec("my_host:123", "fallback", 456, true).port(), 123); +} + +TEST(MakeSNISpecTest, user_info_is_stripped) { + EXPECT_EQ(make_sni_spec("myuser:deprecated@my_host:123", "fallback", 456, false).host(), "my_host"); + EXPECT_EQ(make_sni_spec("myuser:deprecated@my_host:123", "fallback", 456, true).host(), "my_host"); + EXPECT_EQ(make_sni_spec("myuser:deprecated@my_host:123", "fallback", 456, false).port(), 123); + EXPECT_EQ(make_sni_spec("myuser:deprecated@my_host:123", "fallback", 456, true).port(), 123); +} + +TEST(MakeSNISpecTest, port_can_be_skipped) { + EXPECT_EQ(make_sni_spec("my_host", "fallback", 456, false).host(), "my_host"); + EXPECT_EQ(make_sni_spec("my_host", "fallback", 456, true).host(), "my_host"); + EXPECT_EQ(make_sni_spec("my_host", "fallback", 456, false).port(), 80); + EXPECT_EQ(make_sni_spec("my_host", "fallback", 456, true).port(), 443); +} + +TEST(MakeSNISpecTest, quoted_ip_addresses_work_as_expected) { + EXPECT_EQ(make_sni_spec("[::1]:123", "fallback", 456, false).host(), "::1"); + EXPECT_EQ(make_sni_spec("[::1]:123", "fallback", 456, true).host(), "::1"); + EXPECT_EQ(make_sni_spec("[::1]:123", "fallback", 456, false).port(), 123); + EXPECT_EQ(make_sni_spec("[::1]:123", "fallback", 456, true).port(), 123); + EXPECT_EQ(make_sni_spec("[::1]", "fallback", 456, false).host(), "::1"); + EXPECT_EQ(make_sni_spec("[::1]", "fallback", 456, true).host(), "::1"); + EXPECT_EQ(make_sni_spec("[::1]", "fallback", 456, false).port(), 80); + EXPECT_EQ(make_sni_spec("[::1]", "fallback", 456, true).port(), 443); +} + +TEST(MakeSNISpecTest, supplied_host_port_is_used_as_fallback) { + EXPECT_EQ(make_sni_spec("", "fallback", 456, false).host(), "fallback"); + EXPECT_EQ(make_sni_spec("", "fallback", 456, true).host(), "fallback"); + EXPECT_EQ(make_sni_spec("", "fallback", 456, false).port(), 456); + EXPECT_EQ(make_sni_spec("", "fallback", 456, true).port(), 456); +} + +//----------------------------------------------------------------------------- + +TEST(MakeHostHeaderValueTest, host_port_is_formatted_as_expected) { + auto my_spec = SocketSpec::from_host_port("myhost", 123); + EXPECT_EQ(make_host_header_value(my_spec, false), "myhost:123"); + EXPECT_EQ(make_host_header_value(my_spec, true), "myhost:123"); +} + +TEST(MakeHostHeaderValueTest, inappropriate_spec_gives_empty_host_value) { + std::vector<SocketSpec> bad_specs = { + SocketSpec::invalid, + SocketSpec::from_port(123), + SocketSpec::from_name("foo"), + SocketSpec::from_path("bar") + }; + for (const auto &spec: bad_specs) { + EXPECT_EQ(make_host_header_value(spec, false), ""); + EXPECT_EQ(make_host_header_value(spec, true), ""); + } +} + +TEST(MakeHostHeaderValueTest, default_port_is_omitted) { + auto spec1 = SocketSpec::from_host_port("myhost", 80); + auto spec2 = SocketSpec::from_host_port("myhost", 443); + EXPECT_EQ(make_host_header_value(spec1, false), "myhost"); + EXPECT_EQ(make_host_header_value(spec1, true), "myhost:80"); + EXPECT_EQ(make_host_header_value(spec2, false), "myhost:443"); + EXPECT_EQ(make_host_header_value(spec2, true), "myhost"); +} + +TEST(MakeHostHeaderValueTest, ipv6_addresses_are_quoted) { + auto my_spec = SocketSpec::from_host_port("::1", 123); + EXPECT_EQ(make_host_header_value(my_spec, false), "[::1]:123"); + EXPECT_EQ(make_host_header_value(my_spec, true), "[::1]:123"); +} + +//----------------------------------------------------------------------------- + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/fbench/src/util/CMakeLists.txt b/fbench/src/util/CMakeLists.txt index 47cc46ffc8f..3cdff26ce16 100644 --- a/fbench/src/util/CMakeLists.txt +++ b/fbench/src/util/CMakeLists.txt @@ -1,8 +1,9 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(fbench_util STATIC SOURCES + authority.cpp + clientstatus.cpp filereader.cpp timer.cpp - clientstatus.cpp DEPENDS ) diff --git a/fbench/src/util/authority.cpp b/fbench/src/util/authority.cpp new file mode 100644 index 00000000000..6247c72d9b0 --- /dev/null +++ b/fbench/src/util/authority.cpp @@ -0,0 +1,42 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "authority.h" +#include <vespa/vespalib/util/stringfmt.h> +#include <cassert> + +namespace { + +int default_port(bool use_https) { return use_https ? 443 : 80; } + +} + +vespalib::SocketSpec make_sni_spec(const std::string &authority, const char *hostname, int port, bool use_https) { + if (authority.empty()) { + return vespalib::SocketSpec::from_host_port(hostname, port); + } + auto split = authority.rfind('@'); + std::string spec_str = (split == std::string::npos) ? authority : authority.substr(split + 1); + auto a = spec_str.rfind(':'); + auto b = spec_str.rfind(']'); + bool has_port = (a != std::string::npos) && ((b == std::string::npos) || (a > b)); + if (has_port) { + spec_str = "tcp/" + spec_str; + } else { + spec_str = vespalib::make_string("tcp/%s:%d", spec_str.c_str(), default_port(use_https)); + } + // use SocketSpec parser to ensure ipv6 addresses are dequoted + return vespalib::SocketSpec(spec_str); +} + +std::string make_host_header_value(const vespalib::SocketSpec &sni_spec, bool use_https) { + if (sni_spec.host().empty()) { + return ""; + } + if (sni_spec.port() == default_port(use_https)) { + return sni_spec.host(); + } + // use SocketSpec formatter to ensure ipv6 addresses are quoted + std::string spec_str = sni_spec.spec(); + assert(spec_str.find("tcp/") == 0); + return spec_str.substr(4); +} diff --git a/fbench/src/util/authority.h b/fbench/src/util/authority.h new file mode 100644 index 00000000000..49dab4a29fd --- /dev/null +++ b/fbench/src/util/authority.h @@ -0,0 +1,30 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/net/socket_spec.h> + +/** + * Assemble an SNI (Server Name Indication) spec that will be used + * when handshaking over TLS. The authority will be used if + * non-empty. Hostname/port will be used as fall-back. Note that the + * SNI spec will also be used to generate the Host header used in + * subsequent HTTP requests. + * + * @return sni spec + * @param authority user-provided authority + * @param hostname name of the host we are connecting to + * @param port which port we are connecting to + * @param use_https are we using https? (TLS) + **/ +vespalib::SocketSpec make_sni_spec(const std::string &authority, const char *hostname, int port, bool use_https); + +/** + * Use an SNI spec to generate a matching Host header to be used in + * HTTP requests. Note that default port numbers will be omitted. + * + * @return host header value + * @param sni_spec SNI spec + * @param use_https are we using https? (TLS) + **/ +std::string make_host_header_value(const vespalib::SocketSpec &sni_spec, bool use_https); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index fc54c61fca8..bb4e281a85f 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -74,7 +74,7 @@ public class Flags { HOSTNAME, NODE_TYPE); public static final UnboundStringFlag DOCKER_VERSION = defineStringFlag( - "docker-version", "1.13.1-91.git07f3374", + "docker-version", "1.13.1-102.git7f2769b", "The version of the docker to use of the format VERSION-REL: The YUM package to be installed will be " + "2:docker-VERSION-REL.el7.centos.x86_64 in AWS (and without '.centos' otherwise). " + "If docker-version is not of this format, it must be parseable by YumPackageName::fromString.", @@ -126,14 +126,6 @@ public class Flags { "scheduled evenly distributed in the 1x-2x range (and naturally guaranteed at the 2x boundary).", "Takes effect on next run of NodeRebooter"); - public static final UnboundBooleanFlag ENABLE_LARGE_ORCHESTRATOR_LOCKS = defineFeatureFlag( - "enable-large-orchestrator-locks", true, - "If enabled, the orchestrator will accumulate application locks during probe in batch suspension, " + - "and release them in reverse order only after the non-probe is complete. Can be set depending on " + - "parent hostname.", - "Takes immediate effect for new batch suspensions.", - HOSTNAME); - public static final UnboundBooleanFlag RETIRE_WITH_PERMANENTLY_DOWN = defineFeatureFlag( "retire-with-permanently-down", false, "If enabled, retirement will end with setting the host status to PERMANENTLY_DOWN, " + diff --git a/fnet/src/tests/connect/connect_test.cpp b/fnet/src/tests/connect/connect_test.cpp index d94b6759077..62000efb682 100644 --- a/fnet/src/tests/connect/connect_test.cpp +++ b/fnet/src/tests/connect/connect_test.cpp @@ -65,6 +65,8 @@ struct BlockingCryptoEngine : public CryptoEngine { Gate handshake_work_enter; Gate handshake_work_exit; Gate handshake_socket_deleted; + bool use_tls_when_client() const override { return false; } + bool always_use_tls_when_server() const override { return false; } CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &) override { return std::make_unique<BlockingCryptoSocket>(std::move(socket), handshake_work_enter, handshake_work_exit, handshake_socket_deleted); diff --git a/fnet/src/tests/frt/values/values_test.cpp b/fnet/src/tests/frt/values/values_test.cpp index 5bc6b0e2dce..3b36e8989c1 100644 --- a/fnet/src/tests/frt/values/values_test.cpp +++ b/fnet/src/tests/frt/values/values_test.cpp @@ -3,6 +3,7 @@ #include <vespa/fnet/frt/values.h> #include <vespa/fnet/databuffer.h> #include <vespa/fnet/info.h> +#include <vespa/vespalib/util/stash.h> using vespalib::Stash; diff --git a/fnet/src/vespa/fnet/frt/rpcrequest.h b/fnet/src/vespa/fnet/frt/rpcrequest.h index cc871e7ac0c..eaa34a46b7a 100644 --- a/fnet/src/vespa/fnet/frt/rpcrequest.h +++ b/fnet/src/vespa/fnet/frt/rpcrequest.h @@ -5,6 +5,7 @@ #include "values.h" #include "error.h" #include <vespa/fnet/context.h> +#include <vespa/vespalib/util/stash.h> #include <atomic> class FNETConnection; diff --git a/fnet/src/vespa/fnet/frt/values.cpp b/fnet/src/vespa/fnet/frt/values.cpp index a5f59df19b2..3b37aa9a1bc 100644 --- a/fnet/src/vespa/fnet/frt/values.cpp +++ b/fnet/src/vespa/fnet/frt/values.cpp @@ -3,6 +3,7 @@ #include "values.h" #include <vespa/fnet/databuffer.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> #include <cassert> static_assert(sizeof(uint8_t) == 1, "uint8_t must be 1 byte."); @@ -81,7 +82,7 @@ FRT_Values::FRT_Values(Stash &stash) _stash(stash) { } -FRT_Values::~FRT_Values() { } +FRT_Values::~FRT_Values() = default; LocalBlob::LocalBlob(const char *data, uint32_t len) : _data(Alloc::alloc(len)), @@ -294,7 +295,7 @@ FRT_Values::AddSharedData(FRT_ISharedBlob *blob) { } void -FRT_Values::AddData(vespalib::alloc::Alloc buf, uint32_t len) { +FRT_Values::AddData(vespalib::alloc::Alloc && buf, uint32_t len) { AddSharedData(&_stash.create<LocalBlob>(std::move(buf), len)); } diff --git a/fnet/src/vespa/fnet/frt/values.h b/fnet/src/vespa/fnet/frt/values.h index e00aec8423c..2aa7551c423 100644 --- a/fnet/src/vespa/fnet/frt/values.h +++ b/fnet/src/vespa/fnet/frt/values.h @@ -3,9 +3,10 @@ #pragma once #include "isharedblob.h" -#include <vespa/vespalib/util/stash.h> #include <cstring> +namespace vespalib { class Stash; } +namespace vespalib::alloc { class Alloc; } namespace fnet { char * copyString(char *dst, const char *src, size_t len); char * copyData(char *dst, const void *src, size_t len); @@ -216,7 +217,7 @@ public: char *AddString(uint32_t len); FRT_StringValue *AddStringArray(uint32_t len); void AddSharedData(FRT_ISharedBlob *blob); - void AddData(Alloc buf, uint32_t len); + void AddData(Alloc && buf, uint32_t len); void AddData(const char *buf, uint32_t len); char *AddData(uint32_t len); FRT_DataValue *AddDataArray(uint32_t len); diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java index 88db5c99de9..eb292199ea2 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java @@ -5,6 +5,7 @@ import com.yahoo.security.KeyUtils; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.tls.AuthorizationMode; import com.yahoo.security.tls.DefaultTlsContext; +import com.yahoo.security.tls.HostnameVerification; import com.yahoo.security.tls.PeerAuthentication; import com.yahoo.security.tls.TlsContext; import com.yahoo.security.tls.policy.AuthorizedPeers; @@ -51,7 +52,7 @@ public class TlsContextBasedProviderTest { BigInteger.ONE) .build(); return new DefaultTlsContext( - List.of(certificate), keyPair.getPrivate(), List.of(certificate), new AuthorizedPeers(Set.of()), AuthorizationMode.ENFORCE, PeerAuthentication.NEED); + List.of(certificate), keyPair.getPrivate(), List.of(certificate), new AuthorizedPeers(Set.of()), AuthorizationMode.ENFORCE, PeerAuthentication.NEED, HostnameVerification.ENABLED); } private static class SimpleTlsContextBasedProvider extends TlsContextBasedProvider { diff --git a/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java b/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java index 7474220d4e7..a140e87713c 100644 --- a/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java +++ b/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java @@ -21,7 +21,8 @@ public class TlsCryptoEngine implements CryptoEngine { @Override public TlsCryptoSocket createClientCryptoSocket(SocketChannel channel, Spec spec) { - SSLEngine sslEngine = tlsContext.createSslEngine(); + String peerHost = spec.host() != null ? spec.host() : "localhost"; // Use localhost for wildcard address + SSLEngine sslEngine = tlsContext.createSslEngine(peerHost, spec.port()); sslEngine.setUseClientMode(true); return new TlsCryptoSocket(channel, sslEngine); } diff --git a/jrt/tests/com/yahoo/jrt/CryptoUtils.java b/jrt/tests/com/yahoo/jrt/CryptoUtils.java index e7e4eea568d..95ea581cb90 100644 --- a/jrt/tests/com/yahoo/jrt/CryptoUtils.java +++ b/jrt/tests/com/yahoo/jrt/CryptoUtils.java @@ -5,6 +5,7 @@ import com.yahoo.security.KeyUtils; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.tls.AuthorizationMode; import com.yahoo.security.tls.DefaultTlsContext; +import com.yahoo.security.tls.HostnameVerification; import com.yahoo.security.tls.PeerAuthentication; import com.yahoo.security.tls.TlsContext; import com.yahoo.security.tls.policy.AuthorizedPeers; @@ -35,21 +36,23 @@ class CryptoUtils { static final KeyPair keyPair = KeyUtils.generateKeypair(EC); static final X509Certificate certificate = X509CertificateBuilder - .fromKeypair(keyPair, new X500Principal("CN=dummy"), EPOCH, Instant.now().plus(1, DAYS), SHA256_WITH_ECDSA, generateRandomSerialNumber()) + .fromKeypair(keyPair, new X500Principal("CN=localhost"), EPOCH, Instant.now().plus(1, DAYS), SHA256_WITH_ECDSA, generateRandomSerialNumber()) .build(); static final AuthorizedPeers authorizedPeers = new AuthorizedPeers( singleton( new PeerPolicy( - "dummy-policy", + "localhost-policy", singleton( - new Role("dummy-role")), + new Role("localhost-role")), singletonList( new RequiredPeerCredential( - Field.CN, new HostGlobPattern("dummy")))))); + Field.CN, new HostGlobPattern("localhost")))))); static TlsContext createTestTlsContext() { - return new DefaultTlsContext(singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers, AuthorizationMode.ENFORCE, PeerAuthentication.NEED); + return new DefaultTlsContext( + singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers, + AuthorizationMode.ENFORCE, PeerAuthentication.NEED, HostnameVerification.ENABLED); } } diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml index f72ad75c6af..355f420c2a4 100644 --- a/metrics-proxy/pom.xml +++ b/metrics-proxy/pom.xml @@ -132,6 +132,10 @@ <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> </dependency> + <dependency> + <groupId>org.apache.velocity</groupId> + <artifactId>velocity</artifactId> + </dependency> <!-- test scope --> diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java new file mode 100644 index 00000000000..2afc0267434 --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java @@ -0,0 +1,95 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.telegraf; + +import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; +import com.yahoo.log.LogLevel; +import com.yahoo.system.execution.ProcessExecutor; +import com.yahoo.system.execution.ProcessResult; +import com.yahoo.vespa.defaults.Defaults; +import org.apache.velocity.VelocityContext; +import org.apache.velocity.app.VelocityEngine; + +import java.io.FileWriter; +import java.io.InputStreamReader; +import java.io.Reader; +import java.io.Writer; +import java.util.logging.Logger; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * @author olaa + */ +public class Telegraf extends AbstractComponent { + + private static final String TELEGRAF_CONFIG_PATH = "/etc/telegraf/telegraf.conf"; + private static final String TELEGRAF_CONFIG_TEMPLATE_PATH = "templates/telegraf.conf.vm"; + private static final String TELEGRAF_LOG_FILE_PATH = Defaults.getDefaults().underVespaHome("logs/telegraf/telegraf.log"); + private final TelegrafRegistry telegrafRegistry; + + private static final Logger logger = Logger.getLogger(Telegraf.class.getName()); + + @Inject + public Telegraf(TelegrafRegistry telegrafRegistry, TelegrafConfig telegrafConfig) { + this.telegrafRegistry = telegrafRegistry; + telegrafRegistry.addInstance(this); + writeConfig(telegrafConfig, uncheck(() -> new FileWriter(TELEGRAF_CONFIG_PATH))); + restartTelegraf(); + } + + protected static void writeConfig(TelegrafConfig telegrafConfig, Writer writer) { + VelocityContext context = new VelocityContext(); + context.put("logFilePath", TELEGRAF_LOG_FILE_PATH); + context.put("intervalSeconds", telegrafConfig.intervalSeconds()); + context.put("cloudwatchPlugins", telegrafConfig.cloudWatch()); + // TODO: Add node cert if hosted + + VelocityEngine velocityEngine = new VelocityEngine(); + velocityEngine.init(); + velocityEngine.evaluate(context, writer, "TelegrafConfigWriter", getTemplateReader()); + uncheck(writer::close); + } + + private void restartTelegraf() { + executeCommand("service telegraf restart"); + } + + private void stopTelegraf() { + executeCommand("service telegraf stop"); + } + + private void executeCommand(String command) { + logger.info(String.format("Running command: %s", command)); + ProcessExecutor processExecutor = new ProcessExecutor + .Builder(10) + .successExitCodes(0) + .build(); + ProcessResult processResult = uncheck(() -> processExecutor.execute(command)) + .orElseThrow(() -> new RuntimeException("Timed out running command: " + command)); + + logger.log(LogLevel.DEBUG, () -> String.format("Exit code: %d\nstdOut: %s\nstdErr: %s", + processResult.exitCode, + processResult.stdOut, + processResult.stdErr)); + + if (!processResult.stdErr.isBlank()) + logger.warning(String.format("stdErr not empty: %s", processResult.stdErr)); + } + + @SuppressWarnings("ConstantConditions") + private static Reader getTemplateReader() { + return new InputStreamReader(Telegraf.class.getClassLoader() + .getResourceAsStream(TELEGRAF_CONFIG_TEMPLATE_PATH) + ); + + } + + @Override + public void deconstruct() { + telegrafRegistry.removeInstance(this); + if (telegrafRegistry.isEmpty()) { + stopTelegraf(); + } + } +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java new file mode 100644 index 00000000000..23da51ea082 --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java @@ -0,0 +1,33 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.telegraf; + +import com.yahoo.log.LogLevel; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.logging.Logger; + +/** + * @author olaa + */ +public class TelegrafRegistry { + + private static final List<Telegraf> telegrafInstances = Collections.synchronizedList(new ArrayList<>()); + + private static final Logger logger = Logger.getLogger(TelegrafRegistry.class.getName()); + + public void addInstance(Telegraf telegraf) { + logger.log(LogLevel.DEBUG, () -> "Adding Telegraf instance to registry: " + telegraf.hashCode()); + telegrafInstances.add(telegraf); + } + + public void removeInstance(Telegraf telegraf) { + logger.log(LogLevel.DEBUG, () -> "Removing Telegraf instance from registry: " + telegraf.hashCode()); + telegrafInstances.remove(telegraf); + } + + public boolean isEmpty() { + return telegrafInstances.isEmpty(); + } +} diff --git a/metrics-proxy/src/main/resources/configdefinitions/telegraf.def b/metrics-proxy/src/main/resources/configdefinitions/telegraf.def index f3b5db35d52..6abbd7921b5 100644 --- a/metrics-proxy/src/main/resources/configdefinitions/telegraf.def +++ b/metrics-proxy/src/main/resources/configdefinitions/telegraf.def @@ -5,9 +5,8 @@ package=ai.vespa.metricsproxy.telegraf intervalSeconds int default=60 -# The consumer to get metrics for -vespa.consumer string default="default" - +# The Vespa metrics consumer to get metrics for +cloudWatch[].consumer string cloudWatch[].region string default="us-east-1" cloudWatch[].namespace string diff --git a/metrics-proxy/src/main/resources/templates/telegraf.conf.vm b/metrics-proxy/src/main/resources/templates/telegraf.conf.vm new file mode 100644 index 00000000000..e99bab8b02d --- /dev/null +++ b/metrics-proxy/src/main/resources/templates/telegraf.conf.vm @@ -0,0 +1,44 @@ +# Configuration for telegraf agent +[agent] + interval = "${intervalSeconds}s" + round_interval = true + metric_batch_size = 1000 + metric_buffer_limit = 10000 + collection_jitter = "0s" + flush_interval = "${intervalSeconds}s" + flush_jitter = "0s" + precision = "" + logtarget = "file" + logfile = "$logFilePath" + logfile_rotation_interval = "1d" + logfile_rotation_max_size = "20MB" + logfile_rotation_max_archives = 5 + +#foreach( $cloudwatch in $cloudwatchPlugins ) +# Configuration for AWS CloudWatch output. +[[outputs.cloudwatch]] + region = "$cloudwatch.region()" + namespace = "$cloudwatch.namespace()" +#if( $cloudwatch.accessKeyName() != "" ) + access_key = "$cloudwatch.accessKeyName()" + secret_key = "$cloudwatch.secretKeyName()" +#elseif( $cloudwatch.profile() != "" ) + profile = "$cloudwatch.profile()" +#end + tagexclude = ["vespa_consumer"] + [outputs.cloudwatch.tagpass] + vespa_consumer = ["$cloudwatch.consumer()"] + +# Configuration for Vespa input plugin +[[inputs.vespa]] + url = "http://localhost:19092/metrics/v2/values?consumer=$cloudwatch.consumer()" + [inputs.vespa.tags] + vespa_consumer = "$cloudwatch.consumer()" +#* TODO: Add node cert if hosted +#if( $isHosted ) + tls_cert = "${VESPA_CERTIFICATE_PATH}" + tls_key = "${VESPA_KEY_PATH}" + insecure_skip_verify = true +#end +*### +#end
\ No newline at end of file diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/telegraf/TelegrafTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/telegraf/TelegrafTest.java new file mode 100644 index 00000000000..9ad31a0d9e8 --- /dev/null +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/telegraf/TelegrafTest.java @@ -0,0 +1,42 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.telegraf; + +import ai.vespa.metricsproxy.TestUtil; +import org.junit.Test; + +import java.io.StringWriter; + +import static org.junit.Assert.*; + +/** + * @author olaa + */ +public class TelegrafTest { + + @Test + public void test_writing_correct_telegraf_plugin_config() { + TelegrafConfig telegrafConfig = new TelegrafConfig.Builder() + .cloudWatch( + new TelegrafConfig.CloudWatch.Builder() + .accessKeyName("accessKey1") + .namespace("namespace1") + .secretKeyName("secretKey1") + .region("us-east-1") + .consumer("consumer1") + ) + .cloudWatch( + new TelegrafConfig.CloudWatch.Builder() + .namespace("namespace2") + .profile("awsprofile") + .region("us-east-2") + .consumer("consumer2") + ) + .intervalSeconds(300) + .build(); + StringWriter stringWriter = new StringWriter(); + Telegraf.writeConfig(telegrafConfig, stringWriter); + String expectedConfig = TestUtil.getFileContents( "telegraf-config-with-two-cloudwatch-plugins.txt"); + assertEquals(expectedConfig, stringWriter.toString()); + } + +} diff --git a/metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt b/metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt new file mode 100644 index 00000000000..accd2cc87eb --- /dev/null +++ b/metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt @@ -0,0 +1,46 @@ +# Configuration for telegraf agent +[agent] + interval = "300s" + round_interval = true + metric_batch_size = 1000 + metric_buffer_limit = 10000 + collection_jitter = "0s" + flush_interval = "300s" + flush_jitter = "0s" + precision = "" + logtarget = "file" + logfile = "/opt/vespa/logs/telegraf/telegraf.log" + logfile_rotation_interval = "1d" + logfile_rotation_max_size = "20MB" + logfile_rotation_max_archives = 5 + +# Configuration for AWS CloudWatch output. +[[outputs.cloudwatch]] + region = "us-east-1" + namespace = "namespace1" + access_key = "accessKey1" + secret_key = "secretKey1" + tagexclude = ["vespa_consumer"] + [outputs.cloudwatch.tagpass] + vespa_consumer = ["consumer1"] + +# Configuration for Vespa input plugin +[[inputs.vespa]] + url = "http://localhost:19092/metrics/v2/values?consumer=consumer1" + [inputs.vespa.tags] + vespa_consumer = "consumer1" +# Configuration for AWS CloudWatch output. +[[outputs.cloudwatch]] + region = "us-east-2" + namespace = "namespace2" + profile = "awsprofile" + tagexclude = ["vespa_consumer"] + [outputs.cloudwatch.tagpass] + vespa_consumer = ["consumer2"] + +# Configuration for Vespa input plugin +[[inputs.vespa]] + url = "http://localhost:19092/metrics/v2/values?consumer=consumer2" + [inputs.vespa.tags] + vespa_consumer = "consumer2" + diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index c790e73037e..782f8592350 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -66,6 +66,9 @@ public class DockerOperationsImpl implements DockerOperations { .withHostName(context.node().hostname()) .withResources(containerResources) .withManagedBy(MANAGER_NAME) + // The inet6 option is needed to prefer AAAA records with gethostbyname(3), used by (at least) a yca package + // TODO: Try to remove this + .withDnsOption("inet6") .withUlimit("nofile", 262_144, 262_144) // The nproc aka RLIMIT_NPROC resource limit works as follows: // - A process has a (soft) nproc limit, either inherited by the parent or changed with setrlimit(2). diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 2f92ef8affe..3b6145aa6a8 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -76,6 +76,7 @@ public class NodeAgentImpl implements NodeAgent { private boolean hasResumedNode = false; private boolean hasStartedServices = true; private Optional<Instant> firstSuccessfulHealthCheckInstant = Optional.empty(); + private boolean suspendedInOrchestrator = false; private int numberOfUnhandledException = 0; private long currentRebootGeneration = 0; @@ -397,6 +398,7 @@ public class NodeAgentImpl implements NodeAgent { public void converge(NodeAgentContext context) { try { doConverge(context); + context.log(logger, LogLevel.INFO, "Converged"); } catch (ConvergenceException e) { context.log(logger, e.getMessage()); } catch (ContainerNotFoundException e) { @@ -411,7 +413,7 @@ public class NodeAgentImpl implements NodeAgent { } } - // Public for testing + // Non-private for testing void doConverge(NodeAgentContext context) { NodeSpec node = context.node(); Optional<Container> container = getContainer(context); @@ -488,8 +490,11 @@ public class NodeAgentImpl implements NodeAgent { // - Slobrok and internal orchestrator state is used to determine whether // to allow upgrade (suspend). updateNodeRepoWithCurrentAttributes(context); - context.log(logger, "Call resume against Orchestrator"); - orchestrator.resume(context.hostname().value()); + if (suspendedInOrchestrator || node.allowedToBeDown().orElse(false)) { + context.log(logger, "Call resume against Orchestrator"); + orchestrator.resume(context.hostname().value()); + suspendedInOrchestrator = false; + } break; case provisioned: nodeRepository.setNodeState(context.hostname().value(), NodeState.dirty); @@ -562,6 +567,7 @@ public class NodeAgentImpl implements NodeAgent { context.log(logger, "Ask Orchestrator for permission to suspend node"); try { orchestrator.suspend(context.hostname().value()); + suspendedInOrchestrator = true; } catch (OrchestratorException e) { // Ensure the ACLs are up to date: The reason we're unable to suspend may be because some other // node is unable to resume because the ACL rules of SOME Docker container is wrong... diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java index e2ad9e3de97..69bc9f5e092 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java @@ -181,6 +181,11 @@ public class DockerMock implements Docker { } @Override + public CreateContainerCommand withDnsOption(String dnsOption) { + return this; + } + + @Override public CreateContainerCommand withPrivileged(boolean privileged) { return this; } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index f65ad379a63..06d15a3f577 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -80,6 +80,7 @@ public class NodeAgentImplTest { .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) .wantedRestartGeneration(1).currentRestartGeneration(1) + .allowedToBeDown(false) .build(); NodeAgentContext context = createContext(node); @@ -98,7 +99,7 @@ public class NodeAgentImplTest { // TODO: Verify this isn't run unless 1st time inOrder.verify(dockerOperations, never()).startServices(eq(context)); inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context)); - inOrder.verify(orchestrator).resume(hostName); + inOrder.verify(orchestrator, never()).resume(hostName); } @Test @@ -188,7 +189,7 @@ public class NodeAgentImplTest { inOrder.verify(healthChecker, times(1)).verifyHealth(eq(context)); inOrder.verify(nodeRepository).updateNodeAttributes( hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion())); - inOrder.verify(orchestrator).resume(hostName); + inOrder.verify(orchestrator, never()).resume(hostName); } @Test @@ -224,7 +225,8 @@ public class NodeAgentImplTest { .state(NodeState.active) .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) - .wantedRestartGeneration(1).currentRestartGeneration(1); + .wantedRestartGeneration(1).currentRestartGeneration(1) + .allowedToBeDown(false); NodeAgentContext firstContext = createContext(specBuilder.build()); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); @@ -232,17 +234,20 @@ public class NodeAgentImplTest { when(dockerOperations.pullImageAsyncIfNeeded(any())).thenReturn(true); when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.of(201326592000L)); + InOrder inOrder = inOrder(orchestrator, dockerOperations); + nodeAgent.doConverge(firstContext); + inOrder.verify(orchestrator, never()).resume(any(String.class)); + NodeAgentContext secondContext = createContext(specBuilder.diskGb(200).build()); nodeAgent.doConverge(secondContext); + inOrder.verify(orchestrator, never()).resume(any(String.class)); + NodeAgentContext thirdContext = new NodeAgentContextImpl.Builder(specBuilder.vcpu(5).build()).cpuSpeedUp(1.25).build(); nodeAgent.doConverge(thirdContext); ContainerResources resourcesAfterThird = ContainerResources.from(0, 4, 16); mockGetContainer(dockerImage, resourcesAfterThird, true); - InOrder inOrder = inOrder(orchestrator, dockerOperations); - inOrder.verify(orchestrator).resume(any(String.class)); - inOrder.verify(orchestrator).resume(any(String.class)); inOrder.verify(orchestrator).suspend(any(String.class)); inOrder.verify(dockerOperations).updateContainer(eq(thirdContext), eq(resourcesAfterThird)); inOrder.verify(dockerOperations, never()).removeContainer(any(), any()); @@ -254,7 +259,7 @@ public class NodeAgentImplTest { inOrder.verify(orchestrator, never()).suspend(any(String.class)); inOrder.verify(dockerOperations, never()).updateContainer(eq(thirdContext), any()); inOrder.verify(dockerOperations, never()).removeContainer(any(), any()); - inOrder.verify(orchestrator).resume(any(String.class)); + inOrder.verify(orchestrator, never()).resume(any(String.class)); // Set the feature flag flagSource.withDoubleFlag(Flags.CONTAINER_CPU_CAP.id(), 2.3); @@ -537,6 +542,7 @@ public class NodeAgentImplTest { .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) .currentVespaVersion(vespaVersion) .wantedRestartGeneration(1).currentRestartGeneration(1) + .allowedToBeDown(true) .build(); NodeAgentContext context = createContext(node); @@ -611,6 +617,7 @@ public class NodeAgentImplTest { .type(NodeType.config) .wantedDockerImage(dockerImage) .wantedVespaVersion(vespaVersion) + .allowedToBeDown(true) .build(); NodeAgentContext context = createContext(node); @@ -689,6 +696,7 @@ public class NodeAgentImplTest { clock.advance(Duration.ofSeconds(31)); nodeAgent.doConverge(context); + inOrder.verify(orchestrator).suspend(hostName); inOrder.verify(dockerOperations).updateContainer(eq(context), eq(ContainerResources.from(0, 2, 16))); inOrder.verify(dockerOperations, never()).removeContainer(any(), any()); inOrder.verify(dockerOperations, never()).startContainer(any()); @@ -699,7 +707,7 @@ public class NodeAgentImplTest { inOrder.verify(orchestrator, never()).suspend(any(String.class)); inOrder.verify(dockerOperations, never()).updateContainer(eq(context), any()); inOrder.verify(dockerOperations, never()).removeContainer(any(), any()); - inOrder.verify(orchestrator).resume(any(String.class)); + inOrder.verify(orchestrator, never()).resume(any(String.class)); } @Test @@ -724,7 +732,7 @@ public class NodeAgentImplTest { inOrder.verify(orchestrator, never()).suspend(any(String.class)); inOrder.verify(dockerOperations, never()).updateContainer(eq(context), any()); inOrder.verify(dockerOperations, never()).removeContainer(any(), any()); - inOrder.verify(orchestrator).resume(any(String.class)); + inOrder.verify(orchestrator, never()).resume(any(String.class)); } private void verifyThatContainerIsStopped(NodeState nodeState, Optional<ApplicationId> owner) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java index 3392569d1f2..4d04409aaf0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java @@ -25,15 +25,20 @@ public class InfrastructureProvisioner extends Maintainer { this.infraDeployer = infraDeployer; } + public void maintainButThrowOnException() { + try { + infraDeployer.activateAllSupportedInfraApplications(true); + } catch (RuntimeException e) { + logger.log(LogLevel.INFO, "Failed to deploy supported infrastructure applications, " + + "will sleep 30s before propagating failure, to allow inspection of zk", + e.getMessage()); + try { Thread.sleep(30_000); } catch (InterruptedException ignored) { } + throw e; + } + } + @Override protected void maintain() { - infraDeployer.getSupportedInfraDeployments().forEach((application, deployment) -> { - try { - deployment.activate(); - } catch (RuntimeException e) { - logger.log(LogLevel.INFO, "Failed to activate " + application, e); - // loop around to activate the next application - } - }); + infraDeployer.activateAllSupportedInfraApplications(false); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index f265feac788..ecc550527fc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -94,7 +94,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { autoscalingMaintainer = new AutoscalingMaintainer(nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), nodeMetricsDb, deployer, defaults.autoscalingInterval); // The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now - infrastructureProvisioner.maintain(); + infrastructureProvisioner.maintainButThrowOnException(); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java index a56ace07c82..1086a3a7cd9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java @@ -20,7 +20,6 @@ import com.yahoo.vespa.service.monitor.DuperModelInfraApi; import com.yahoo.vespa.service.monitor.InfraApplicationApi; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -51,9 +50,22 @@ public class InfraDeployerImpl implements InfraDeployer { } @Override - public Map<ApplicationId, Deployment> getSupportedInfraDeployments() { - return duperModel.getSupportedInfraApplications().stream() - .collect(Collectors.toMap(InfraApplicationApi::getApplicationId, InfraDeployment::new)); + public void activateAllSupportedInfraApplications(boolean propagateException) { + duperModel.getSupportedInfraApplications().forEach(api -> { + var application = api.getApplicationId(); + var deployment = new InfraDeployment(api); + try { + deployment.activate(); + } catch (RuntimeException e) { + logger.log(LogLevel.INFO, "Failed to activate " + application, e); + if (propagateException) { + throw e; + } + // loop around to activate the next application + } + }); + + duperModel.infraApplicationsIsNowComplete(); } private class InfraDeployment implements Deployment { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java index ef3d1995df9..e7ebf049e51 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java @@ -50,4 +50,8 @@ public class MockDuperModel implements DuperModelInfraApi { public void infraApplicationRemoved(ApplicationId applicationId) { activeApps.remove(applicationId); } + + @Override + public void infraApplicationsIsNowComplete() { + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java index e7bf76986ca..742a863fb38 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Deployment; import com.yahoo.config.provision.InfraDeployer; -import java.util.Map; import java.util.Optional; public class MockInfraDeployer implements InfraDeployer { @@ -15,7 +14,5 @@ public class MockInfraDeployer implements InfraDeployer { } @Override - public Map<ApplicationId, Deployment> getSupportedInfraDeployments() { - return Map.of(); - } + public void activateAllSupportedInfraApplications(boolean propagateException) { } } diff --git a/orchestrator/pom.xml b/orchestrator/pom.xml index 5dd4e7ea87d..fed0e617c79 100644 --- a/orchestrator/pom.xml +++ b/orchestrator/pom.xml @@ -81,12 +81,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.easytesting</groupId> - <artifactId>fest-assert</artifactId> - <version>1.4</version> - <scope>test</scope> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>config-provisioning</artifactId> <version>${project.version}</version> diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java index eb6a4119f8a..af59eb11e9b 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java @@ -38,9 +38,11 @@ public class OrchestratorContext implements AutoCloseable { private final HashMap<ApplicationInstanceReference, Runnable> locks = new HashMap<>(); /** Create an OrchestratorContext for operations on multiple applications. */ - public static OrchestratorContext createContextForMultiAppOp(Clock clock, boolean largeLocks) { + public static OrchestratorContext createContextForMultiAppOp(Clock clock) { return new OrchestratorContext(null, clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_BATCH_OP), - false, largeLocks, false); + false, // probe + true, // large locks + false); // use permanently down status } /** Create an OrchestratorContext for an operation on a single application. */ diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java index fbe6864274c..d0062966a6d 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java @@ -64,7 +64,6 @@ public class OrchestratorImpl implements Orchestrator { private final ClusterControllerClientFactory clusterControllerClientFactory; private final Clock clock; private final ApplicationApiFactory applicationApiFactory; - private final BooleanFlag enableLargeOrchestratorLocks; private final BooleanFlag retireWithPermanentlyDownFlag; @Inject @@ -101,7 +100,6 @@ public class OrchestratorImpl implements Orchestrator { this.instanceLookupService = instanceLookupService; this.clock = clock; this.applicationApiFactory = applicationApiFactory; - this.enableLargeOrchestratorLocks = Flags.ENABLE_LARGE_ORCHESTRATOR_LOCKS.bindTo(flagSource); this.retireWithPermanentlyDownFlag = Flags.RETIRE_WITH_PERMANENTLY_DOWN.bindTo(flagSource); } @@ -167,7 +165,7 @@ public class OrchestratorImpl implements Orchestrator { OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock); try (MutableStatusRegistry statusRegistry = statusService .lockApplicationInstance_forCurrentThreadOnly(context, appInstance.reference())) { - HostStatus currentHostState = statusRegistry.getHostInfo(hostName).status(); + HostStatus currentHostState = statusRegistry.getHostInfos().getOrNoRemarks(hostName).status(); if (currentHostState == HostStatus.NO_REMARKS) { return; } @@ -258,10 +256,7 @@ public class OrchestratorImpl implements Orchestrator { @Override public void suspendAll(HostName parentHostname, List<HostName> hostNames) throws BatchHostStateChangeDeniedException, BatchHostNameNotFoundException, BatchInternalErrorException { - boolean largeLocks = enableLargeOrchestratorLocks - .with(FetchVector.Dimension.HOSTNAME, parentHostname.s()) - .value(); - try (OrchestratorContext context = OrchestratorContext.createContextForMultiAppOp(clock, largeLocks)) { + try (OrchestratorContext context = OrchestratorContext.createContextForMultiAppOp(clock)) { List<NodeGroup> nodeGroupsOrderedByApplication; try { nodeGroupsOrderedByApplication = nodeGroupsOrderedForSuspend(hostNames); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java index e9bb4984c2e..65c45c8df76 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java @@ -26,7 +26,6 @@ public interface ClusterApi { Optional<StorageNode> storageNodeInGroup(); Optional<StorageNode> upStorageNodeInGroup(); - String servicesDownAndNotInGroupDescription(); - String nodesAllowedToBeDownNotInGroupDescription(); + String downDescription(); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java index b747d8c2e22..24f56eac85d 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java @@ -131,25 +131,56 @@ class ClusterApiImpl implements ClusterApi { return numberOfServicesDown * 100 / (serviceCluster.serviceInstances().size() + missingServices); } + /** + * A description of the hosts outside the group that are allowed to be down, + * and a description of the services outside the group and outside of the allowed services + * that are down. + */ @Override - public String servicesDownAndNotInGroupDescription() { - // Sort these for readability and testing stability - return Stream - .concat(servicesDownAndNotInGroup.stream().map(ServiceInstance::toString).sorted(), - missingServices > 0 ? Stream.of(descriptionOfMissingServices) : Stream.of()) - .collect(Collectors.toList()) - .toString(); - } + public String downDescription() { + StringBuilder description = new StringBuilder(); - @Override - public String nodesAllowedToBeDownNotInGroupDescription() { - return servicesNotInGroup.stream() + Set<HostName> suspended = servicesNotInGroup.stream() .map(ServiceInstance::hostName) .filter(hostName -> hostStatus(hostName).isSuspended()) - .sorted() - .distinct() - .collect(Collectors.toList()) - .toString(); + .collect(Collectors.toSet()); + + if (suspended.size() > 0) { + description.append(" "); + + final int nodeLimit = 3; + description.append("Suspended hosts: "); + description.append(suspended.stream().sorted().distinct().limit(nodeLimit).collect(Collectors.toList()).toString()); + if (suspended.size() > nodeLimit) { + description.append(", and " + (suspended.size() - nodeLimit) + " more"); + } + description.append("."); + } + + Set<ServiceInstance> downElsewhere = servicesDownAndNotInGroup.stream() + .filter(serviceInstance -> !suspended.contains(serviceInstance.hostName())) + .collect(Collectors.toSet()); + + final int downElsewhereTotal = downElsewhere.size() + missingServices; + if (downElsewhereTotal > 0) { + description.append(" "); + + final int serviceLimit = 2; // services info is verbose + description.append("Services down on resumed hosts: "); + description.append(Stream.concat( + downElsewhere.stream().map(ServiceInstance::toString).sorted(), + missingServices > 0 ? Stream.of(descriptionOfMissingServices) : Stream.of()) + .limit(serviceLimit) + .collect(Collectors.toList()) + .toString()); + + if (downElsewhereTotal > serviceLimit) { + description.append(", and " + (downElsewhereTotal - serviceLimit) + " more"); + } + description.append("."); + } + + return description.toString(); } private Optional<StorageNode> storageNodeInGroup(Predicate<ServiceInstance> storageServicePredicate) { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java index e13cf17d420..a6b3cbc87dc 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java @@ -2,55 +2,43 @@ package com.yahoo.vespa.orchestrator.policy; import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.orchestrator.OrchestrationException; import com.yahoo.vespa.orchestrator.model.NodeGroup; -import java.util.Optional; - /** * @author bakksjo */ public class HostStateChangeDeniedException extends OrchestrationException { private final String constraintName; - private final Optional<ServiceType> serviceType; public HostStateChangeDeniedException(HostName hostName, String constraintName, String message) { this(hostName, constraintName, message, null); } public HostStateChangeDeniedException(HostName hostName, String constraintName, String message, Exception e) { - this(hostName.s(), constraintName, Optional.empty(), message, e); + this(hostName.s(), constraintName, message, e); } public HostStateChangeDeniedException(NodeGroup nodeGroup, String constraintName, String message) { - this(nodeGroup.toCommaSeparatedString(), constraintName, Optional.empty(), message, null); + this(nodeGroup.toCommaSeparatedString(), constraintName, message, null); } private HostStateChangeDeniedException(String nodes, String constraintName, - Optional<ServiceType> serviceType, String message, Throwable cause) { - super(createMessage(nodes, constraintName, serviceType, message), cause); + super(createMessage(nodes, constraintName, message), cause); this.constraintName = constraintName; - this.serviceType = serviceType; } private static String createMessage(String nodes, String constraintName, - Optional<ServiceType> serviceType, String message) { return "Changing the state of " + nodes + " would violate " + constraintName - + (serviceType.isPresent() ? " for service type " + serviceType.get() : "") + ": " + message; } public String getConstraintName() { return constraintName; } - - public Optional<ServiceType> getServiceType() { - return serviceType; - } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java index 1e895d0e757..ccb0bb57186 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java @@ -26,13 +26,11 @@ public class HostedVespaClusterPolicy implements ClusterPolicy { throw new HostStateChangeDeniedException( clusterApi.getNodeGroup(), ENOUGH_SERVICES_UP_CONSTRAINT, - "Suspension percentage for service type " + clusterApi.serviceType() + "Suspension for service type " + clusterApi.serviceType() + " would increase from " + clusterApi.percentageOfServicesDown() + "% to " + clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() + "%, over the limit of " + percentageOfServicesAllowedToBeDown + "%." - + " These instances may be down: " + clusterApi.servicesDownAndNotInGroupDescription() - + " and these hosts are allowed to be down: " - + clusterApi.nodesAllowedToBeDownNotInGroupDescription()); + + clusterApi.downDescription()); } @Override @@ -56,9 +54,7 @@ public class HostedVespaClusterPolicy implements ClusterPolicy { "Down percentage for service type " + clusterApi.serviceType() + " would increase to " + clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() + "%, over the limit of " + percentageOfServicesAllowedToBeDown + "%." - + " These instances may be down: " + clusterApi.servicesDownAndNotInGroupDescription() - + " and these hosts are allowed to be down: " - + clusterApi.nodesAllowedToBeDownNotInGroupDescription()); + + clusterApi.downDescription()); } // Non-private for testing purposes diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java index fc5c5eb5004..c93bb9e1d4a 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java @@ -157,8 +157,9 @@ public class HostResource implements HostApi { private static WebApplicationException webExceptionFromTimeout(String operationDescription, HostName hostName, UncheckedTimeoutException e) { - return createWebException(operationDescription, hostName, e, HostedVespaPolicy.DEADLINE_CONSTRAINT, e.getMessage(), - Response.Status.GATEWAY_TIMEOUT); + // Return timeouts as 409 Conflict instead of 504 Gateway Timeout to reduce noise in 5xx graphs. + return createWebException(operationDescription, hostName, e, + HostedVespaPolicy.DEADLINE_CONSTRAINT, e.getMessage(), Response.Status.CONFLICT); } private static WebApplicationException webExceptionWithDenialReason( diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java index 79e0fc0f3e9..6e857563f9b 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java @@ -18,7 +18,6 @@ import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.util.List; -import java.util.concurrent.TimeoutException; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -48,7 +47,7 @@ public class HostSuspensionResource implements HostSuspensionApi { throw createWebApplicationException(e.getMessage(), Response.Status.CONFLICT); } catch (UncheckedTimeoutException e) { log.log(LogLevel.DEBUG, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); - throw createWebApplicationException(e.getMessage(), Response.Status.GATEWAY_TIMEOUT); + throw createWebApplicationException(e.getMessage(), Response.Status.CONFLICT); } catch (BatchHostNameNotFoundException e) { log.log(LogLevel.DEBUG, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); // Note that we're returning BAD_REQUEST instead of NOT_FOUND because the resource identified diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java index d2042dc9fd2..24da83364aa 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java @@ -17,9 +17,6 @@ public interface MutableStatusRegistry extends AutoCloseable { /** Returns the status of this application. */ ApplicationInstanceStatus getStatus(); - /** Returns the host info of the given host. */ - HostInfo getHostInfo(HostName hostName); - /** Returns a snapshot of all host infos for this application. */ HostInfos getHostInfos(); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java index b4025167187..2cdce350377 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java @@ -371,11 +371,6 @@ public class ZookeeperStatusService implements StatusService { } @Override - public HostInfo getHostInfo(HostName hostName) { - return ZookeeperStatusService.this.getHostInfo(applicationInstanceReference, hostName); - } - - @Override public HostInfos getHostInfos() { return hostInfosCache.getHostInfos(applicationInstanceReference); } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorContextTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorContextTest.java index 607894ee104..16b66b2804e 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorContextTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorContextTest.java @@ -23,7 +23,7 @@ public class OrchestratorContextTest { var mutable = new Object() { boolean locked = true; }; Runnable unlock = () -> mutable.locked = false; - try (OrchestratorContext rootContext = OrchestratorContext.createContextForMultiAppOp(new ManualClock(), true)) { + try (OrchestratorContext rootContext = OrchestratorContext.createContextForMultiAppOp(new ManualClock())) { try (OrchestratorContext probeContext = rootContext.createSubcontextForSingleAppOp(true)) { assertFalse(probeContext.hasLock(application)); assertTrue(probeContext.registerLockAcquisition(application, unlock)); @@ -41,19 +41,4 @@ public class OrchestratorContextTest { } assertFalse(mutable.locked); } - - @Test - public void testLargeLocksDisabled() { - var mutable = new Object() { boolean locked = true; }; - Runnable unlock = () -> mutable.locked = false; - - try (OrchestratorContext rootContext = OrchestratorContext.createContextForMultiAppOp(new ManualClock(), false)) { - try (OrchestratorContext probeContext = rootContext.createSubcontextForSingleAppOp(true)) { - assertFalse(probeContext.hasLock(application)); - assertFalse(probeContext.registerLockAcquisition(application, unlock)); - } - } - - assertTrue(mutable.locked); - } }
\ No newline at end of file diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java index 77ec824da54..bfe4b523e4a 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java @@ -17,7 +17,6 @@ import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock; @@ -325,8 +324,6 @@ public class OrchestratorImplTest { @Test public void testLargeLocks() throws Exception { - flagSource.withBooleanFlag(Flags.ENABLE_LARGE_ORCHESTRATOR_LOCKS.id(), true); - var tenantId = new TenantId("tenant"); var applicationInstanceId = new ApplicationInstanceId("app:dev:us-east-1:default"); var applicationInstanceReference = new ApplicationInstanceReference(tenantId, applicationInstanceId); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java index 62925dc003e..a5cb5cfa630 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; -import java.util.HashMap; import java.util.List; import java.util.Optional; import java.util.Set; @@ -81,15 +80,10 @@ public class ClusterApiImplTest { assertEquals("{ clusterId=cluster, serviceType=service-type }", clusterApi.clusterInfo()); assertFalse(clusterApi.isStorageCluster()); - assertEquals("[ServiceInstance{configId=service-2, hostName=host2, serviceStatus=" + - "ServiceStatusInfo{status=DOWN, since=Optional.empty, lastChecked=Optional.empty}}, " - + "ServiceInstance{configId=service-3, hostName=host3, serviceStatus=" + - "ServiceStatusInfo{status=UP, since=Optional.empty, lastChecked=Optional.empty}}, " - + "ServiceInstance{configId=service-4, hostName=host4, serviceStatus=" + - "ServiceStatusInfo{status=DOWN, since=Optional.empty, lastChecked=Optional.empty}}]", - clusterApi.servicesDownAndNotInGroupDescription()); - assertEquals("[host3, host4]", - clusterApi.nodesAllowedToBeDownNotInGroupDescription()); + assertEquals(" Suspended hosts: [host3, host4]. Services down on resumed hosts: [" + + "ServiceInstance{configId=service-2, hostName=host2, serviceStatus=" + + "ServiceStatusInfo{status=DOWN, since=Optional.empty, lastChecked=Optional.empty}}].", + clusterApi.downDescription()); assertEquals(60, clusterApi.percentageOfServicesDown()); assertEquals(80, clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()); } @@ -110,9 +104,9 @@ public class ClusterApiImplTest { fail(); } catch (HostStateChangeDeniedException e) { assertThat(e.getMessage(), - containsString("Changing the state of cfg1 would violate enough-services-up: Suspension percentage " + - "for service type configserver would increase from 33% to 66%, over the limit of 10%. " + - "These instances may be down: [1 missing config server] and these hosts are allowed to be down: []")); + containsString("Changing the state of cfg1 would violate enough-services-up: " + + "Suspension for service type configserver would increase from 33% to 66%, " + + "over the limit of 10%. Services down on resumed hosts: [1 missing config server].")); } } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java index d834034c9a8..4462e886d1b 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java @@ -101,8 +101,7 @@ public class HostedVespaClusterPolicyTest { when(clusterApi.serviceType()).thenReturn(new ServiceType("service-type")); when(clusterApi.percentageOfServicesDown()).thenReturn(5); when(clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()).thenReturn(percentageOfServicesDownIfGroupIsAllowedToBeDown); - when(clusterApi.servicesDownAndNotInGroupDescription()).thenReturn("services-down-and-not-in-group"); - when(clusterApi.nodesAllowedToBeDownNotInGroupDescription()).thenReturn("allowed-to-be-down"); + when(clusterApi.downDescription()).thenReturn(" Down description"); NodeGroup nodeGroup = mock(NodeGroup.class); when(clusterApi.getNodeGroup()).thenReturn(nodeGroup); @@ -116,11 +115,9 @@ public class HostedVespaClusterPolicyTest { } } catch (HostStateChangeDeniedException e) { if (!expectSuccess) { - assertEquals("Changing the state of node-group would violate enough-services-up: " - + "Suspension percentage for service type service-type would increase from " - + "5% to 13%, over the limit of 10%. These instances may be down: " - + "services-down-and-not-in-group and these hosts are allowed to be down: " - + "allowed-to-be-down", e.getMessage()); + assertEquals("Changing the state of node-group would violate enough-services-up: " + + "Suspension for service type service-type would increase from 5% to 13%, " + + "over the limit of 10%. Down description", e.getMessage()); assertEquals("enough-services-up", e.getConstraintName()); } } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java index dc26c1a3770..bfa68145828 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java @@ -60,9 +60,9 @@ import java.util.Optional; import java.util.Set; import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceClusterSet; -import static org.fest.assertions.Assertions.assertThat; -import static org.fest.assertions.Fail.fail; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -167,7 +167,7 @@ public class HostResourceTest { UpdateHostResponse response = hostResource.suspend(hostName); - assertThat(response.hostname()).isEqualTo(hostName); + assertEquals(hostName, response.hostname()); } @Test @@ -175,14 +175,14 @@ public class HostResourceTest { HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(alwaysAllowOrchestrator); BatchOperationResult response = hostSuspensionResource.suspendAll("parentHostname", Arrays.asList("hostname1", "hostname2")); - assertThat(response.success()); + assertTrue(response.success()); } @Test public void returns_200_empty_batch() { HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(alwaysAllowOrchestrator); BatchOperationResult response = hostSuspensionResource.suspendAll("parentHostname", List.of()); - assertThat(response.success()); + assertTrue(response.success()); } @Test @@ -193,7 +193,7 @@ public class HostResourceTest { hostResource.suspend("hostname"); fail(); } catch (WebApplicationException w) { - assertThat(w.getResponse().getStatus()).isEqualTo(404); + assertEquals(404, w.getResponse().getStatus()); } } @@ -207,7 +207,7 @@ public class HostResourceTest { hostSuspensionResource.suspendAll("parentHostname", Arrays.asList("hostname1", "hostname2")); fail(); } catch (WebApplicationException w) { - assertThat(w.getResponse().getStatus()).isEqualTo(400); + assertEquals(400, w.getResponse().getStatus()); } } @@ -259,7 +259,7 @@ public class HostResourceTest { hostResource.suspend("hostname"); fail(); } catch (WebApplicationException w) { - assertThat(w.getResponse().getStatus()).isEqualTo(409); + assertEquals(409, w.getResponse().getStatus()); } } @@ -280,7 +280,7 @@ public class HostResourceTest { hostSuspensionResource.suspendAll("parentHostname", Arrays.asList("hostname1", "hostname2")); fail(); } catch (WebApplicationException w) { - assertThat(w.getResponse().getStatus()).isEqualTo(409); + assertEquals(409, w.getResponse().getStatus()); } } @@ -368,7 +368,7 @@ public class HostResourceTest { } @Test - public void throws_504_on_timeout() throws HostNameNotFoundException, HostStateChangeDeniedException { + public void throws_409_on_timeout() throws HostNameNotFoundException, HostStateChangeDeniedException { Orchestrator orchestrator = mock(Orchestrator.class); doThrow(new UncheckedTimeoutException("Timeout Message")).when(orchestrator).resume(any(HostName.class)); @@ -377,13 +377,13 @@ public class HostResourceTest { hostResource.resume("hostname"); fail(); } catch (WebApplicationException w) { - assertThat(w.getResponse().getStatus()).isEqualTo(504); + assertEquals(409, w.getResponse().getStatus()); assertEquals("resume failed: Timeout Message [deadline]", w.getMessage()); } } @Test - public void throws_504_on_suspendAll_timeout() throws BatchHostStateChangeDeniedException, BatchHostNameNotFoundException, BatchInternalErrorException { + public void throws_409_on_suspendAll_timeout() throws BatchHostStateChangeDeniedException, BatchHostNameNotFoundException, BatchInternalErrorException { Orchestrator orchestrator = mock(Orchestrator.class); doThrow(new UncheckedTimeoutException("Timeout Message")).when(orchestrator).suspendAll(any(), any()); @@ -392,7 +392,7 @@ public class HostResourceTest { resource.suspendAll("parenthost", Arrays.asList("h1", "h2", "h3")); fail(); } catch (WebApplicationException w) { - assertThat(w.getResponse().getStatus()).isEqualTo(504); + assertEquals(409, w.getResponse().getStatus()); } } } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java index 12622f22837..687ea951f88 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java @@ -117,7 +117,7 @@ public class ZookeeperStatusServiceTest { TestIds.HOST_NAME1, hostStatus); - assertThat(statusRegistry.getHostInfo(TestIds.HOST_NAME1).status(), + assertThat(statusRegistry.getHostInfos().getOrNoRemarks(TestIds.HOST_NAME1).status(), is(hostStatus)); } } @@ -182,7 +182,7 @@ public class ZookeeperStatusServiceTest { killSession(curator.framework(), testingServer); //Throws SessionFailedException if the SessionFailRetryLoop has not been closed. - statusRegistry.getHostInfo(TestIds.HOST_NAME1); + statusRegistry.getHostInfos().getOrNoRemarks(TestIds.HOST_NAME1); }); assertThat(resultOfZkOperationAfterLockFailure, notHoldsException()); diff --git a/parent/pom.xml b/parent/pom.xml index b1ca2539ef5..68ee698e7a5 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -735,8 +735,8 @@ <properties> <antlr.version>3.5.2</antlr.version> <antlr4.version>4.5</antlr4.version> - <apache.httpclient.version>4.5.10</apache.httpclient.version> - <apache.httpcore.version>4.4.12</apache.httpcore.version> + <apache.httpclient.version>4.5.11</apache.httpclient.version> + <apache.httpcore.version>4.4.13</apache.httpcore.version> <asm.version>7.0</asm.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> <athenz.version>1.8.44</athenz.version> diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.cpp b/searchcommon/src/vespa/searchcommon/attribute/config.cpp index 53e57fd9c66..b4e05875820 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/config.cpp +++ b/searchcommon/src/vespa/searchcommon/attribute/config.cpp @@ -17,7 +17,8 @@ Config::Config() : _growStrategy(), _compactionStrategy(), _predicateParams(), - _tensorType(vespalib::eval::ValueType::error_type()) + _tensorType(vespalib::eval::ValueType::error_type()), + _hnsw_index_params() { } @@ -34,7 +35,8 @@ Config::Config(BasicType bt, CollectionType ct, bool fastSearch_, bool huge_) _growStrategy(), _compactionStrategy(), _predicateParams(), - _tensorType(vespalib::eval::ValueType::error_type()) + _tensorType(vespalib::eval::ValueType::error_type()), + _hnsw_index_params() { } @@ -60,7 +62,8 @@ Config::operator==(const Config &b) const _compactionStrategy == b._compactionStrategy && _predicateParams == b._predicateParams && (_basicType.type() != BasicType::Type::TENSOR || - _tensorType == b._tensorType); + _tensorType == b._tensorType) && + _hnsw_index_params == b._hnsw_index_params; } } diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.h b/searchcommon/src/vespa/searchcommon/attribute/config.h index 2f767061f7a..836fcfed84a 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/config.h +++ b/searchcommon/src/vespa/searchcommon/attribute/config.h @@ -4,15 +4,21 @@ #include "basictype.h" #include "collectiontype.h" +#include "hnsw_index_params.h" #include "predicate_params.h" -#include <vespa/searchcommon/common/growstrategy.h> #include <vespa/searchcommon/common/compaction_strategy.h> +#include <vespa/searchcommon/common/growstrategy.h> #include <vespa/eval/eval/value_type.h> +#include <optional> namespace search::attribute { -class Config -{ +/** + * Configuration for an attribute vector. + * + * Used to determine which implementation to instantiate. + */ +class Config { public: Config(); Config(BasicType bt, CollectionType ct = CollectionType::SINGLE, @@ -29,6 +35,7 @@ public: bool huge() const { return _huge; } const PredicateParams &predicateParams() const { return _predicateParams; } vespalib::eval::ValueType tensorType() const { return _tensorType; } + const std::optional<HnswIndexParams>& hnsw_index_params() const { return _hnsw_index_params; } /** * Check if attribute posting list can consist of a bitvector in @@ -60,6 +67,10 @@ public: _tensorType = tensorType_in; return *this; } + Config& set_hnsw_index_params(const HnswIndexParams& params) { + _hnsw_index_params = params; + return *this; + } /** * Enable attribute posting list to consist of a bitvector in @@ -107,6 +118,7 @@ private: CompactionStrategy _compactionStrategy; PredicateParams _predicateParams; vespalib::eval::ValueType _tensorType; + std::optional<HnswIndexParams> _hnsw_index_params; }; } diff --git a/searchcommon/src/vespa/searchcommon/attribute/hnsw_index_params.h b/searchcommon/src/vespa/searchcommon/attribute/hnsw_index_params.h new file mode 100644 index 00000000000..9e98a8c5fb7 --- /dev/null +++ b/searchcommon/src/vespa/searchcommon/attribute/hnsw_index_params.h @@ -0,0 +1,32 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace search::attribute { + +/** + * Configuration parameters for a hnsw index used together with a 1-dimensional indexed tensor + * for approximate nearest neighbor search. + */ +class HnswIndexParams { +private: + uint32_t _max_links_per_node; + uint32_t _neighbors_to_explore_at_insert; + +public: + HnswIndexParams(uint32_t max_links_per_node_in, + uint32_t neighbors_to_explore_at_insert_in) + : _max_links_per_node(max_links_per_node_in), + _neighbors_to_explore_at_insert(neighbors_to_explore_at_insert_in) + {} + + uint32_t max_links_per_node() const { return _max_links_per_node; } + uint32_t neighbors_to_explore_at_insert() const { return _neighbors_to_explore_at_insert; } + + bool operator==(const HnswIndexParams& rhs) const { + return _max_links_per_node == rhs._max_links_per_node && + _neighbors_to_explore_at_insert == rhs._neighbors_to_explore_at_insert; + } +}; + +} diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index 3c3ed33b8ea..14ad82a0aa1 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -208,10 +208,10 @@ Fixture::initViewSet(ViewSet &views) views._lidReuseDelayer = std::make_unique<documentmetastore::LidReuseDelayer>(views._writeService, metaStore->get()); IndexSearchable::SP indexSearchable; MatchView::SP matchView(new MatchView(matchers, indexSearchable, attrMgr, sesMgr, metaStore, views._docIdLimit)); - views.searchView.set(make_shared<SearchView> + views.searchView.set(SearchView::create (summaryMgr->createSummarySetup(SummaryConfig(), SummarymapConfig(), JuniperrcConfig(), views.repo, attrMgr), - matchView)); + std::move(matchView))); views.feedView.set( make_shared<SearchableFeedView>(StoreOnlyFeedView::Context(summaryAdapter, schema, diff --git a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp index 340619f09bd..37ac5f0d65c 100644 --- a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp +++ b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp @@ -1,12 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("threading_service_config_test"); #include <vespa/searchcore/config/config-proton.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/server/threading_service_config.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/log/log.h> +LOG_SETUP("threading_service_config_test"); + using namespace proton; using ProtonConfig = vespa::config::search::core::ProtonConfig; using ProtonConfigBuilder = vespa::config::search::core::ProtonConfigBuilder; @@ -42,6 +43,7 @@ TEST_F("require that indexing threads are set based on cpu cores and feeding con TEST_DO(f.assertIndexingThreads(3, 18)); TEST_DO(f.assertIndexingThreads(4, 19)); TEST_DO(f.assertIndexingThreads(4, 24)); + TEST_DO(f.assertIndexingThreads(4, 64)); // Ensure it is capped at 4 } TEST_F("require that indexing threads is always >= 1", Fixture(0)) diff --git a/searchcore/src/tests/proton/matchengine/matchengine.cpp b/searchcore/src/tests/proton/matchengine/matchengine.cpp index 9ae938981a4..fd48fffcb5c 100644 --- a/searchcore/src/tests/proton/matchengine/matchengine.cpp +++ b/searchcore/src/tests/proton/matchengine/matchengine.cpp @@ -18,18 +18,17 @@ class MySearchHandler : public ISearchHandler { std::string _reply; public: MySearchHandler(size_t numHits = 0) : - _numHits(numHits), _name("my"), _reply("myreply") {} - virtual DocsumReply::UP getDocsums(const DocsumRequest &) override { - return DocsumReply::UP(new DocsumReply); + _numHits(numHits), _name("my"), _reply("myreply") + {} + DocsumReply::UP getDocsums(const DocsumRequest &) override { + return std::make_unique<DocsumReply>(); } - virtual search::engine::SearchReply::UP match( - const ISearchHandler::SP &, - const search::engine::SearchRequest &, - vespalib::ThreadBundle &) const override { - SearchReply::UP retval(new SearchReply); + SearchReply::UP match(const SearchRequest &, vespalib::ThreadBundle &) const override + { + auto retval = std::make_unique<SearchReply>(); for (size_t i = 0; i < _numHits; ++i) { - retval->hits.push_back(SearchReply::Hit()); + retval->hits.emplace_back(); } return retval; } @@ -43,7 +42,7 @@ private: public: LocalSearchClient(); - ~LocalSearchClient(); + ~LocalSearchClient() override; void searchDone(SearchReply::UP reply) override { std::lock_guard<std::mutex> guard(_lock); _reply = std::move(reply); @@ -62,8 +61,8 @@ public: } }; -LocalSearchClient::LocalSearchClient() {} -LocalSearchClient::~LocalSearchClient() {} +LocalSearchClient::LocalSearchClient() = default; +LocalSearchClient::~LocalSearchClient() = default; TEST("requireThatSearchesExecute") { @@ -71,23 +70,23 @@ TEST("requireThatSearchesExecute") MatchEngine engine(numMatcherThreads, 1, 7); engine.setNodeUp(true); - MySearchHandler::SP handler(new MySearchHandler); + auto handler = std::make_shared<MySearchHandler>(); DocTypeName dtnvfoo("foo"); engine.putSearchHandler(dtnvfoo, handler); LocalSearchClient client; SearchRequest::Source request(new SearchRequest()); SearchReply::UP reply = engine.search(std::move(request), client); - EXPECT_TRUE(reply.get() == NULL); + EXPECT_FALSE(reply); reply = client.getReply(10000); - EXPECT_TRUE(reply.get() != NULL); + EXPECT_TRUE(reply); } bool assertSearchReply(MatchEngine & engine, const std::string & searchDocType, size_t expHits) { - SearchRequest *request = new SearchRequest(); + auto *request = new SearchRequest(); request->propertiesMap.lookupCreate(search::MapNames::MATCH).add("documentdb.searchdoctype", searchDocType); LocalSearchClient client; engine.search(SearchRequest::Source(request), client); @@ -99,9 +98,9 @@ TEST("requireThatCorrectHandlerIsUsed") { MatchEngine engine(1, 1, 7); engine.setNodeUp(true); - ISearchHandler::SP h1(new MySearchHandler(2)); - ISearchHandler::SP h2(new MySearchHandler(4)); - ISearchHandler::SP h3(new MySearchHandler(6)); + auto h1 = std::make_shared<MySearchHandler>(2); + auto h2 = std::make_shared<MySearchHandler>(4); + auto h3 = std::make_shared<MySearchHandler>(6); DocTypeName dtnvfoo("foo"); DocTypeName dtnvbar("bar"); DocTypeName dtnvbaz("baz"); @@ -120,13 +119,12 @@ struct ObserveBundleMatchHandler : MySearchHandler { mutable size_t bundleSize; ObserveBundleMatchHandler() : bundleSize(0) {} - virtual search::engine::SearchReply::UP match( - const ISearchHandler::SP &, + search::engine::SearchReply::UP match( const search::engine::SearchRequest &, vespalib::ThreadBundle &threadBundle) const override { bundleSize = threadBundle.size(); - return SearchReply::UP(new SearchReply); + return std::make_unique<SearchReply>(); } }; @@ -135,7 +133,7 @@ TEST("requireThatBundlesAreUsed") MatchEngine engine(15, 5, 7); engine.setNodeUp(true); - ObserveBundleMatchHandler::SP handler(new ObserveBundleMatchHandler()); + auto handler = std::make_shared<ObserveBundleMatchHandler>(); DocTypeName dtnvfoo("foo"); engine.putSearchHandler(dtnvfoo, handler); @@ -151,20 +149,20 @@ TEST("requireThatHandlersCanBeRemoved") { MatchEngine engine(1, 1, 7); engine.setNodeUp(true); - ISearchHandler::SP h(new MySearchHandler(1)); + auto h = std::make_shared<MySearchHandler>(1); DocTypeName docType("foo"); engine.putSearchHandler(docType, h); ISearchHandler::SP r = engine.getSearchHandler(docType); - EXPECT_TRUE(r.get() != NULL); + EXPECT_TRUE(r); EXPECT_TRUE(h.get() == r.get()); r = engine.removeSearchHandler(docType); - EXPECT_TRUE(r.get() != NULL); + EXPECT_TRUE(r); EXPECT_TRUE(h.get() == r.get()); r = engine.getSearchHandler(docType); - EXPECT_TRUE(r.get() == NULL); + EXPECT_FALSE(r); } TEST("requireThatEmptySearchReplyIsReturnedWhenEngineIsClosed") @@ -175,7 +173,7 @@ TEST("requireThatEmptySearchReplyIsReturnedWhenEngineIsClosed") LocalSearchClient client; SearchRequest::Source request(new SearchRequest()); SearchReply::UP reply = engine.search(std::move(request), client); - EXPECT_TRUE(reply.get() != NULL); + EXPECT_TRUE(reply ); EXPECT_EQUAL(0u, reply->hits.size()); EXPECT_EQUAL(7u, reply->getDistributionKey()); } diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index 95ab43dbcba..00a319db394 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -289,7 +289,7 @@ struct MyWorld { DocsumReply::UP getDocsums(const DocsumRequest &) override { return DocsumReply::UP(); } - SearchReply::UP match(const ISearchHandler::SP &, const SearchRequest &, vespalib::ThreadBundle &) const override { + SearchReply::UP match(const SearchRequest &, vespalib::ThreadBundle &) const override { return SearchReply::UP(); } }; diff --git a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp index a9cae7d8ab7..7cdd8d767c6 100644 --- a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp +++ b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp @@ -42,7 +42,7 @@ public: : _name(name), _reply(reply) {} - virtual DocsumReply::UP getDocsums(const DocsumRequest &request) override { + DocsumReply::UP getDocsums(const DocsumRequest &request) override { return (request.useRootSlime()) ? std::make_unique<DocsumReply>(createSlimeReply(request.hits.size())) : createOldDocSum(request); @@ -62,7 +62,7 @@ public: } DocsumReply::UP createOldDocSum(const DocsumRequest &request) { - DocsumReply::UP retval(new DocsumReply()); + auto retval = std::make_unique<DocsumReply>(); for (size_t i = 0; i < request.hits.size(); i++) { const DocsumRequest::Hit &h = request.hits[i]; DocsumReply::Docsum docsum; @@ -74,11 +74,8 @@ public: return retval; } - virtual search::engine::SearchReply::UP match( - const ISearchHandler::SP &, - const search::engine::SearchRequest &, - vespalib::ThreadBundle &) const override { - return SearchReply::UP(new SearchReply); + SearchReply::UP match(const SearchRequest &, vespalib::ThreadBundle &) const override { + return std::make_unique<SearchReply>(); } }; @@ -90,8 +87,7 @@ private: public: MyDocsumClient(); - - ~MyDocsumClient(); + ~MyDocsumClient() override; void getDocsumsDone(DocsumReply::UP reply) override { std::lock_guard<std::mutex> guard(_lock); @@ -111,19 +107,19 @@ public: } }; -MyDocsumClient::MyDocsumClient() {} +MyDocsumClient::MyDocsumClient() = default; -MyDocsumClient::~MyDocsumClient() {} +MyDocsumClient::~MyDocsumClient() = default; DocsumRequest::UP createRequest(size_t num = 1) { - DocsumRequest::UP r(new DocsumRequest()); + auto r = std::make_unique<DocsumRequest>(); if (num == 1) { r->hits.emplace_back(GlobalId("aaaaaaaaaaaa")); } else { for (size_t i = 0; i < num; i++) { vespalib::string s = vespalib::make_string("aaaaaaaaaaa%c", char('a' + i % 26)); - r->hits.push_back(GlobalId(s.c_str())); + r->hits.emplace_back(GlobalId(s.c_str())); } } return r; @@ -132,7 +128,7 @@ createRequest(size_t num = 1) { TEST("requireThatGetDocsumsExecute") { int numSummaryThreads = 2; SummaryEngine engine(numSummaryThreads); - ISearchHandler::SP handler(new MySearchHandler); + auto handler = std::make_shared<MySearchHandler>(); DocTypeName dtnvfoo("foo"); engine.putSearchHandler(dtnvfoo, handler); @@ -140,9 +136,9 @@ TEST("requireThatGetDocsumsExecute") { { // async call when engine running DocsumRequest::Source request(createRequest()); DocsumReply::UP reply = engine.getDocsums(std::move(request), client); - EXPECT_TRUE(reply.get() == NULL); + EXPECT_FALSE(reply); reply = client.getReply(10000); - EXPECT_TRUE(reply.get() != NULL); + EXPECT_TRUE(reply); EXPECT_EQUAL(1u, reply->docsums.size()); EXPECT_EQUAL(10u, reply->docsums[0].docid); EXPECT_EQUAL(GlobalId("aaaaaaaaaaaa"), reply->docsums[0].gid); @@ -152,7 +148,7 @@ TEST("requireThatGetDocsumsExecute") { { // sync call when engine closed DocsumRequest::Source request(createRequest()); DocsumReply::UP reply = engine.getDocsums(std::move(request), client); - EXPECT_TRUE(reply.get() != NULL); + EXPECT_TRUE(reply); } } @@ -161,23 +157,23 @@ TEST("requireThatHandlersAreStored") { DocTypeName dtnvbar("bar"); int numSummaryThreads = 2; SummaryEngine engine(numSummaryThreads); - ISearchHandler::SP h1(new MySearchHandler("foo")); - ISearchHandler::SP h2(new MySearchHandler("bar")); - ISearchHandler::SP h3(new MySearchHandler("baz")); + auto h1 = std::make_shared<MySearchHandler>("foo"); + auto h2 = std::make_shared<MySearchHandler>("bar"); + auto h3 = std::make_shared<MySearchHandler>("baz"); // not found - EXPECT_TRUE(engine.getSearchHandler(dtnvfoo).get() == NULL); - EXPECT_TRUE(engine.removeSearchHandler(dtnvfoo).get() == NULL); + EXPECT_FALSE(engine.getSearchHandler(dtnvfoo)); + EXPECT_FALSE(engine.removeSearchHandler(dtnvfoo)); // put & get - EXPECT_TRUE(engine.putSearchHandler(dtnvfoo, h1).get() == NULL); + EXPECT_FALSE(engine.putSearchHandler(dtnvfoo, h1)); EXPECT_EQUAL(engine.getSearchHandler(dtnvfoo).get(), h1.get()); - EXPECT_TRUE(engine.putSearchHandler(dtnvbar, h2).get() == NULL); + EXPECT_FALSE(engine.putSearchHandler(dtnvbar, h2)); EXPECT_EQUAL(engine.getSearchHandler(dtnvbar).get(), h2.get()); // replace EXPECT_TRUE(engine.putSearchHandler(dtnvfoo, h3).get() == h1.get()); EXPECT_EQUAL(engine.getSearchHandler(dtnvfoo).get(), h3.get()); // remove EXPECT_EQUAL(engine.removeSearchHandler(dtnvfoo).get(), h3.get()); - EXPECT_TRUE(engine.getSearchHandler(dtnvfoo).get() == NULL); + EXPECT_FALSE(engine.getSearchHandler(dtnvfoo)); } bool @@ -196,9 +192,9 @@ TEST("requireThatCorrectHandlerIsUsed") { DocTypeName dtnvbar("bar"); DocTypeName dtnvbaz("baz"); SummaryEngine engine(1); - ISearchHandler::SP h1(new MySearchHandler("foo", "foo reply")); - ISearchHandler::SP h2(new MySearchHandler("bar", "bar reply")); - ISearchHandler::SP h3(new MySearchHandler("baz", "baz reply")); + auto h1 = std::make_shared<MySearchHandler>("foo", "foo reply"); + auto h2 = std::make_shared<MySearchHandler>("bar", "bar reply"); + auto h3 = std::make_shared<MySearchHandler>("baz", "baz reply"); engine.putSearchHandler(dtnvfoo, h1); engine.putSearchHandler(dtnvbar, h2); engine.putSearchHandler(dtnvbaz, h3); @@ -369,7 +365,7 @@ public: Server::Server() : BaseServer(), engine(2), - handler(new MySearchHandler("slime", stringref(buf.GetDrainPos(), buf.GetUsedLen()))), + handler(std::make_shared<MySearchHandler>("slime", stringref(buf.GetDrainPos(), buf.GetUsedLen()))), docsumBySlime(engine), docsumByRPC(docsumBySlime) { @@ -377,7 +373,7 @@ Server::Server() engine.putSearchHandler(dtnvfoo, handler); } -Server::~Server() {} +Server::~Server() = default; vespalib::string getAnswer(size_t num) { diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index 74e3e903540..f1035776a8f 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -361,7 +361,7 @@ public: _immediateCommit(immediateCommit), _onWriteDone(onWriteDone) {} - ~BatchRemoveTask() override {} + ~BatchRemoveTask() override = default; void run() override { for (auto field : _writeCtx.getFields()) { auto &attr = field.getAttribute(); @@ -469,9 +469,9 @@ AttributeWriter::internalRemove(SerialNum serialNum, DocumentIdT lid, bool immed } } -AttributeWriter::AttributeWriter(const proton::IAttributeManager::SP &mgr) - : _mgr(mgr), - _attributeFieldWriter(mgr->getAttributeFieldWriter()), +AttributeWriter::AttributeWriter(proton::IAttributeManager::SP mgr) + : _mgr(std::move(mgr)), + _attributeFieldWriter(_mgr->getAttributeFieldWriter()), _writeContexts(), _dataType(nullptr), _hasStructFieldAttribute(false), diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h index 9e5d8f4ce5d..8c9b756cd89 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h @@ -74,7 +74,7 @@ private: bool immediateCommit, OnWriteDoneType onWriteDone); public: - AttributeWriter(const proton::IAttributeManager::SP &mgr); + AttributeWriter(proton::IAttributeManager::SP mgr); ~AttributeWriter(); /* Only for in tests that add attributes after AttributeWriter construction. */ diff --git a/searchcore/src/vespa/searchcore/proton/common/doctypename.cpp b/searchcore/src/vespa/searchcore/proton/common/doctypename.cpp index 70daeeebeca..21b27ef6ffd 100644 --- a/searchcore/src/vespa/searchcore/proton/common/doctypename.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/doctypename.cpp @@ -4,20 +4,16 @@ #include <vespa/searchlib/engine/request.h> #include <vespa/document/datatype/documenttype.h> -namespace proton -{ +namespace proton { DocTypeName::DocTypeName(const search::engine::Request &request) : _name(request.propertiesMap.matchProperties().lookup("documentdb", "searchdoctype").get("")) -{ -} +{} DocTypeName::DocTypeName(const document::DocumentType &docType) : _name(docType.getName()) -{ -} +{} - -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp b/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp index 3fb32a9d1e0..e762ac527f0 100644 --- a/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp +++ b/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp @@ -53,7 +53,6 @@ public: bool valid() const override { return (_offset < _handlers.size()); } T *get() const override { return _handlers[_offset].get(); } - HandlerSP getSP() const { return _handlers[_offset]; } void next() override { ++_offset; } }; diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/imatchhandler.h b/searchcore/src/vespa/searchcore/proton/matchengine/imatchhandler.h deleted file mode 100644 index 63283de0a4a..00000000000 --- a/searchcore/src/vespa/searchcore/proton/matchengine/imatchhandler.h +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/searchcore/proton/summaryengine/isearchhandler.h> -#include <vespa/vespalib/util/thread_bundle.h> - -namespace searche { -namespace engine { - class SearchRequest; - class SearchReply; -} -} - -namespace proton { - -/** - * This interface describes a sync match operation handler. It is implemented by - * the DocumentDB class, and used by the MatchEngine class to delegate - * operations to the appropriate db. - */ -class IMatchHandler { -protected: - using SearchReply = search::engine::SearchReply; - using SearchRequest = search::engine::SearchRequest; - using ThreadBundle = vespalib::ThreadBundle; - IMatchHandler() = default; -public: - IMatchHandler(const IMatchHandler &) = delete; - IMatchHandler & operator = (const IMatchHandler &) = delete; - /** - * Convenience typedefs. - */ - typedef std::unique_ptr<IMatchHandler> UP; - typedef std::shared_ptr<IMatchHandler> SP; - - virtual ~IMatchHandler() { } - - /** - * @return Use the request and produce the matching result. - */ - virtual std::unique_ptr<SearchReply> - match(const ISearchHandler::SP &searchHandler, const SearchRequest &req, ThreadBundle &threadBundle) const = 0; -}; - -} // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp index b170f60d71f..ad47fb98232 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp @@ -121,12 +121,12 @@ MatchEngine::performSearch(search::engine::SearchRequest::Source req, ISearchHandler::SP searchHandler; vespalib::SimpleThreadBundle::UP threadBundle = _threadBundlePool.obtain(); { // try to find the match handler corresponding to the specified search doc type - std::lock_guard<std::mutex> guard(_lock); DocTypeName docTypeName(*searchRequest); + std::lock_guard<std::mutex> guard(_lock); searchHandler = _handlers.getHandler(docTypeName); } if (searchHandler) { - ret = searchHandler->match(searchHandler, *searchRequest, *threadBundle); + ret = searchHandler->match(*searchRequest, *threadBundle); } else { HandlerMap<ISearchHandler>::Snapshot snapshot; { @@ -134,8 +134,7 @@ MatchEngine::performSearch(search::engine::SearchRequest::Source req, snapshot = _handlers.snapshot(); } if (snapshot.valid()) { - ISearchHandler::SP handler = snapshot.getSP(); - ret = handler->match(handler, *searchRequest, *threadBundle); // use the first handler + ret = snapshot.get()->match(*searchRequest, *threadBundle); // use the first handler } } _threadBundlePool.release(std::move(threadBundle)); diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h index 896ac83403a..34aacdafcad 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "imatchhandler.h" +#include <vespa/searchcore/proton/summaryengine/isearchhandler.h> #include <vespa/searchcore/proton/common/doctypename.h> #include <vespa/searchcore/proton/common/handlermap.hpp> #include <vespa/searchcore/proton/common/statusreport.h> @@ -50,7 +50,7 @@ public: * Frees any allocated resources. this will also stop all internal threads * and wait for them to finish. All pending search requests are deleted. */ - ~MatchEngine(); + ~MatchEngine() override; /** * Observe and reset internal executor stats @@ -123,13 +123,11 @@ public: StatusReport::UP reportStatus() const; - // Implements SearchServer. search::engine::SearchReply::UP search( search::engine::SearchRequest::Source request, search::engine::SearchClient &client) override; - // Implements vespalib::StateExplorer - virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; + void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp index 7c5e7584eed..0e80d31a063 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp @@ -35,7 +35,7 @@ namespace { struct WaitTimer { double &wait_time_s; vespalib::Timer wait_time; - WaitTimer(double &wait_time_s_in) + explicit WaitTimer(double &wait_time_s_in) : wait_time_s(wait_time_s_in), wait_time() { } void done() { diff --git a/searchcore/src/vespa/searchcore/proton/matching/search_session.h b/searchcore/src/vespa/searchcore/proton/matching/search_session.h index 0aec02e9d31..13d24624597 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/search_session.h +++ b/searchcore/src/vespa/searchcore/proton/matching/search_session.h @@ -26,7 +26,7 @@ public: OwnershipBundle(OwnershipBundle &&) = default; OwnershipBundle & operator = (OwnershipBundle &&) = default; ~OwnershipBundle(); - ISearchHandler::SP search_handler; + std::shared_ptr<const ISearchHandler> search_handler; std::unique_ptr<search::fef::Properties> feature_overrides; std::unique_ptr<MatchContext> context; IDocumentMetaStoreContext::IReadGuard::UP readGuard; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 1b182c3e618..302ffc93f6a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -756,11 +756,11 @@ DocumentDB::getNewestFlushedSerial() } std::unique_ptr<SearchReply> -DocumentDB::match(const ISearchHandler::SP &, const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const +DocumentDB::match(const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const { // Ignore input searchhandler. Use readysubdb's searchhandler instead. ISearchHandler::SP view(_subDBs.getReadySubDB()->getSearchView()); - return view->match(view, req, threadBundle); + return view->match(req, threadBundle); } std::unique_ptr<DocsumReply> diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index 02ad144af68..f296e264903 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -368,8 +368,7 @@ public: virtual SerialNum getNewestFlushedSerial(); std::unique_ptr<search::engine::SearchReply> - match(const ISearchHandler::SP &searchHandler, - const search::engine::SearchRequest &req, + match(const search::engine::SearchRequest &req, vespalib::ThreadBundle &threadBundle) const; std::unique_ptr<search::engine::DocsumReply> diff --git a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp index 90cf0011685..9f2ec26ad4b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp @@ -14,10 +14,7 @@ using search::engine::SearchRequest; namespace proton { -EmptySearchView::EmptySearchView() - : ISearchHandler() -{ -} +EmptySearchView::EmptySearchView() = default; DocsumReply::UP @@ -25,20 +22,16 @@ EmptySearchView::getDocsums(const DocsumRequest &req) { LOG(debug, "getDocsums(): resultClass(%s), numHits(%zu)", req.resultClassName.c_str(), req.hits.size()); - DocsumReply::UP reply(new DocsumReply()); - for (size_t i = 0; i < req.hits.size(); ++i) { - reply->docsums.push_back(DocsumReply::Docsum()); - reply->docsums.back().gid = req.hits[i].gid; + auto reply = std::make_unique<DocsumReply>(); + for (const auto & hit : req.hits) { + reply->docsums.emplace_back(hit.gid); } return reply; } SearchReply::UP -EmptySearchView::match(const ISearchHandler::SP &, - const SearchRequest &, - vespalib::ThreadBundle &) const { - SearchReply::UP reply(new SearchReply); - return reply; +EmptySearchView::match(const SearchRequest &, vespalib::ThreadBundle &) const { + return std::make_unique<SearchReply>(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h index a3e2a2b176c..92fb97f6177 100644 --- a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h +++ b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h @@ -13,15 +13,10 @@ public: EmptySearchView(); - /** - * Implements ISearchHandler - */ - virtual std::unique_ptr<DocsumReply> getDocsums(const DocsumRequest & req) override; - - virtual std::unique_ptr<SearchReply> - match(const ISearchHandler::SP &searchHandler, - const SearchRequest &req, - vespalib::ThreadBundle &threadBundle) const override; + std::unique_ptr<DocsumReply> getDocsums(const DocsumRequest & req) override; + + std::unique_ptr<SearchReply> + match(const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp index 6f32f886637..7ba9b971715 100644 --- a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp @@ -31,17 +31,17 @@ using matching::ISearchContext; using matching::Matcher; using matching::SessionManager; -MatchView::MatchView(const Matchers::SP &matchers, - const IndexSearchable::SP &indexSearchable, - const IAttributeManager::SP &attrMgr, - const SessionManagerSP &sessionMgr, - const IDocumentMetaStoreContext::SP &metaStore, +MatchView::MatchView(Matchers::SP matchers, + IndexSearchable::SP indexSearchable, + IAttributeManager::SP attrMgr, + SessionManagerSP sessionMgr, + IDocumentMetaStoreContext::SP metaStore, DocIdLimit &docIdLimit) - : _matchers(matchers), - _indexSearchable(indexSearchable), - _attrMgr(attrMgr), - _sessionMgr(sessionMgr), - _metaStore(metaStore), + : _matchers(std::move(matchers)), + _indexSearchable(std::move(indexSearchable)), + _attrMgr(std::move(attrMgr)), + _sessionMgr(std::move(sessionMgr)), + _metaStore(std::move(metaStore)), _docIdLimit(docIdLimit) { } @@ -68,12 +68,12 @@ MatchView::createContext() const { std::unique_ptr<SearchReply> -MatchView::match(const ISearchHandler::SP &searchHandler, const SearchRequest &req, +MatchView::match(std::shared_ptr<const ISearchHandler> searchHandler, const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const { Matcher::SP matcher = getMatcher(req.ranking); SearchSession::OwnershipBundle owned_objects; - owned_objects.search_handler = searchHandler; + owned_objects.search_handler = std::move(searchHandler); owned_objects.context = createContext(); owned_objects.readGuard = _metaStore->getReadGuard();; MatchContext *ctx = owned_objects.context.get(); diff --git a/searchcore/src/vespa/searchcore/proton/server/matchview.h b/searchcore/src/vespa/searchcore/proton/server/matchview.h index 983e06d3414..721cb439508 100644 --- a/searchcore/src/vespa/searchcore/proton/server/matchview.h +++ b/searchcore/src/vespa/searchcore/proton/server/matchview.h @@ -36,11 +36,11 @@ public: MatchView(const MatchView &) = delete; MatchView & operator = (const MatchView &) = delete; - MatchView(const Matchers::SP &matchers, - const searchcorespi::IndexSearchable::SP &indexSearchable, - const IAttributeManager::SP &attrMgr, - const SessionManagerSP &sessionMgr, - const IDocumentMetaStoreContext::SP &metaStore, + MatchView(Matchers::SP matchers, + searchcorespi::IndexSearchable::SP indexSearchable, + IAttributeManager::SP attrMgr, + SessionManagerSP sessionMgr, + IDocumentMetaStoreContext::SP metaStore, DocIdLimit &docIdLimit); ~MatchView(); @@ -62,7 +62,7 @@ public: matching::MatchContext::UP createContext() const; std::unique_ptr<search::engine::SearchReply> - match(const std::shared_ptr<ISearchHandler> &searchHandler, + match(std::shared_ptr<const ISearchHandler> searchHandler, const search::engine::SearchRequest &req, vespalib::ThreadBundle &threadBundle) const; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 28de4dff917..20de5bb07c1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -628,8 +628,7 @@ Proton::removeDocumentDB(const DocTypeName &docTypeName) { // Not allowed to get to service layer to call pause(). std::unique_lock<std::shared_timed_mutex> persistenceWguard(_persistenceEngine->getWLock()); - IPersistenceHandler::SP oldHandler; - oldHandler = _persistenceEngine->removeHandler(persistenceWguard, old->getBucketSpace(), docTypeName); + IPersistenceHandler::SP oldHandler = _persistenceEngine->removeHandler(persistenceWguard, old->getBucketSpace(), docTypeName); if (_initComplete && oldHandler) { // TODO: Fix race with bucket db modifying ops. _persistenceEngine->grabExtraModifiedBuckets(old->getBucketSpace(), *oldHandler); diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp index 069d3727850..9ef038b7325 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp @@ -84,17 +84,17 @@ reconfigureMatchView(const Matchers::SP &matchers, } void -SearchableDocSubDBConfigurer::reconfigureSearchView(const MatchView::SP &matchView) +SearchableDocSubDBConfigurer::reconfigureSearchView(MatchView::SP matchView) { SearchView::SP curr = _searchView.get(); - _searchView.set(SearchView::SP(new SearchView(curr->getSummarySetup(), matchView))); + _searchView.set(SearchView::create(curr->getSummarySetup(), std::move(matchView))); } void -SearchableDocSubDBConfigurer::reconfigureSearchView(const ISummaryManager::ISummarySetup::SP &summarySetup, - const MatchView::SP &matchView) +SearchableDocSubDBConfigurer::reconfigureSearchView(ISummaryManager::ISummarySetup::SP summarySetup, + MatchView::SP matchView) { - _searchView.set(SearchView::SP(new SearchView(summarySetup, matchView))); + _searchView.set(SearchView::create(std::move(summarySetup), std::move(matchView))); } SearchableDocSubDBConfigurer:: diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h index 5aac069853b..459c4651e67 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h @@ -65,11 +65,10 @@ private: const IAttributeManager::SP &attrMgr); void - reconfigureSearchView(const MatchView::SP &matchView); + reconfigureSearchView(MatchView::SP matchView); void - reconfigureSearchView(const ISummaryManager::ISummarySetup::SP &summarySetup, - const MatchView::SP &matchView); + reconfigureSearchView(ISummaryManager::ISummarySetup::SP summarySetup, MatchView::SP matchView); public: SearchableDocSubDBConfigurer(const SearchableDocSubDBConfigurer &) = delete; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp index 77852dcc918..3ca8a4cff49 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp @@ -203,16 +203,16 @@ SearchableDocSubDB::initViews(const DocumentDBConfig &configSnapshot, const Sess const IIndexManager::SP &indexMgr = getIndexManager(); _constantValueRepo.reconfigure(configSnapshot.getRankingConstants()); Matchers::SP matchers(_configurer.createMatchers(schema, configSnapshot.getRankProfilesConfig()).release()); - auto matchView = std::make_shared<MatchView>(matchers, indexMgr->getSearchable(), attrMgr, + auto matchView = std::make_shared<MatchView>(std::move(matchers), indexMgr->getSearchable(), attrMgr, sessionManager, _metaStoreCtx, _docIdLimit); - _rSearchView.set(std::make_shared<SearchView>( + _rSearchView.set(SearchView::create( getSummaryManager()->createSummarySetup( configSnapshot.getSummaryConfig(), configSnapshot.getSummarymapConfig(), configSnapshot.getJuniperrcConfig(), configSnapshot.getDocumentTypeRepoSP(), - matchView->getAttributeManager()), - matchView)); + attrMgr), + std::move(matchView))); auto attrWriter = std::make_shared<AttributeWriter>(attrMgr); { diff --git a/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.cpp index f611b4e1f4c..6da6c09cdba 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.cpp @@ -7,8 +7,8 @@ namespace proton { -SearchHandlerProxy::SearchHandlerProxy(const DocumentDB::SP &documentDB) - : _documentDB(documentDB) +SearchHandlerProxy::SearchHandlerProxy(DocumentDB::SP documentDB) + : _documentDB(std::move(documentDB)) { _documentDB->retain(); } @@ -25,11 +25,9 @@ SearchHandlerProxy::getDocsums(const DocsumRequest & request) } std::unique_ptr<search::engine::SearchReply> -SearchHandlerProxy::match(const ISearchHandler::SP &searchHandler, - const SearchRequest &req, - vespalib::ThreadBundle &threadBundle) const +SearchHandlerProxy::match(const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const { - return _documentDB->match(searchHandler, req, threadBundle); + return _documentDB->match(req, threadBundle); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h index 4e9f5bdab8a..fc4f517fb36 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h @@ -13,11 +13,11 @@ class SearchHandlerProxy : public ISearchHandler private: std::shared_ptr<DocumentDB> _documentDB; public: - SearchHandlerProxy(const std::shared_ptr<DocumentDB> &documentDB); + SearchHandlerProxy(std::shared_ptr<DocumentDB> documentDB); - virtual~SearchHandlerProxy(); + ~SearchHandlerProxy() override; std::unique_ptr<DocsumReply> getDocsums(const DocsumRequest & request) override; - std::unique_ptr<SearchReply> match(const ISearchHandler::SP &searchHandler, const SearchRequest &req, ThreadBundle &threadBundle) const override; + std::unique_ptr<SearchReply> match(const SearchRequest &req, ThreadBundle &threadBundle) const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/searchview.cpp b/searchcore/src/vespa/searchcore/proton/server/searchview.cpp index 9830048cdd8..36d873d9948 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchview.cpp @@ -105,11 +105,14 @@ createEmptyReply(const DocsumRequest & request) } -SearchView::SearchView(const ISummaryManager::ISummarySetup::SP & summarySetup, - const MatchView::SP & matchView) +std::shared_ptr<SearchView> +SearchView::create(ISummaryManager::ISummarySetup::SP summarySetup, MatchView::SP matchView) { + return std::shared_ptr<SearchView>( new SearchView(std::move(summarySetup), std::move(matchView))); +} +SearchView::SearchView(ISummaryManager::ISummarySetup::SP summarySetup, MatchView::SP matchView) : ISearchHandler(), - _summarySetup(summarySetup), - _matchView(matchView) + _summarySetup(std::move(summarySetup)), + _matchView(std::move(matchView)) { } SearchView::~SearchView() = default; @@ -161,8 +164,8 @@ SearchView::getDocsumsInternal(const DocsumRequest & req) } std::unique_ptr<SearchReply> -SearchView::match(const ISearchHandler::SP &self, const SearchRequest &req, ThreadBundle &threadBundle) const { - return _matchView->match(self, req, threadBundle); +SearchView::match(const SearchRequest &req, ThreadBundle &threadBundle) const { + return _matchView->match(shared_from_this(), req, threadBundle); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/searchview.h b/searchcore/src/vespa/searchcore/proton/server/searchview.h index 28c7ffd2b36..630d29c32fd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchview.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchview.h @@ -8,7 +8,7 @@ namespace proton { -class SearchView : public ISearchHandler +class SearchView : public ISearchHandler, public std::enable_shared_from_this<SearchView> { public: using SessionManagerSP = std::shared_ptr<matching::SessionManager>; @@ -16,12 +16,12 @@ public: using InternalDocsumReply = std::pair<std::unique_ptr<DocsumReply>, bool>; typedef std::shared_ptr<SearchView> SP; - SearchView(const ISummaryManager::ISummarySetup::SP &summarySetup, const MatchView::SP &matchView); + static std::shared_ptr<SearchView> create(ISummaryManager::ISummarySetup::SP summarySetup, MatchView::SP matchView); SearchView(const SearchView &) = delete; SearchView(SearchView &&) = delete; SearchView &operator=(const SearchView &) = delete; SearchView &operator=(SearchView &&) = delete; - ~SearchView(); + ~SearchView() override; const ISummaryManager::ISummarySetup::SP & getSummarySetup() const { return _summarySetup; } const MatchView::SP & getMatchView() const { return _matchView; } @@ -34,8 +34,9 @@ public: matching::MatchingStats getMatcherStats(const vespalib::string &rankProfile) const { return _matchView->getMatcherStats(rankProfile); } std::unique_ptr<DocsumReply> getDocsums(const DocsumRequest & req) override; - std::unique_ptr<SearchReply> match(const ISearchHandler::SP &self, const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const override; + std::unique_ptr<SearchReply> match(const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const override; private: + SearchView(ISummaryManager::ISummarySetup::SP summarySetup, MatchView::SP matchView); InternalDocsumReply getDocsumsInternal(const DocsumRequest & req); ISummaryManager::ISummarySetup::SP _summarySetup; MatchView::SP _matchView; diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h index 9d5b8c18d01..14f62513c34 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h @@ -13,7 +13,6 @@ #include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h> #include <vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h> -#include <vespa/searchcore/proton/matchengine/imatchhandler.h> #include <vespa/searchcore/proton/summaryengine/isearchhandler.h> #include <vespa/searchcore/proton/common/commit_time_tracker.h> #include <vespa/searchcore/proton/persistenceengine/i_document_retriever.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp index 8f1c3560e9b..55aa1a20ef6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp @@ -22,7 +22,10 @@ namespace { uint32_t calculateIndexingThreads(uint32_t cfgIndexingThreads, double concurrency, const HwInfo::Cpu &cpuInfo) { - double scaledCores = cpuInfo.cores() * concurrency; + // We are capping at 12 threads to reduce cost of waking up threads + // to achieve a better throughput. + // TODO: Fix this in a simpler/better way. + double scaledCores = std::min(12.0, cpuInfo.cores() * concurrency); uint32_t indexingThreads = std::max((uint32_t)std::ceil(scaledCores / 3), cfgIndexingThreads); return std::max(indexingThreads, 1u); } diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/isearchhandler.h b/searchcore/src/vespa/searchcore/proton/summaryengine/isearchhandler.h index ffcdfbaf365..b7dd438ae29 100644 --- a/searchcore/src/vespa/searchcore/proton/summaryengine/isearchhandler.h +++ b/searchcore/src/vespa/searchcore/proton/summaryengine/isearchhandler.h @@ -26,12 +26,11 @@ protected: using DocsumRequest = search::engine::DocsumRequest; using ThreadBundle = vespalib::ThreadBundle; public: - typedef std::unique_ptr<ISearchHandler> UP; typedef std::shared_ptr<ISearchHandler> SP; ISearchHandler(const ISearchHandler &) = delete; ISearchHandler & operator = (const ISearchHandler &) = delete; - virtual ~ISearchHandler() { } + virtual ~ISearchHandler() = default; /** * @return Use the request and produce the document summary result. @@ -39,7 +38,7 @@ public: virtual std::unique_ptr<DocsumReply> getDocsums(const DocsumRequest & request) = 0; virtual std::unique_ptr<SearchReply> - match(const ISearchHandler::SP &self, const SearchRequest &req, ThreadBundle &threadBundle) const = 0; + match(const SearchRequest &req, ThreadBundle &threadBundle) const = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp index 692b05899f4..1189e8a550c 100644 --- a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp @@ -107,13 +107,13 @@ SummaryEngine::getDocsums(DocsumRequest::Source request, DocsumClient & client) { if (_closed) { LOG(warning, "Receiving docsumrequest after engine has been shutdown"); - DocsumReply::UP ret(new DocsumReply()); + auto ret = std::make_unique<DocsumReply>(); // TODO: Notify closed. return ret; } - vespalib::Executor::Task::UP task(new DocsumTask(*this, std::move(request), client)); + auto task =std::make_unique<DocsumTask>(*this, std::move(request), client); _executor.execute(std::move(task)); return DocsumReply::UP(); } diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index a7689cd6b9f..60f3a6b7664 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -16,7 +16,6 @@ #include <vespa/searchlib/index/i_field_length_inspector.h> #include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/query/tree/simplequery.h> -#include <vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h> #include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/searchlib/queryeval/fake_search.h> #include <vespa/searchlib/queryeval/fake_searchable.h> @@ -24,7 +23,6 @@ #include <vespa/searchlib/test/index/mock_field_length_inspector.h> #include <vespa/searchlib/util/rand48.h> #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/log/log.h> diff --git a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp index 7d09b2aa0b8..850a967ed3d 100644 --- a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp +++ b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp @@ -278,6 +278,22 @@ AttributeManagerTest::testConfigConvert() AttributeVector::Config out = ConfigConverter::convert(a); EXPECT_EQUAL("tensor(x[5])", out.tensorType().to_spec()); } + { // hnsw index params (enabled) + CACA a; + a.index.hnsw.enabled = true; + a.index.hnsw.maxlinkspernode = 32; + a.index.hnsw.neighborstoexploreatinsert = 300; + auto out = ConfigConverter::convert(a); + EXPECT_TRUE(out.hnsw_index_params().has_value()); + EXPECT_EQUAL(32u, out.hnsw_index_params().value().max_links_per_node()); + EXPECT_EQUAL(300u, out.hnsw_index_params().value().neighbors_to_explore_at_insert()); + } + { // hnsw index params (disabled) + CACA a; + a.index.hnsw.enabled = false; + auto out = ConfigConverter::convert(a); + EXPECT_FALSE(out.hnsw_index_params().has_value()); + } } bool gt_attribute(const attribute::IAttributeVector * a, const attribute::IAttributeVector * b) { diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 7e0fcdc0ccc..5089743a54a 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -1,34 +1,48 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/base/exceptions.h> -#include <vespa/searchlib/tensor/tensor_attribute.h> -#include <vespa/searchlib/tensor/generic_tensor_attribute.h> -#include <vespa/searchlib/tensor/dense_tensor_attribute.h> -#include <vespa/searchlib/attribute/attributeguard.h> -#include <vespa/eval/tensor/tensor.h> -#include <vespa/eval/tensor/dense/dense_tensor.h> #include <vespa/eval/tensor/default_tensor_engine.h> -#include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib/data/fileheader.h> +#include <vespa/eval/tensor/dense/dense_tensor.h> +#include <vespa/eval/tensor/tensor.h> #include <vespa/fastos/file.h> +#include <vespa/searchlib/attribute/attributeguard.h> +#include <vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h> +#include <vespa/searchlib/tensor/dense_tensor_attribute.h> +#include <vespa/searchlib/tensor/doc_vector_access.h> +#include <vespa/searchlib/tensor/generic_tensor_attribute.h> +#include <vespa/searchlib/tensor/hnsw_index.h> +#include <vespa/searchlib/tensor/nearest_neighbor_index.h> +#include <vespa/searchlib/tensor/nearest_neighbor_index_factory.h> +#include <vespa/searchlib/tensor/tensor_attribute.h> +#include <vespa/vespalib/data/fileheader.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/vespalib/testkit/test_kit.h> + #include <vespa/log/log.h> LOG_SETUP("tensorattribute_test"); using document::WrongTensorTypeException; -using search::tensor::TensorAttribute; -using search::tensor::DenseTensorAttribute; -using search::tensor::GenericTensorAttribute; using search::AttributeGuard; using search::AttributeVector; -using vespalib::eval::ValueType; +using search::attribute::HnswIndexParams; +using search::tensor::DefaultNearestNeighborIndexFactory; +using search::tensor::DenseTensorAttribute; +using search::tensor::DocVectorAccess; +using search::tensor::GenericTensorAttribute; +using search::tensor::HnswIndex; +using search::tensor::NearestNeighborIndex; +using search::tensor::NearestNeighborIndexFactory; +using search::tensor::TensorAttribute; using vespalib::eval::TensorSpec; -using vespalib::tensor::Tensor; -using vespalib::tensor::DenseTensor; +using vespalib::eval::ValueType; using vespalib::tensor::DefaultTensorEngine; +using vespalib::tensor::DenseTensor; +using vespalib::tensor::Tensor; + +using DoubleVector = std::vector<double>; -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { static bool operator==(const Tensor &lhs, const Tensor &rhs) { @@ -36,10 +50,10 @@ static bool operator==(const Tensor &lhs, const Tensor &rhs) } } -} vespalib::string sparseSpec("tensor(x{},y{})"); vespalib::string denseSpec("tensor(x[2],y[3])"); +vespalib::string vec_2d_spec("tensor(x[2])"); Tensor::UP createTensor(const TensorSpec &spec) { auto value = DefaultTensorEngine::ref().from_spec(spec); @@ -52,6 +66,78 @@ Tensor::UP createTensor(const TensorSpec &spec) { return Tensor::UP(tensor); } +TensorSpec +vec_2d(double x0, double x1) +{ + return TensorSpec(vec_2d_spec).add({{"x", 0}}, x0).add({{"x", 1}}, x1); +} + +class MockNearestNeighborIndex : public NearestNeighborIndex { +private: + using Entry = std::pair<uint32_t, DoubleVector>; + using EntryVector = std::vector<Entry>; + + const DocVectorAccess& _vectors; + EntryVector _adds; + EntryVector _removes; + +public: + MockNearestNeighborIndex(const DocVectorAccess& vectors) + : _vectors(vectors), + _adds(), + _removes() + { + } + void clear() { + _adds.clear(); + _removes.clear(); + } + void expect_empty_add() const { + EXPECT_TRUE(_adds.empty()); + } + void expect_add(uint32_t exp_docid, const DoubleVector& exp_vector) const { + EXPECT_EQUAL(1u, _adds.size()); + EXPECT_EQUAL(exp_docid, _adds.back().first); + EXPECT_EQUAL(exp_vector, _adds.back().second); + } + void expect_adds(const EntryVector &exp_adds) const { + EXPECT_EQUAL(exp_adds, _adds); + } + void expect_empty_remove() const { + EXPECT_TRUE(_removes.empty()); + } + void expect_remove(uint32_t exp_docid, const DoubleVector& exp_vector) const { + EXPECT_EQUAL(1u, _removes.size()); + EXPECT_EQUAL(exp_docid, _removes.back().first); + EXPECT_EQUAL(exp_vector, _removes.back().second); + } + void add_document(uint32_t docid) override { + auto vector = _vectors.get_vector(docid).typify<double>(); + _adds.emplace_back(docid, DoubleVector(vector.begin(), vector.end())); + } + void remove_document(uint32_t docid) override { + auto vector = _vectors.get_vector(docid).typify<double>(); + _removes.emplace_back(docid, DoubleVector(vector.begin(), vector.end())); + } + std::vector<Neighbor> find_top_k(uint32_t k, vespalib::tensor::TypedCells vector, uint32_t explore_k) const override { + (void) k; + (void) vector; + (void) explore_k; + return std::vector<Neighbor>(); + } +}; + +class MockNearestNeighborIndexFactory : public NearestNeighborIndexFactory { + + std::unique_ptr<NearestNeighborIndex> make(const DocVectorAccess& vectors, + ValueType::CellType cell_type, + const search::attribute::HnswIndexParams& params) const override { + (void) params; + assert(cell_type == ValueType::CellType::DOUBLE); + return std::make_unique<MockNearestNeighborIndex>(vectors); + } +}; + struct Fixture { using BasicType = search::attribute::BasicType; @@ -61,16 +147,20 @@ struct Fixture Config _cfg; vespalib::string _name; vespalib::string _typeSpec; + std::unique_ptr<NearestNeighborIndexFactory> _index_factory; std::shared_ptr<TensorAttribute> _tensorAttr; std::shared_ptr<AttributeVector> _attr; bool _denseTensors; bool _useDenseTensorAttribute; Fixture(const vespalib::string &typeSpec, - bool useDenseTensorAttribute = false) + bool useDenseTensorAttribute = false, + bool enable_hnsw_index = false, + bool use_mock_index = false) : _cfg(BasicType::TENSOR, CollectionType::SINGLE), _name("test"), _typeSpec(typeSpec), + _index_factory(std::make_unique<DefaultNearestNeighborIndexFactory>()), _tensorAttr(), _attr(), _denseTensors(false), @@ -80,20 +170,40 @@ struct Fixture if (_cfg.tensorType().is_dense()) { _denseTensors = true; } + if (enable_hnsw_index) { + _cfg.set_hnsw_index_params(HnswIndexParams(4, 20)); + if (use_mock_index) { + _index_factory = std::make_unique<MockNearestNeighborIndexFactory>(); + } + } _tensorAttr = makeAttr(); _attr = _tensorAttr; _attr->addReservedDoc(); } + ~Fixture() {} std::shared_ptr<TensorAttribute> makeAttr() { if (_useDenseTensorAttribute) { assert(_denseTensors); - return std::make_shared<DenseTensorAttribute>(_name, _cfg); + return std::make_shared<DenseTensorAttribute>(_name, _cfg, *_index_factory); } else { return std::make_shared<GenericTensorAttribute>(_name, _cfg); } } + const DenseTensorAttribute& as_dense_tensor() const { + auto result = dynamic_cast<const DenseTensorAttribute*>(_tensorAttr.get()); + assert(result != nullptr); + return *result; + } + + MockNearestNeighborIndex& mock_index() { + assert(as_dense_tensor().nearest_neighbor_index() != nullptr); + auto mock_index = dynamic_cast<const MockNearestNeighborIndex*>(as_dense_tensor().nearest_neighbor_index()); + assert(mock_index != nullptr); + return *const_cast<MockNearestNeighborIndex*>(mock_index); + } + void ensureSpace(uint32_t docId) { while (_attr->getNumDocs() <= docId) { uint32_t newDocId = 0u; @@ -108,7 +218,15 @@ struct Fixture _attr->commit(); } - void setTensor(uint32_t docId, const Tensor &tensor) { + void set_tensor(uint32_t docid, const TensorSpec &spec) { + set_tensor_internal(docid, *createTensor(spec)); + } + + void set_empty_tensor(uint32_t docid) { + set_tensor_internal(docid, *_tensorAttr->getEmptyTensor()); + } + + void set_tensor_internal(uint32_t docId, const Tensor &tensor) { ensureSpace(docId); _tensorAttr->setTensor(docId, tensor); _attr->commit(); @@ -119,27 +237,18 @@ struct Fixture return _attr->getStatus(); } - void - assertGetNoTensor(uint32_t docId) { + void assertGetNoTensor(uint32_t docId) { AttributeGuard guard(_attr); Tensor::UP actTensor = _tensorAttr->getTensor(docId); EXPECT_FALSE(actTensor); } - void - assertGetTensor(const Tensor &expTensor, uint32_t docId) - { + void assertGetTensor(const TensorSpec &expSpec, uint32_t docId) { + Tensor::UP expTensor = createTensor(expSpec); AttributeGuard guard(_attr); Tensor::UP actTensor = _tensorAttr->getTensor(docId); EXPECT_TRUE(static_cast<bool>(actTensor)); - EXPECT_EQUAL(expTensor, *actTensor); - } - - void - assertGetTensor(const TensorSpec &expSpec, uint32_t docId) - { - Tensor::UP expTensor = createTensor(expSpec); - assertGetTensor(*expTensor, docId); + EXPECT_EQUAL(*expTensor, *actTensor); } void save() { @@ -154,23 +263,20 @@ struct Fixture EXPECT_TRUE(loadok); } - Tensor::UP expDenseTensor3() const - { - return createTensor(TensorSpec(denseSpec) - .add({{"x", 0}, {"y", 1}}, 11) - .add({{"x", 1}, {"y", 2}}, 0)); + TensorSpec expDenseTensor3() const { + return TensorSpec(denseSpec) + .add({{"x", 0}, {"y", 1}}, 11) + .add({{"x", 1}, {"y", 2}}, 0); } - Tensor::UP expDenseFillTensor() const - { - return createTensor(TensorSpec(denseSpec) - .add({{"x", 0}, {"y", 0}}, 5) - .add({{"x", 1}, {"y", 2}}, 0)); + TensorSpec expDenseFillTensor() const { + return TensorSpec(denseSpec) + .add({{"x", 0}, {"y", 0}}, 5) + .add({{"x", 1}, {"y", 2}}, 0); } - Tensor::UP expEmptyDenseTensor() const - { - return createTensor(TensorSpec(denseSpec)); + TensorSpec expEmptyDenseTensor() const { + return TensorSpec(denseSpec); } vespalib::string expEmptyDenseTensorSpec() const { @@ -200,21 +306,21 @@ Fixture::testSetTensorValue() EXPECT_EQUAL(5u, _attr->getNumDocs()); EXPECT_EQUAL(5u, _attr->getCommittedDocIdLimit()); TEST_DO(assertGetNoTensor(4)); - EXPECT_EXCEPTION(setTensor(4, *createTensor(TensorSpec("double"))), + EXPECT_EXCEPTION(set_tensor(4, TensorSpec("double")), WrongTensorTypeException, "but other tensor type is 'double'"); TEST_DO(assertGetNoTensor(4)); - setTensor(4, *_tensorAttr->getEmptyTensor()); + set_empty_tensor(4); if (_denseTensors) { - TEST_DO(assertGetTensor(*expEmptyDenseTensor(), 4)); - setTensor(3, *expDenseTensor3()); - TEST_DO(assertGetTensor(*expDenseTensor3(), 3)); + TEST_DO(assertGetTensor(expEmptyDenseTensor(), 4)); + set_tensor(3, expDenseTensor3()); + TEST_DO(assertGetTensor(expDenseTensor3(), 3)); } else { TEST_DO(assertGetTensor(TensorSpec(sparseSpec), 4)); - setTensor(3, *createTensor(TensorSpec(sparseSpec) - .add({{"x", ""}, {"y", ""}}, 11))); + set_tensor(3, TensorSpec(sparseSpec) + .add({{"x", ""}, {"y", ""}}, 11)); TEST_DO(assertGetTensor(TensorSpec(sparseSpec) - .add({{"x", ""}, {"y", ""}}, 11), 3)); + .add({{"x", ""}, {"y", ""}}, 11), 3)); } TEST_DO(assertGetNoTensor(2)); TEST_DO(clearTensor(3)); @@ -225,23 +331,23 @@ void Fixture::testSaveLoad() { ensureSpace(4); - setTensor(4, *_tensorAttr->getEmptyTensor()); + set_empty_tensor(4); if (_denseTensors) { - setTensor(3, *expDenseTensor3()); + set_tensor(3, expDenseTensor3()); } else { - setTensor(3, *createTensor(TensorSpec(sparseSpec) - .add({{"x", ""}, {"y", "1"}}, 11))); + set_tensor(3, TensorSpec(sparseSpec) + .add({{"x", ""}, {"y", "1"}}, 11)); } TEST_DO(save()); TEST_DO(load()); EXPECT_EQUAL(5u, _attr->getNumDocs()); EXPECT_EQUAL(5u, _attr->getCommittedDocIdLimit()); if (_denseTensors) { - TEST_DO(assertGetTensor(*expDenseTensor3(), 3)); - TEST_DO(assertGetTensor(*expEmptyDenseTensor(), 4)); + TEST_DO(assertGetTensor(expDenseTensor3(), 3)); + TEST_DO(assertGetTensor(expEmptyDenseTensor(), 4)); } else { TEST_DO(assertGetTensor(TensorSpec(sparseSpec) - .add({{"x", ""}, {"y", "1"}}, 11), 3)); + .add({{"x", ""}, {"y", "1"}}, 11), 3)); TEST_DO(assertGetTensor(TensorSpec(sparseSpec), 4)); } TEST_DO(assertGetNoTensor(2)); @@ -256,29 +362,28 @@ Fixture::testCompaction() return; } ensureSpace(4); - Tensor::UP emptytensor = _tensorAttr->getEmptyTensor(); - Tensor::UP emptyxytensor = createTensor(TensorSpec(sparseSpec)); - Tensor::UP simpletensor = createTensor(TensorSpec(sparseSpec) - .add({{"x", ""}, {"y", "1"}}, 11)); - Tensor::UP filltensor = createTensor(TensorSpec(sparseSpec) - .add({{"x", ""}, {"y", ""}}, 5)); + TensorSpec empty_xy_tensor(sparseSpec); + TensorSpec simple_tensor = TensorSpec(sparseSpec) + .add({{"x", ""}, {"y", "1"}}, 11); + TensorSpec fill_tensor = TensorSpec(sparseSpec) + .add({{"x", ""}, {"y", ""}}, 5); if (_denseTensors) { - emptyxytensor = expEmptyDenseTensor(); - simpletensor = expDenseTensor3(); - filltensor = expDenseFillTensor(); + empty_xy_tensor = expEmptyDenseTensor(); + simple_tensor = expDenseTensor3(); + fill_tensor = expDenseFillTensor(); } - setTensor(4, *emptytensor); - setTensor(3, *simpletensor); - setTensor(2, *filltensor); + set_empty_tensor(4); + set_tensor(3, simple_tensor); + set_tensor(2, fill_tensor); clearTensor(2); - setTensor(2, *filltensor); + set_tensor(2, fill_tensor); search::attribute::Status oldStatus = getStatus(); search::attribute::Status newStatus = oldStatus; uint64_t iter = 0; uint64_t iterLimit = 100000; for (; iter < iterLimit; ++iter) { clearTensor(2); - setTensor(2, *filltensor); + set_tensor(2, fill_tensor); newStatus = getStatus(); if (newStatus.getUsed() < oldStatus.getUsed()) { break; @@ -290,9 +395,9 @@ Fixture::testCompaction() "iter = %" PRIu64 ", memory usage %" PRIu64 ", -> %" PRIu64, iter, oldStatus.getUsed(), newStatus.getUsed()); TEST_DO(assertGetNoTensor(1)); - TEST_DO(assertGetTensor(*filltensor, 2)); - TEST_DO(assertGetTensor(*simpletensor, 3)); - TEST_DO(assertGetTensor(*emptyxytensor, 4)); + TEST_DO(assertGetTensor(fill_tensor, 2)); + TEST_DO(assertGetTensor(simple_tensor, 3)); + TEST_DO(assertGetTensor(empty_xy_tensor, 4)); } void @@ -357,4 +462,73 @@ TEST("Test dense tensors with dense tensor attribute") testAll([]() { return std::make_shared<Fixture>(denseSpec, true); }); } +TEST_F("Hnsw index is NOT instantiated in dense tensor attribute by default", + Fixture(vec_2d_spec, true, false)) +{ + const auto& tensor = f.as_dense_tensor(); + EXPECT_TRUE(tensor.nearest_neighbor_index() == nullptr); +} + +TEST_F("Hnsw index is instantiated in dense tensor attribute when specified in config", + Fixture(vec_2d_spec, true, true)) +{ + const auto& tensor = f.as_dense_tensor(); + ASSERT_TRUE(tensor.nearest_neighbor_index() != nullptr); + auto hnsw_index = dynamic_cast<const HnswIndex*>(tensor.nearest_neighbor_index()); + ASSERT_TRUE(hnsw_index != nullptr); + + const auto& cfg = hnsw_index->config(); + EXPECT_EQUAL(8u, cfg.max_links_at_level_0()); + EXPECT_EQUAL(4u, cfg.max_links_at_hierarchic_levels()); + EXPECT_EQUAL(20u, cfg.neighbors_to_explore_at_construction()); + EXPECT_TRUE(cfg.heuristic_select_neighbors()); +} + +class DenseTensorAttributeMockIndex : public Fixture { +public: + DenseTensorAttributeMockIndex() : Fixture(vec_2d_spec, true, true, true) {} +}; + +TEST_F("setTensor() updates nearest neighbor index", DenseTensorAttributeMockIndex) +{ + auto& index = f.mock_index(); + + f.set_tensor(1, vec_2d(3, 5)); + index.expect_add(1, {3, 5}); + index.expect_empty_remove(); + index.clear(); + + // Replaces previous value. + f.set_tensor(1, vec_2d(7, 9)); + index.expect_remove(1, {3, 5}); + index.expect_add(1, {7, 9}); +} + +TEST_F("clearDoc() updates nearest neighbor index", DenseTensorAttributeMockIndex) +{ + auto& index = f.mock_index(); + + // Nothing to clear. + f.clearTensor(1); + index.expect_empty_remove(); + index.expect_empty_add(); + + // Clears previous value. + f.set_tensor(1, vec_2d(3, 5)); + index.clear(); + f.clearTensor(1); + index.expect_remove(1, {3, 5}); + index.expect_empty_add(); +} + +TEST_F("onLoad() updates nearest neighbor index", DenseTensorAttributeMockIndex) +{ + f.set_tensor(1, vec_2d(3, 5)); + f.set_tensor(2, vec_2d(7, 9)); + f.save(); + f.load(); + auto& index = f.mock_index(); + index.expect_adds({{1, {3, 5}}, {2, {7, 9}}}); +} + TEST_MAIN() { TEST_RUN_ALL(); vespalib::unlink("test.dat"); } diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/.gitignore b/searchlib/src/tests/common/sequencedtaskexecutor/.gitignore index 35d038b0b7c..4bd94f124fb 100644 --- a/searchlib/src/tests/common/sequencedtaskexecutor/.gitignore +++ b/searchlib/src/tests/common/sequencedtaskexecutor/.gitignore @@ -1 +1,2 @@ searchlib_sequencedtaskexecutor_test_app +searchlib_sequencedtaskexecutor_benchmark_app diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt b/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt index 6ba30a1647f..6c593d20683 100644 --- a/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt +++ b/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt @@ -1,4 +1,11 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_sequencedtaskexecutor_benchmark_app TEST + SOURCES + sequencedtaskexecutor_benchmark.cpp + DEPENDS + searchlib +) + vespa_add_executable(searchlib_sequencedtaskexecutor_test_app TEST SOURCES sequencedtaskexecutor_test.cpp diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp b/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp new file mode 100644 index 00000000000..a51becfbf13 --- /dev/null +++ b/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp @@ -0,0 +1,24 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchlib/common/sequencedtaskexecutor.h> +#include <vespa/vespalib/util/lambdatask.h> +#include <atomic> + +using search::SequencedTaskExecutor; +using ExecutorId = search::ISequencedTaskExecutor::ExecutorId; + +int main(int argc, char *argv[]) { + unsigned long numTasks = 1000000; + unsigned numThreads = 4; + std::atomic<long> counter(0); + if (argc > 1) + numTasks = atol(argv[1]); + if (argc > 2) + numThreads = atoi(argv[2]); + + SequencedTaskExecutor executor(numThreads); + for (unsigned long tid(0); tid < numTasks; tid++) { + executor.executeTask(ExecutorId(tid%numThreads), vespalib::makeLambdaTask([&counter] { counter++; })); + } + return 0; +} diff --git a/searchlib/src/tests/features/prod_features.cpp b/searchlib/src/tests/features/prod_features.cpp index 56aaf2dcbc9..de436dffff1 100644 --- a/searchlib/src/tests/features/prod_features.cpp +++ b/searchlib/src/tests/features/prod_features.cpp @@ -9,16 +9,15 @@ #include <vespa/searchlib/attribute/floatbase.h> #include <vespa/searchlib/attribute/integerbase.h> #include <vespa/searchlib/attribute/stringbase.h> +#include <vespa/searchlib/attribute/singleboolattribute.h> #include <vespa/searchlib/features/agefeature.h> #include <vespa/searchlib/features/array_parser.hpp> #include <vespa/searchlib/features/attributefeature.h> -#include <vespa/searchlib/features/attributematchfeature.h> #include <vespa/searchlib/features/closenessfeature.h> #include <vespa/searchlib/features/distancefeature.h> #include <vespa/searchlib/features/dotproductfeature.h> #include <vespa/searchlib/features/fieldlengthfeature.h> #include <vespa/searchlib/features/fieldmatchfeature.h> -#include <vespa/searchlib/features/fieldtermmatchfeature.h> #include <vespa/searchlib/features/firstphasefeature.h> #include <vespa/searchlib/features/foreachfeature.h> #include <vespa/searchlib/features/freshnessfeature.h> @@ -35,7 +34,6 @@ #include <vespa/searchlib/features/setup.h> #include <vespa/searchlib/features/termfeature.h> #include <vespa/searchlib/features/utils.h> -#include <vespa/searchlib/features/valuefeature.h> #include <vespa/searchlib/features/weighted_set_parser.hpp> #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/indexproperties.h> @@ -60,6 +58,7 @@ using search::AttributeFactory; using search::IntegerAttribute; using search::FloatingPointAttribute; using search::StringAttribute; +using search::SingleBoolAttribute; using search::WeightedSetStringExtAttribute; using search::attribute::WeightedEnumContent; @@ -212,7 +211,7 @@ Test::setupForAgeTest(FtFeatureTest & ft, uint64_t docTime) doctime->addReservedDoc(); doctime->addDocs(1); ft.getIndexEnv().getAttributeMap().add(doctime); - (static_cast<IntegerAttribute *>(doctime.get()))->update(1, docTime); + (dynamic_cast<IntegerAttribute *>(doctime.get()))->update(1, docTime); doctime->commit(); } @@ -240,7 +239,12 @@ Test::testAttribute() RankResult exp; exp.addScore("attribute(sint)", 10). addScore("attribute(sint,0)", 10). + addScore("attribute(slong)", 20). + addScore("attribute(sbyte)", 37). + addScore("attribute(sbool)", 1). + addScore("attribute(sebool)", 0). addScore("attribute(sfloat)", 60.5f). + addScore("attribute(sdouble)", 67.5f). addScore("attribute(sstr)", (feature_t)vespalib::hash_code("foo")). addScore("attribute(sint).count", 1). addScore("attribute(sfloat).count", 1). @@ -250,12 +254,18 @@ Test::testAttribute() addScore("attribute(udefstr)", (feature_t)vespalib::hash_code("")); FtFeatureTest ft(_factory, exp.getKeys()); - ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sint"). - addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sfloat"). - addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sstr"). - addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefint"). - addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udeffloat"). - addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefstr"); + ft.getIndexEnv().getBuilder() + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sint") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "slong") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sbyte") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sbool") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sebool") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sfloat") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sdouble") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sstr") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefint") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udeffloat") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefstr"); setupForAttributeTest(ft); ASSERT_TRUE(ft.setup()); ASSERT_TRUE(ft.execute(exp)); @@ -370,6 +380,11 @@ Test::setupForAttributeTest(FtFeatureTest &ft, bool setup_env) avs.push_back(AttributeFactory::createAttribute("udefint", AVC(AVBT::INT32, AVCT::SINGLE))); // 9 avs.push_back(AttributeFactory::createAttribute("udeffloat", AVC(AVBT::FLOAT, AVCT::SINGLE))); // 10 avs.push_back(AttributeFactory::createAttribute("udefstr", AVC(AVBT::STRING, AVCT::SINGLE))); // 11 + avs.push_back(AttributeFactory::createAttribute("sbyte", AVC(AVBT::INT64, AVCT::SINGLE))); // 12 + avs.push_back(AttributeFactory::createAttribute("slong", AVC(AVBT::INT64, AVCT::SINGLE))); // 13 + avs.push_back(AttributeFactory::createAttribute("sbool", AVC(AVBT::BOOL, AVCT::SINGLE))); // 14 + avs.push_back(AttributeFactory::createAttribute("sebool", AVC(AVBT::BOOL, AVCT::SINGLE))); // 15 + avs.push_back(AttributeFactory::createAttribute("sdouble", AVC(AVBT::DOUBLE, AVCT::SINGLE))); // 16 // simulate a unique only attribute as specified in sd AVC cfg(AVBT::INT32, AVCT::SINGLE); @@ -391,36 +406,46 @@ Test::setupForAttributeTest(FtFeatureTest &ft, bool setup_env) .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefint") .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udeffloat") .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefstr") - .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "unique"); + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "unique") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "slong") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sdouble") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sbyte") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sbool") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sebool"); } - for (uint32_t i = 0; i < avs.size(); ++i) { - avs[i]->addReservedDoc(); - avs[i]->addDocs(1); - ft.getIndexEnv().getAttributeMap().add(avs[i]); + for (const auto & attr : avs) { + attr->addReservedDoc(); + attr->addDocs(1); + ft.getIndexEnv().getAttributeMap().add(attr); } // integer attributes - (static_cast<IntegerAttribute *>(avs[0].get()))->update(1, 10); - (static_cast<IntegerAttribute *>(avs[1].get()))->append(1, 20, 0); - (static_cast<IntegerAttribute *>(avs[1].get()))->append(1, 30, 0); - (static_cast<IntegerAttribute *>(avs[2].get()))->append(1, 40, 10); - (static_cast<IntegerAttribute *>(avs[2].get()))->append(1, 50, 20); - (static_cast<IntegerAttribute *>(avs[9].get()))->update(1, search::attribute::getUndefined<int32_t>()); + (dynamic_cast<IntegerAttribute *>(avs[0].get()))->update(1, 10); + (dynamic_cast<IntegerAttribute *>(avs[12].get()))->update(1, 37); + (dynamic_cast<IntegerAttribute *>(avs[13].get()))->update(1, 20); + (dynamic_cast<SingleBoolAttribute *>(avs[14].get()))->update(1, 1); + (dynamic_cast<SingleBoolAttribute *>(avs[15].get()))->update(1, 0); + (dynamic_cast<IntegerAttribute *>(avs[1].get()))->append(1, 20, 0); + (dynamic_cast<IntegerAttribute *>(avs[1].get()))->append(1, 30, 0); + (dynamic_cast<IntegerAttribute *>(avs[2].get()))->append(1, 40, 10); + (dynamic_cast<IntegerAttribute *>(avs[2].get()))->append(1, 50, 20); + (dynamic_cast<IntegerAttribute *>(avs[9].get()))->update(1, search::attribute::getUndefined<int32_t>()); // feature_t attributes - (static_cast<FloatingPointAttribute *>(avs[3].get()))->update(1, 60.5f); - (static_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 70.5f, 0); - (static_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 80.5f, 0); - (static_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 90.5f, -30); - (static_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 100.5f, -40); - (static_cast<FloatingPointAttribute *>(avs[10].get()))->update(1, search::attribute::getUndefined<float>()); + (dynamic_cast<FloatingPointAttribute *>(avs[3].get()))->update(1, 60.5f); + (dynamic_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 70.5f, 0); + (dynamic_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 80.5f, 0); + (dynamic_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 90.5f, -30); + (dynamic_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 100.5f, -40); + (dynamic_cast<FloatingPointAttribute *>(avs[10].get()))->update(1, search::attribute::getUndefined<float>()); + (dynamic_cast<FloatingPointAttribute *>(avs[16].get()))->update(1, 67.5); // string attributes - (static_cast<StringAttribute *>(avs[6].get()))->update(1, "foo"); - (static_cast<StringAttribute *>(avs[7].get()))->append(1, "bar", 0); - (static_cast<StringAttribute *>(avs[7].get()))->append(1, "baz", 0); - (static_cast<StringAttribute *>(avs[8].get()))->append(1, "qux", 11); - (static_cast<StringAttribute *>(avs[8].get()))->append(1, "quux", 12); - (static_cast<StringAttribute *>(avs[11].get()))->update(1, ""); + (dynamic_cast<StringAttribute *>(avs[6].get()))->update(1, "foo"); + (dynamic_cast<StringAttribute *>(avs[7].get()))->append(1, "bar", 0); + (dynamic_cast<StringAttribute *>(avs[7].get()))->append(1, "baz", 0); + (dynamic_cast<StringAttribute *>(avs[8].get()))->append(1, "qux", 11); + (dynamic_cast<StringAttribute *>(avs[8].get()))->append(1, "quux", 12); + (dynamic_cast<StringAttribute *>(avs[11].get()))->update(1, ""); for (uint32_t i = 0; i < avs.size() - 1; ++i) { // do not commit the noupdate attribute avs[i]->commit(); @@ -475,7 +500,7 @@ Test::assertCloseness(feature_t exp, const vespalib::string & attr, double dista FtFeatureTest ft(_factory, feature); std::vector<std::pair<int32_t, int32_t> > positions; int32_t x = 0; - positions.push_back(std::make_pair(x, x)); + positions.emplace_back(x, x); setupForDistanceTest(ft, "pos", positions, false); ft.getQueryEnv().getLocation().setXPosition((int)distance); ft.getQueryEnv().getLocation().setValid(true); @@ -572,7 +597,7 @@ Test::assertFieldMatch(const vespalib::string & spec, const vespalib::string & field, uint32_t totalTermWeight) { - assertFieldMatch(spec, query, field, NULL, totalTermWeight); + assertFieldMatch(spec, query, field, nullptr, totalTermWeight); } void @@ -581,7 +606,7 @@ Test::assertFieldMatchTS(const vespalib::string & spec, const vespalib::string & field, feature_t totalSignificance) { - assertFieldMatch(spec, query, field, NULL, 0, totalSignificance); + assertFieldMatch(spec, query, field, nullptr, 0, totalSignificance); } @@ -871,12 +896,12 @@ Test::setupForDistanceTest(FtFeatureTest &ft, const vespalib::string & attrName, pos->addDocs(1); ft.getIndexEnv().getAttributeMap().add(pos); - IntegerAttribute * ia = static_cast<IntegerAttribute *>(pos.get()); - for (uint32_t i = 0; i < positions.size(); ++i) { + auto ia = dynamic_cast<IntegerAttribute *>(pos.get()); + for (const auto & p : positions) { if (zcurve) { - ia->append(1, vespalib::geo::ZCurve::encode(positions[i].first, positions[i].second), 0); + ia->append(1, vespalib::geo::ZCurve::encode(p.first, p.second), 0); } else { - ia->append(1, positions[i].first, 0); + ia->append(1, p.first, 0); } } @@ -891,11 +916,11 @@ Test::assert2DZDistance(feature_t exp, const vespalib::string & positions, FtFeatureTest ft(_factory, "distance(pos)"); std::vector<vespalib::string> ta = FtUtil::tokenize(positions, ","); std::vector<std::pair<int32_t, int32_t> > pos; - for (uint32_t i = 0; i < ta.size(); ++i) { - std::vector<vespalib::string> tb = FtUtil::tokenize(ta[i], ":"); - int32_t x = util::strToNum<int32_t>(tb[0]); - int32_t y = util::strToNum<int32_t>(tb[1]); - pos.push_back(std::make_pair(x, y)); + for (const auto & s : ta) { + std::vector<vespalib::string> tb = FtUtil::tokenize(s, ":"); + auto x = util::strToNum<int32_t>(tb[0]); + auto y = util::strToNum<int32_t>(tb[1]); + pos.emplace_back(x, y); } setupForDistanceTest(ft, "pos", pos, true); ft.getQueryEnv().getLocation().setXPosition(xquery); @@ -927,7 +952,7 @@ Test::testDistanceToPath() { // Test executor. std::vector<std::pair<int32_t, int32_t> > pos; - pos.push_back(std::make_pair(0, 0)); + pos.emplace_back(0, 0); // invalid path assertDistanceToPath(pos, "a"); @@ -965,7 +990,7 @@ Test::testDistanceToPath() assertDistanceToPath(pos, "(-3,2,2,2,2,-1,0,-1)", 1, 1, 2); // multiple document locations - pos.push_back(std::make_pair(0, 1)); + pos.emplace_back(0, 1); assertDistanceToPath(pos, "(-1,1,1,1)", 0, 0.5); assertDistanceToPath(pos, "(-2,-1,-1,1)", 1, 1, 2); assertDistanceToPath(pos, "(-1,0.25,1,0.25)", 0.25, 0.5, 0.5); @@ -1017,7 +1042,7 @@ Test::testDistanceToPath() } void -Test::assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > pos, +Test::assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > & pos, const vespalib::string &path, feature_t distance, feature_t traveled, feature_t product) { LOG(info, "Testing distance to path '%s' with %zd document locations.", path.c_str(), pos.size()); @@ -1033,20 +1058,6 @@ Test::assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > pos, .addScore("distanceToPath(pos).product", product))); } -void -Test::setupForDocumentTest(FtFeatureTest &ft, const vespalib::string & attrName, const vespalib::string & docType) -{ - AttributePtr type = AttributeFactory::createAttribute(attrName, AVC(AVBT::STRING, AVCT::SINGLE)); - - type->addReservedDoc(); - type->addDocs(1); - ft.getIndexEnv().getAttributeMap().add(type); - - (static_cast<StringAttribute *>(type.get()))->update(1, docType); - type->commit(); -} - - namespace { void @@ -1264,6 +1275,8 @@ void Test::setupForDotProductTest(FtFeatureTest & ft) { struct Config { + Config() : name(nullptr), dataType(AVBT::BOOL), collectionType(AVCT::SINGLE), fastSearch(false) {} + Config(const char *n, AVBT dt, AVCT ct, bool fs) : name(n), dataType(dt), collectionType(ct), fastSearch(fs) {} const char * name; AVBT dataType; AVCT collectionType; @@ -1296,11 +1309,11 @@ Test::setupForDotProductTest(FtFeatureTest & ft) baf->addDocs(2); ft.getIndexEnv().getAttributeMap().add(baf); for (size_t i(1); i < 6; i++) { - IntegerAttribute * ia = dynamic_cast<IntegerAttribute *>(baf.get()); + auto ia = dynamic_cast<IntegerAttribute *>(baf.get()); if (ia) { ia->append(1, i, i); } else { - FloatingPointAttribute * fa = dynamic_cast<FloatingPointAttribute *>(baf.get()); + auto fa = dynamic_cast<FloatingPointAttribute *>(baf.get()); fa->append(1, i, i); } } @@ -1315,14 +1328,14 @@ Test::setupForDotProductTest(FtFeatureTest & ft) ft.getIndexEnv().getAttributeMap().add(c); ft.getIndexEnv().getAttributeMap().add(d); - StringAttribute * sa = static_cast<StringAttribute *>(a.get()); + auto sa = dynamic_cast<StringAttribute *>(a.get()); sa->append(1, "a", 1); sa->append(1, "b", 2); sa->append(1, "c", 3); sa->append(1, "d", 4); sa->append(1, "e", 5); - WeightedSetStringExtAttribute * ea = static_cast<WeightedSetStringExtAttribute *>(d.get()); + auto ea = dynamic_cast<WeightedSetStringExtAttribute *>(d.get()); EXPECT_TRUE(!ea->hasEnum()); uint32_t docId; ea->addDoc(docId); // reserved doc @@ -1517,9 +1530,9 @@ Test::testMatchCount() ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "foo"); ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "bar"); ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "baz"); - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL); // query term 0, hit in foo - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != NULL); // query term 1, hit in bar - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL); // query term 2, hit in foo + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr); // query term 0, hit in foo + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != nullptr); // query term 1, hit in bar + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr); // query term 2, hit in foo ASSERT_TRUE(ft.setup()); MatchDataBuilder::UP mdb = ft.createMatchDataBuilder(); @@ -1577,9 +1590,9 @@ Test::testMatches() ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "foo"); ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "bar"); ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "baz"); - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL); // query term 0, hit in foo - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != NULL); // query term 1, hit in bar - ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL); // query term 2, hit in foo + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr); // query term 0, hit in foo + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != nullptr); // query term 1, hit in bar + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr); // query term 2, hit in foo ASSERT_TRUE(ft.setup()); MatchDataBuilder::UP mdb = ft.createMatchDataBuilder(); @@ -1613,11 +1626,9 @@ Test::assertMatches(uint32_t output, ASSERT_TRUE(ft.execute(output, EPS, docId)); // Execute and compare results. - if (!EXPECT_TRUE(ft.execute(output, EPS, docId))) return false; - return true; + return EXPECT_TRUE(ft.execute(output, EPS, docId)); } - void Test::testQuery() { @@ -1721,7 +1732,7 @@ Test::testRandom() search::Rand48 rnd; rnd.srand48(100); for (uint32_t i = 0; i < 5; ++i) { - feature_t exp = rnd.lrand48() / (feature_t)0x80000000u; + feature_t exp = static_cast<feature_t>(rnd.lrand48()) / static_cast<feature_t>(0x80000000u); ASSERT_TRUE(ft.execute(exp, EPS, i + 1)); } } @@ -1744,7 +1755,7 @@ Test::testRandom() search::Rand48 rnd; for (uint32_t i = 1; i <= 5; ++i) { rnd.srand48(100 + i); // seed + lid - feature_t exp = rnd.lrand48() / (feature_t)0x80000000u; + feature_t exp = static_cast<feature_t>(rnd.lrand48()) / static_cast<feature_t>(0x80000000u); ASSERT_TRUE(ft.execute(exp, EPS, i)); } } @@ -2067,10 +2078,7 @@ Test::assertTermDistance(const TermDistanceCalculator::Result & exp, rr.addScore(feature + ".forwardTermPosition", exp.forwardTermPos); rr.addScore(feature + ".reverse", exp.reverseDist); rr.addScore(feature + ".reverseTermPosition", exp.reverseTermPos); - if (!EXPECT_TRUE(ft.execute(rr, docId))) { - return false; - } - return true; + return EXPECT_TRUE(ft.execute(rr, docId)); } void diff --git a/searchlib/src/tests/features/prod_features.h b/searchlib/src/tests/features/prod_features.h index 8e6578f34bd..6d30dd3fcd8 100644 --- a/searchlib/src/tests/features/prod_features.h +++ b/searchlib/src/tests/features/prod_features.h @@ -10,10 +10,10 @@ class Test : public FtTestApp { public: Test(); - ~Test(); + ~Test() override; int Main() override; void testFramework(); - void testFtLib(); + static void testFtLib(); void testAge(); void testAttribute(); void testAttributeMatch(); @@ -39,10 +39,9 @@ public: void testRankingExpression(); void testTerm(); void testTermDistance(); - void testUtils(); + static void testUtils(); static void setupForDotProductTest(FtFeatureTest & ft); - static void setupForDocumentTest(FtFeatureTest &ft, const vespalib::string & attrName, const vespalib::string & docType); private: void testFieldMatchBluePrint(); @@ -81,21 +80,21 @@ private: void testFieldMatchExecutorRemaining(); void assertAge(feature_t expAge, const vespalib::string & attr, uint64_t now, uint64_t docTime); - void setupForAgeTest(FtFeatureTest & ft, uint64_t docTime); - void setupForAttributeTest(FtFeatureTest &ft, bool setup_env = true); + static void setupForAgeTest(FtFeatureTest & ft, uint64_t docTime); + static void setupForAttributeTest(FtFeatureTest &ft, bool setup_env = true); void assertCloseness(feature_t exp, const vespalib::string & attr, double distance, double maxDistance = 0, double halfResponse = 0); - void setupForDistanceTest(FtFeatureTest & ft, const vespalib::string & attrName, - const std::vector<std::pair<int32_t, int32_t> > & positions, bool zcurve); + static void setupForDistanceTest(FtFeatureTest & ft, const vespalib::string & attrName, + const std::vector<std::pair<int32_t, int32_t> > & positions, bool zcurve); void assert2DZDistance(feature_t exp, const vespalib::string & positions, int32_t xquery, int32_t yquery, uint32_t xAspect = 0); - void assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > pos, const vespalib::string &path, + void assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > & pos, const vespalib::string &path, feature_t distance = search::features::DistanceToPathExecutor::DEFAULT_DISTANCE, feature_t traveled = 1, feature_t product = 0); void assertDotProduct(feature_t exp, const vespalib::string & vector, uint32_t docId = 1, const vespalib::string & attribute = "wsstr", const vespalib::string & attributeOverride=""); void assertFieldMatch(const vespalib::string & spec, const vespalib::string & query, const vespalib::string & field, - const search::features::fieldmatch::Params * params = NULL, uint32_t totalTermWeight = 0, feature_t totalSignificance = 0.0f); + const search::features::fieldmatch::Params * params = nullptr, uint32_t totalTermWeight = 0, feature_t totalSignificance = 0.0f); void assertFieldMatch(const vespalib::string & spec, const vespalib::string & query, const vespalib::string & field, uint32_t totalTermWeight); void assertFieldMatchTS(const vespalib::string & spec, const vespalib::string & query, const vespalib::string & field, diff --git a/searchlib/src/tests/fef/resolver/resolver_test.cpp b/searchlib/src/tests/fef/resolver/resolver_test.cpp index 955c072810f..7591d028620 100644 --- a/searchlib/src/tests/fef/resolver/resolver_test.cpp +++ b/searchlib/src/tests/fef/resolver/resolver_test.cpp @@ -4,10 +4,13 @@ #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/fef/test/indexenvironment.h> #include <vespa/searchlib/features/valuefeature.h> +#include <vespa/searchlib/features/rankingexpressionfeature.h> #include <vespa/log/log.h> LOG_SETUP("resolver_test"); +using search::features::RankingExpressionBlueprint; + namespace search { namespace fef { @@ -58,6 +61,7 @@ class Test : public vespalib::TestApp { private: BlueprintFactory _factory; void requireThatWeGetUniqueBlueprints(); + void require_that_bad_input_is_handled(); public: Test(); ~Test(); @@ -69,6 +73,7 @@ Test::Test() : { _factory.addPrototype(Blueprint::SP(new BaseBlueprint())); _factory.addPrototype(Blueprint::SP(new CombineBlueprint())); + _factory.addPrototype(std::make_shared<RankingExpressionBlueprint>()); } Test::~Test() {} @@ -85,12 +90,28 @@ Test::requireThatWeGetUniqueBlueprints() EXPECT_TRUE(dynamic_cast<CombineBlueprint *>(spec[1].blueprint.get()) != NULL); } +void +Test::require_that_bad_input_is_handled() +{ + test::IndexEnvironment ienv; + ienv.getProperties().add(indexproperties::eval::LazyExpressions::NAME, "false"); + ienv.getProperties().add("rankingExpression(badinput).rankingScript", "base.foobad + base.bar"); + BlueprintResolver::SP res(new BlueprintResolver(_factory, ienv)); + res->addSeed("rankingExpression(badinput)"); + EXPECT_FALSE(res->compile()); + const BlueprintResolver::ExecutorSpecList & spec = res->getExecutorSpecs(); + EXPECT_EQUAL(2u, spec.size()); + EXPECT_TRUE(dynamic_cast<BaseBlueprint *>(spec[0].blueprint.get()) != nullptr); + EXPECT_TRUE(dynamic_cast<RankingExpressionBlueprint *>(spec[1].blueprint.get()) != nullptr); +} + int Test::Main() { TEST_INIT("resolver_test"); requireThatWeGetUniqueBlueprints(); + require_that_bad_input_is_handled(); TEST_DONE(); } diff --git a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp index 7bc582ab442..ee8d4b787bd 100644 --- a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp +++ b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp @@ -12,6 +12,7 @@ #include <vespa/searchlib/queryeval/simpleresult.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/searchlib/queryeval/nns_index_iterator.h> #include <vespa/log/log.h> LOG_SETUP("nearest_neighbor_test"); @@ -190,4 +191,70 @@ TEST("require that NearestNeighborIterator sets expected rawscore") { TEST_DO(verify_iterator_sets_expected_rawscore(denseSpecFloat, denseSpecDouble)); } +TEST("require that NnsIndexIterator works as expected") { + std::vector<NnsIndexIterator::Hit> hits{{2,4.0}, {3,9.0}, {5,1.0}, {8,16.0}, {9,36.0}}; + auto md = MatchData::makeTestInstance(2, 2); + auto &tfmd = *(md->resolveTermField(0)); + auto search = NnsIndexIterator::create(tfmd, hits); + uint32_t docid = 1; + search->initFullRange(); + bool match = search->seek(docid); + EXPECT_FALSE(match); + EXPECT_FALSE(search->isAtEnd()); + EXPECT_EQUAL(2u, search->getDocId()); + docid = 2; + match = search->seek(docid); + EXPECT_TRUE(match); + EXPECT_FALSE(search->isAtEnd()); + EXPECT_EQUAL(docid, search->getDocId()); + search->unpack(docid); + EXPECT_EQUAL(2.0, tfmd.getRawScore()); + + docid = 3; + match = search->seek(docid); + EXPECT_TRUE(match); + EXPECT_FALSE(search->isAtEnd()); + EXPECT_EQUAL(docid, search->getDocId()); + search->unpack(docid); + EXPECT_EQUAL(3.0, tfmd.getRawScore()); + + docid = 4; + match = search->seek(docid); + EXPECT_FALSE(match); + EXPECT_FALSE(search->isAtEnd()); + EXPECT_EQUAL(5u, search->getDocId()); + + docid = 6; + match = search->seek(docid); + EXPECT_FALSE(match); + EXPECT_FALSE(search->isAtEnd()); + EXPECT_EQUAL(8u, search->getDocId()); + docid = 8; + search->unpack(docid); + EXPECT_EQUAL(4.0, tfmd.getRawScore()); + docid = 9; + match = search->seek(docid); + EXPECT_TRUE(match); + EXPECT_FALSE(search->isAtEnd()); + docid = 10; + match = search->seek(docid); + EXPECT_FALSE(match); + EXPECT_TRUE(search->isAtEnd()); + + docid = 4; + search->initRange(docid, 7); + match = search->seek(docid); + EXPECT_FALSE(match); + EXPECT_FALSE(search->isAtEnd()); + EXPECT_EQUAL(5u, search->getDocId()); + docid = 5; + search->unpack(docid); + EXPECT_EQUAL(1.0, tfmd.getRawScore()); + EXPECT_FALSE(search->isAtEnd()); + docid = 6; + match = search->seek(docid); + EXPECT_FALSE(match); + EXPECT_TRUE(search->isAtEnd()); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp b/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp index b10da86dd8c..861af3527ca 100644 --- a/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp +++ b/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.h> #include <vespa/searchlib/fef/test/indexenvironment.h> #include <vespa/searchlib/fef/test/queryenvironment.h> +#include <vespa/vespalib/util/stash.h> #include <set> using namespace search::features::rankingexpression; diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp index c6246bb8434..52f45860c1e 100644 --- a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -48,8 +48,7 @@ using HnswIndexUP = std::unique_ptr<HnswIndex>; class HnswIndexTest : public ::testing::Test { public: FloatVectors vectors; - FloatSqEuclideanDistance distance_func; - LevelGenerator level_generator; + LevelGenerator* level_generator; HnswIndexUP index; HnswIndexTest() @@ -62,11 +61,14 @@ public: .set(7, {3, 5}).set(8, {0, 3}).set(9, {4, 5}); } void init(bool heuristic_select_neighbors) { - index = std::make_unique<HnswIndex>(vectors, distance_func, level_generator, + auto generator = std::make_unique<LevelGenerator>(); + level_generator = generator.get(); + index = std::make_unique<HnswIndex>(vectors, std::make_unique<FloatSqEuclideanDistance>(), + std::move(generator), HnswIndex::Config(2, 1, 10, heuristic_select_neighbors)); } void add_document(uint32_t docid, uint32_t max_level = 0) { - level_generator.level = max_level; + level_generator->level = max_level; index->add_document(docid); } void remove_document(uint32_t docid) { @@ -100,8 +102,10 @@ public: if (exp_hits.size() == k) { std::vector<uint32_t> expected_by_docid = exp_hits; std::sort(expected_by_docid.begin(), expected_by_docid.end()); - std::vector<uint32_t> got_by_docid = index->find_top_k(k, qv, k); - EXPECT_EQ(expected_by_docid, got_by_docid); + auto got_by_docid = index->find_top_k(k, qv, k); + for (idx = 0; idx < k; ++idx) { + EXPECT_EQ(expected_by_docid[idx], got_by_docid[idx].docid); + } } } }; @@ -262,5 +266,38 @@ TEST_F(HnswIndexTest, 2d_vectors_inserted_in_hierarchic_graph_with_heuristic_sel expect_level_0(7, {3, 6}); } +TEST_F(HnswIndexTest, manual_insert) +{ + init(false); + + std::vector<uint32_t> nbl; + HnswNode empty{nbl}; + index->set_node(1, empty); + index->set_node(2, empty); + + HnswNode three{{1,2}}; + index->set_node(3, three); + expect_level_0(1, {3}); + expect_level_0(2, {3}); + expect_level_0(3, {1,2}); + + expect_entry_point(1, 0); + + HnswNode twolevels{{{1},nbl}}; + index->set_node(4, twolevels); + + expect_entry_point(4, 1); + expect_level_0(1, {3,4}); + + HnswNode five{{{1,2}, {4}}}; + index->set_node(5, five); + + expect_levels(1, {{3,4,5}}); + expect_levels(2, {{3,5}}); + expect_levels(3, {{1,2}}); + expect_levels(4, {{1}, {5}}); + expect_levels(5, {{1,2}, {4}}); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp index 535e81fc032..10e1a1edb52 100644 --- a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp @@ -73,6 +73,10 @@ ConfigConverter::convert(const AttributesConfig::Attribute & cfg) predicateParams.setBounds(cfg.lowerbound, cfg.upperbound); predicateParams.setDensePostingListThreshold(cfg.densepostinglistthreshold); retval.setPredicateParams(predicateParams); + if (cfg.index.hnsw.enabled) { + retval.set_hnsw_index_params(HnswIndexParams(cfg.index.hnsw.maxlinkspernode, + cfg.index.hnsw.neighborstoexploreatinsert)); + } if (retval.basicType().type() == BasicType::Type::TENSOR) { if (!cfg.tensortype.empty()) { retval.setTensorType(ValueType::from_spec(cfg.tensortype)); diff --git a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp index bb779b659ab..617e4bf5c85 100644 --- a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp +++ b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp @@ -2,7 +2,6 @@ #include "sequencedtaskexecutor.h" #include <vespa/vespalib/util/blockingthreadstackexecutor.h> -#include <vespa/vespalib/stllike/hash_map.hpp> using vespalib::BlockingThreadStackExecutor; diff --git a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h index 2b7e70d69c7..9337f393150 100644 --- a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h +++ b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h @@ -2,7 +2,6 @@ #pragma once #include "isequencedtaskexecutor.h" -#include <vespa/vespalib/stllike/hash_map.h> #include <vector> namespace vespalib { @@ -16,7 +15,7 @@ namespace search { * Class to run multiple tasks in parallel, but tasks with same * id has to be run in sequence. */ -class SequencedTaskExecutor : public ISequencedTaskExecutor +class SequencedTaskExecutor final : public ISequencedTaskExecutor { using Stats = vespalib::ExecutorStats; std::vector<std::shared_ptr<vespalib::BlockingThreadStackExecutor>> _executors; diff --git a/searchlib/src/vespa/searchlib/features/agefeature.cpp b/searchlib/src/vespa/searchlib/features/agefeature.cpp index 258648408f8..e93691e7241 100644 --- a/searchlib/src/vespa/searchlib/features/agefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/agefeature.cpp @@ -4,6 +4,7 @@ #include "valuefeature.h" #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/matchdata.h> +#include <vespa/vespalib/util/stash.h> using search::attribute::IAttributeVector; @@ -18,20 +19,18 @@ AgeExecutor::AgeExecutor(const IAttributeVector *attribute) : _attribute(attribute), _buf() { - if (_attribute != NULL) { + if (_attribute != nullptr) { _buf.allocate(attribute->getMaxValueCount()); } } -AgeBlueprint::~AgeBlueprint() -{ -} +AgeBlueprint::~AgeBlueprint() = default; void AgeExecutor::execute(uint32_t docId) { feature_t age = 10000000000.0; - if (_attribute != NULL) { + if (_attribute != nullptr) { _buf.fill(*_attribute, docId); int64_t docTime = _buf[0]; feature_t currTime = inputs().get_number(0); @@ -65,7 +64,7 @@ AgeBlueprint::setup(const search::fef::IIndexEnvironment &env, search::fef::Blueprint::UP AgeBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new AgeBlueprint()); + return std::make_unique<AgeBlueprint>(); } search::fef::FeatureExecutor & diff --git a/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp b/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp index 4776437a14b..c26d18eb11c 100644 --- a/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp @@ -8,6 +8,7 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/parameterdescriptions.h> #include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/vespalib/util/stash.h> #include <vespa/log/log.h> LOG_SETUP(".features.attributematchfeature"); diff --git a/searchlib/src/vespa/searchlib/features/bm25_feature.cpp b/searchlib/src/vespa/searchlib/features/bm25_feature.cpp index e16b4bba996..64e365b25ac 100644 --- a/searchlib/src/vespa/searchlib/features/bm25_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/bm25_feature.cpp @@ -6,8 +6,8 @@ #include <vespa/searchlib/fef/itermfielddata.h> #include <vespa/searchlib/fef/objectstore.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <cmath> -#include <memory> #include <stdexcept> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/features/closenessfeature.cpp b/searchlib/src/vespa/searchlib/features/closenessfeature.cpp index 2358e54e9f5..ccd76f5435f 100644 --- a/searchlib/src/vespa/searchlib/features/closenessfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/closenessfeature.cpp @@ -3,14 +3,14 @@ #include "closenessfeature.h" #include "utils.h" #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <vespa/log/log.h> LOG_SETUP(".features.closenessfeature"); using namespace search::fef; -namespace search { -namespace features { +namespace search::features { ClosenessExecutor::ClosenessExecutor(feature_t maxDistance, feature_t scaleDistance) : FeatureExecutor(), @@ -96,7 +96,7 @@ ClosenessBlueprint::setup(const IIndexEnvironment & env, Blueprint::UP ClosenessBlueprint::createInstance() const { - return Blueprint::UP(new ClosenessBlueprint()); + return std::make_unique<ClosenessBlueprint>(); } FeatureExecutor & @@ -105,6 +105,4 @@ ClosenessBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash &s return stash.create<ClosenessExecutor>(_maxDistance, _scaleDistance); } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/constant_feature.cpp b/searchlib/src/vespa/searchlib/features/constant_feature.cpp index ced9d95fb33..7fc0c5c05fc 100644 --- a/searchlib/src/vespa/searchlib/features/constant_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/constant_feature.cpp @@ -4,14 +4,14 @@ #include "valuefeature.h" #include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/eval/eval/value_cache/constant_value.h> +#include <vespa/vespalib/util/stash.h> #include <vespa/log/log.h> LOG_SETUP(".features.constant_feature"); using namespace search::fef; -namespace search { -namespace features { +namespace search::features { /** * Feature executor that returns a constant value. @@ -25,8 +25,8 @@ public: ConstantFeatureExecutor(const vespalib::eval::Value &value) : _value(value) {} - virtual bool isPure() override { return true; } - virtual void execute(uint32_t) override { + bool isPure() override { return true; } + void execute(uint32_t) override { outputs().set_object(0, _value); } static FeatureExecutor &create(const vespalib::eval::Value &value, vespalib::Stash &stash) { @@ -41,9 +41,7 @@ ConstantBlueprint::ConstantBlueprint() { } -ConstantBlueprint::~ConstantBlueprint() -{ -} +ConstantBlueprint::~ConstantBlueprint() = default; void ConstantBlueprint::visitDumpFeatures(const IIndexEnvironment &, @@ -54,7 +52,7 @@ ConstantBlueprint::visitDumpFeatures(const IIndexEnvironment &, Blueprint::UP ConstantBlueprint::createInstance() const { - return Blueprint::UP(new ConstantBlueprint()); + return std::make_unique<ConstantBlueprint>(); } bool @@ -88,5 +86,4 @@ ConstantBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash } } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h b/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h index 2ab7b98fabe..785eb357795 100644 --- a/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h +++ b/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h @@ -7,7 +7,7 @@ #include <vespa/eval/eval/value.h> #include <vespa/eval/eval/value_type.h> #include <vespa/eval/tensor/default_tensor_engine.h> -#include <memory> +#include <vespa/vespalib/util/stash.h> namespace search::features { diff --git a/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp b/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp index 86a71184f43..8cc75a9a424 100644 --- a/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp +++ b/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp @@ -2,6 +2,8 @@ #include "debug_attribute_wait.h" #include <vespa/vespalib/util/time.h> +#include <vespa/vespalib/util/stash.h> + using search::attribute::IAttributeVector; using namespace search::fef; diff --git a/searchlib/src/vespa/searchlib/features/debug_wait.cpp b/searchlib/src/vespa/searchlib/features/debug_wait.cpp index fb002564572..57d19618ba4 100644 --- a/searchlib/src/vespa/searchlib/features/debug_wait.cpp +++ b/searchlib/src/vespa/searchlib/features/debug_wait.cpp @@ -2,6 +2,7 @@ #include "debug_wait.h" #include <vespa/vespalib/util/time.h> +#include <vespa/vespalib/util/stash.h> using namespace search::fef; diff --git a/searchlib/src/vespa/searchlib/features/distancefeature.cpp b/searchlib/src/vespa/searchlib/features/distancefeature.cpp index 501bfd7cd14..4d7d77fe315 100644 --- a/searchlib/src/vespa/searchlib/features/distancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/distancefeature.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/fef/matchdata.h> #include <vespa/document/datatype/positiondatatype.h> #include <vespa/vespalib/geo/zcurve.h> +#include <vespa/vespalib/util/stash.h> #include <cmath> #include <limits> @@ -13,13 +14,12 @@ LOG_SETUP(".features.distancefeature"); using namespace search::fef; -namespace search { -namespace features { +namespace search::features { feature_t DistanceExecutor::calculateDistance(uint32_t docId) { - if (_location.isValid() && _pos != NULL) { + if (_location.isValid() && _pos != nullptr) { return calculate2DZDistance(docId); } return DEFAULT_DISTANCE; @@ -66,7 +66,7 @@ DistanceExecutor::DistanceExecutor(const Location & location, _pos(pos), _intBuf() { - if (_pos != NULL) { + if (_pos != nullptr) { _intBuf.allocate(_pos->getMaxValueCount()); } } @@ -86,9 +86,7 @@ DistanceBlueprint::DistanceBlueprint() : { } -DistanceBlueprint::~DistanceBlueprint() -{ -} +DistanceBlueprint::~DistanceBlueprint() = default; void DistanceBlueprint::visitDumpFeatures(const IIndexEnvironment &, @@ -99,7 +97,7 @@ DistanceBlueprint::visitDumpFeatures(const IIndexEnvironment &, Blueprint::UP DistanceBlueprint::createInstance() const { - return Blueprint::UP(new DistanceBlueprint()); + return std::make_unique<DistanceBlueprint>(); } bool @@ -116,26 +114,26 @@ DistanceBlueprint::setup(const IIndexEnvironment & env, FeatureExecutor & DistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const { - const search::attribute::IAttributeVector * pos = NULL; + const search::attribute::IAttributeVector * pos = nullptr; const Location & location = env.getLocation(); LOG(debug, "DistanceBlueprint::createExecutor location.valid='%s', '%s', alternatively '%s'", location.isValid() ? "true" : "false", _posAttr.c_str(), document::PositionDataType::getZCurveFieldName(_posAttr).c_str()); if (location.isValid()) { pos = env.getAttributeContext().getAttribute(_posAttr); - if (pos == NULL) { + if (pos == nullptr) { LOG(debug, "Failed to find attribute '%s', resorting too '%s'", _posAttr.c_str(), document::PositionDataType::getZCurveFieldName(_posAttr).c_str()); pos = env.getAttributeContext().getAttribute(document::PositionDataType::getZCurveFieldName(_posAttr)); } - if (pos != NULL) { + if (pos != nullptr) { if (!pos->isIntegerType()) { LOG(warning, "The position attribute '%s' is not an integer attribute. Will use default distance.", pos->getName().c_str()); - pos = NULL; + pos = nullptr; } else if (pos->getCollectionType() == attribute::CollectionType::WSET) { LOG(warning, "The position attribute '%s' is a weighted set attribute. Will use default distance.", pos->getName().c_str()); - pos = NULL; + pos = nullptr; } } else { LOG(warning, "The position attribute '%s' was not found. Will use default distance.", _posAttr.c_str()); @@ -145,7 +143,4 @@ DistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash return stash.create<DistanceExecutor>(location, pos); } - - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp b/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp index 834f5913af9..a2f9225d3c4 100644 --- a/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp @@ -6,10 +6,10 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/document/datatype/positiondatatype.h> #include <vespa/vespalib/geo/zcurve.h> +#include <vespa/vespalib/util/stash.h> #include <boost/algorithm/string/split.hpp> #include <boost/algorithm/string/classification.hpp> #include <cmath> -#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".features.distancetopathfeature"); @@ -25,7 +25,7 @@ DistanceToPathExecutor::DistanceToPathExecutor(std::vector<Vector2> &path, _path(), _pos(pos) { - if (_pos != NULL) { + if (_pos != nullptr) { _intBuf.allocate(_pos->getMaxValueCount()); } _path.swap(path); // avoid copy @@ -34,7 +34,7 @@ DistanceToPathExecutor::DistanceToPathExecutor(std::vector<Vector2> &path, void DistanceToPathExecutor::execute(uint32_t docId) { - if (_path.size() > 1 && _pos != NULL) { + if (_path.size() > 1 && _pos != nullptr) { double pos = -1, trip = 0, product = 0; double minSqDist = std::numeric_limits<double>::max(); _intBuf.fill(*_pos, docId); @@ -145,21 +145,21 @@ DistanceToPathBlueprint::createExecutor(const search::fef::IQueryEnvironment &en } // Lookup the attribute vector that holds document positions. - const search::attribute::IAttributeVector *pos = NULL; + const search::attribute::IAttributeVector *pos = nullptr; if (path.size() > 1) { pos = env.getAttributeContext().getAttribute(_posAttr); - if (pos == NULL) { + if (pos == nullptr) { pos = env.getAttributeContext().getAttribute(document::PositionDataType::getZCurveFieldName(_posAttr)); } - if (pos != NULL) { + if (pos != nullptr) { if (!pos->isIntegerType()) { LOG(warning, "The position attribute '%s' is not an integer attribute. Will use default distance.", pos->getName().c_str()); - pos = NULL; + pos = nullptr; } else if (pos->getCollectionType() == attribute::CollectionType::WSET) { LOG(warning, "The position attribute '%s' is a weighted set attribute. Will use default distance.", pos->getName().c_str()); - pos = NULL; + pos = nullptr; } } else { LOG(warning, "The position attribute '%s' was not found. Will use default distance.", _posAttr.c_str()); diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp index 8998f01b59e..1072607aa8a 100644 --- a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp @@ -10,12 +10,11 @@ #include <vespa/searchlib/attribute/floatbase.h> #include <vespa/searchlib/attribute/multinumericattribute.h> #include <vespa/searchlib/attribute/multienumattribute.h> -#include <type_traits> - -#include <vespa/log/log.h> #include <vespa/eval/tensor/serialization/typed_binary_format.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/stash.h> +#include <vespa/log/log.h> LOG_SETUP(".features.dotproduct"); using namespace search::attribute; diff --git a/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp b/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp index 1622a87e733..1014fe4679a 100644 --- a/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/locale/c.h> +#include <vespa/vespalib/util/stash.h> #include <cassert> namespace search::features { @@ -116,7 +117,7 @@ ElementCompletenessBlueprint::visitDumpFeatures(const fef::IIndexEnvironment &en fef::Blueprint::UP ElementCompletenessBlueprint::createInstance() const { - return Blueprint::UP(new ElementCompletenessBlueprint()); + return std::make_unique<ElementCompletenessBlueprint>(); } bool diff --git a/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp b/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp index 0676b0a46c4..45fcd013fbf 100644 --- a/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/eval/eval/llvm/compiled_function.h> #include <vespa/eval/eval/llvm/compile_cache.h> +#include <vespa/vespalib/util/stash.h> #include <vespa/log/log.h> LOG_SETUP(".features.elementsimilarity"); diff --git a/searchlib/src/vespa/searchlib/features/euclidean_distance_feature.cpp b/searchlib/src/vespa/searchlib/features/euclidean_distance_feature.cpp index cd007f1396f..9d71d358b46 100644 --- a/searchlib/src/vespa/searchlib/features/euclidean_distance_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/euclidean_distance_feature.cpp @@ -4,9 +4,9 @@ #include "euclidean_distance_feature.h" #include "array_parser.hpp" #include <vespa/searchlib/attribute/integerbase.h> -#include <vespa/searchlib/attribute/floatbase.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/vespalib/util/stash.h> #include <cmath> #include <vespa/log/log.h> @@ -15,8 +15,7 @@ LOG_SETUP(".features.euclidean_distance_feature"); using namespace search::attribute; using namespace search::fef; -namespace search { -namespace features { +namespace search::features { template <typename DataType> @@ -57,7 +56,7 @@ EuclideanDistanceBlueprint::EuclideanDistanceBlueprint() : { } -EuclideanDistanceBlueprint::~EuclideanDistanceBlueprint() {} +EuclideanDistanceBlueprint::~EuclideanDistanceBlueprint() = default; void EuclideanDistanceBlueprint::visitDumpFeatures(const IIndexEnvironment &, IDumpFeatureVisitor &) const @@ -78,7 +77,7 @@ EuclideanDistanceBlueprint::setup(const IIndexEnvironment &env, const ParameterL Blueprint::UP EuclideanDistanceBlueprint::createInstance() const { - return Blueprint::UP(new EuclideanDistanceBlueprint()); + return std::make_unique<EuclideanDistanceBlueprint>(); } namespace { @@ -97,7 +96,7 @@ FeatureExecutor & EuclideanDistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const { const IAttributeVector * attribute = env.getAttributeContext().getAttribute(_attributeName); - if (attribute == NULL) { + if (attribute == nullptr) { LOG(warning, "The attribute vector '%s' was not found in the attribute manager, returning executor with default value.", _attributeName.c_str()); return stash.create<SingleZeroValueExecutor>(); @@ -118,6 +117,5 @@ EuclideanDistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespali } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp b/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp index d7e17187ff4..18a15ccf541 100644 --- a/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp @@ -7,12 +7,11 @@ #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/fieldtype.h> #include <vespa/searchlib/fef/featurenamebuilder.h> -#include <vespa/searchlib/fef/itermdata.h> #include <vespa/searchlib/fef/handle.h> +#include <vespa/vespalib/util/stash.h> #include <sstream> -namespace search { -namespace features { +namespace search::features { IndexFieldInfoExecutor::IndexFieldInfoExecutor(feature_t type, feature_t isFilter, [[maybe_unused]] uint32_t field, uint32_t fieldHandle) @@ -238,5 +237,4 @@ FieldInfoBlueprint::createExecutor(const fef::IQueryEnvironment &queryEnv, vespa return stash.create<ValueExecutor>(values); } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp b/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp index d0680e8fc19..74bfa156b5f 100644 --- a/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp @@ -6,13 +6,12 @@ #include <vespa/searchlib/fef/itermdata.h> #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> -#include <vespa/searchlib/fef/fieldtype.h> -#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> + using namespace search::fef; -namespace search { -namespace features { +namespace search::features { FieldLengthExecutor:: FieldLengthExecutor(const IQueryEnvironment &env, @@ -63,7 +62,7 @@ FieldLengthExecutor::handle_bind_match_data(const MatchData &md) FieldLengthBlueprint::FieldLengthBlueprint() : Blueprint("fieldLength"), - _field(NULL) + _field(nullptr) { } @@ -86,7 +85,7 @@ FieldLengthBlueprint::setup(const IIndexEnvironment &env, Blueprint::UP FieldLengthBlueprint::createInstance() const { - return Blueprint::UP(new FieldLengthBlueprint()); + return std::make_unique<FieldLengthBlueprint>(); } FeatureExecutor & @@ -100,4 +99,4 @@ FieldLengthBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Sta return stash.create<FieldLengthExecutor>(env, _field->id()); } -}} +} diff --git a/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp b/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp index 583afa6e698..7ad9eef0506 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp @@ -7,6 +7,7 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/locale/c.h> +#include <vespa/vespalib/util/stash.h> using namespace search::fef; using CollectionType = FieldInfo::CollectionType; @@ -92,14 +93,12 @@ FieldMatchExecutor::handle_bind_match_data(const fef::MatchData &md) FieldMatchBlueprint::FieldMatchBlueprint() : Blueprint("fieldMatch"), - _field(NULL), + _field(nullptr), _params() { } -FieldMatchBlueprint::~FieldMatchBlueprint() -{ -} +FieldMatchBlueprint::~FieldMatchBlueprint() = default; void FieldMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env, @@ -158,7 +157,7 @@ FieldMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env, Blueprint::UP FieldMatchBlueprint::createInstance() const { - return Blueprint::UP(new FieldMatchBlueprint()); + return std::make_unique<FieldMatchBlueprint>(); } bool @@ -306,5 +305,4 @@ FieldMatchBlueprint::createExecutor(const IQueryEnvironment & env, vespalib::Sta return stash.create<FieldMatchExecutor>(env, *_field, _params); } - } diff --git a/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp b/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp index a7a00bee956..2f065fee289 100644 --- a/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp @@ -8,9 +8,10 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/itermdata.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> -namespace search { -namespace features { + +namespace search::features { FieldTermMatchExecutor::FieldTermMatchExecutor(const search::fef::IQueryEnvironment &env, uint32_t fieldId, uint32_t termId) : @@ -122,7 +123,7 @@ FieldTermMatchBlueprint::setup(const search::fef::IIndexEnvironment &env, search::fef::Blueprint::UP FieldTermMatchBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new FieldTermMatchBlueprint()); + return std::make_unique<FieldTermMatchBlueprint>(); } search::fef::FeatureExecutor & @@ -131,4 +132,4 @@ FieldTermMatchBlueprint::createExecutor(const search::fef::IQueryEnvironment &en return stash.create<FieldTermMatchExecutor>(env, _fieldId, _termId); } -}} +} diff --git a/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp b/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp index 2e6bae14a44..9d1831c6102 100644 --- a/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp @@ -4,11 +4,12 @@ #include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> + using namespace search::fef; -namespace search { -namespace features { +namespace search::features { void FirstPhaseExecutor::execute(uint32_t) @@ -34,7 +35,7 @@ FirstPhaseBlueprint::visitDumpFeatures(const IIndexEnvironment &, Blueprint::UP FirstPhaseBlueprint::createInstance() const { - return Blueprint::UP(new FirstPhaseBlueprint()); + return std::make_unique<FirstPhaseBlueprint>(); } bool @@ -54,5 +55,4 @@ FirstPhaseBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash & } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp b/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp index eda83b991bf..e43faaec4e1 100644 --- a/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp @@ -7,6 +7,8 @@ #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/locale/c.h> +#include <vespa/vespalib/util/stash.h> + #include <cassert> #include <vespa/log/log.h> @@ -285,7 +287,7 @@ FlowCompletenessBlueprint::visitDumpFeatures(const fef::IIndexEnvironment &env, fef::Blueprint::UP FlowCompletenessBlueprint::createInstance() const { - return Blueprint::UP(new FlowCompletenessBlueprint()); + return std::make_unique<FlowCompletenessBlueprint>(); } bool @@ -318,6 +320,4 @@ FlowCompletenessBlueprint::createExecutor(const fef::IQueryEnvironment &env, ves return stash.create<FlowCompletenessExecutor>(env, _params); } -//----------------------------------------------------------------------------- - } diff --git a/searchlib/src/vespa/searchlib/features/foreachfeature.cpp b/searchlib/src/vespa/searchlib/features/foreachfeature.cpp index a67d8001a36..21167dd23d4 100644 --- a/searchlib/src/vespa/searchlib/features/foreachfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/foreachfeature.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> #include <boost/algorithm/string/replace.hpp> #include <vespa/log/log.h> @@ -120,9 +121,7 @@ ForeachBlueprint::ForeachBlueprint() : { } -ForeachBlueprint::~ForeachBlueprint() -{ -} +ForeachBlueprint::~ForeachBlueprint() = default; void ForeachBlueprint::visitDumpFeatures(const IIndexEnvironment &, @@ -171,13 +170,13 @@ ForeachBlueprint::setup(const IIndexEnvironment & env, Blueprint::UP ForeachBlueprint::createInstance() const { - return Blueprint::UP(new ForeachBlueprint()); + return std::make_unique<ForeachBlueprint>(); } FeatureExecutor & ForeachBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash &stash) const { - if (_executorCreator.get() != NULL) { + if (_executorCreator) { return _executorCreator->create(_num_inputs, stash); } return stash.create<SingleZeroValueExecutor>(); diff --git a/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp b/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp index 11ae8305e16..6e621f61034 100644 --- a/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp @@ -3,14 +3,14 @@ #include "freshnessfeature.h" #include "utils.h" #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <vespa/log/log.h> LOG_SETUP(".features.freshnessfeature"); using namespace search::fef; -namespace search { -namespace features { +namespace search::features { FreshnessExecutor::FreshnessExecutor(feature_t maxAge, feature_t scaleAge) : FeatureExecutor(), @@ -86,7 +86,7 @@ FreshnessBlueprint::setup(const IIndexEnvironment & env, Blueprint::UP FreshnessBlueprint::createInstance() const { - return Blueprint::UP(new FreshnessBlueprint()); + return std::make_unique<FreshnessBlueprint>(); } fef::ParameterDescriptions @@ -101,7 +101,4 @@ FreshnessBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash &s return stash.create<FreshnessExecutor>(_maxAge, _scaleAge); } - -} // namespace features -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp index fd1faeae5ea..e2e8a206099 100644 --- a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp @@ -5,12 +5,13 @@ #include "weighted_set_parser.h" #include "dotproductfeature.h" -#include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/attribute/imported_attribute_vector_read_guard.h> #include <vespa/searchlib/attribute/multinumericattribute.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/searchcommon/common/datatype.h> +#include <vespa/vespalib/util/stash.h> + #include <vespa/log/log.h> LOG_SETUP(".features.internalmaxreduceprodjoin"); diff --git a/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp b/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp index 45baf646656..89f7751a369 100644 --- a/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp @@ -3,11 +3,11 @@ #include "item_raw_score_feature.h" #include "valuefeature.h" #include "utils.h" +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { void ItemRawScoreExecutor::execute(uint32_t docId) @@ -89,6 +89,4 @@ ItemRawScoreBlueprint::resolve(const IQueryEnvironment &env, return handles; } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp b/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp index a5e3e2da5ba..dc689459ff3 100644 --- a/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp @@ -6,6 +6,8 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/locale/c.h> +#include <vespa/vespalib/util/stash.h> + namespace search::features { @@ -168,7 +170,7 @@ JaroWinklerDistanceBlueprint::setup(const search::fef::IIndexEnvironment &env, search::fef::Blueprint::UP JaroWinklerDistanceBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new JaroWinklerDistanceBlueprint()); + return std::make_unique<JaroWinklerDistanceBlueprint>(); } search::fef::FeatureExecutor & diff --git a/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp b/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp index fd453e17eb1..c061ace4854 100644 --- a/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp @@ -3,11 +3,11 @@ #include "matchcountfeature.h" #include "utils.h" #include "valuefeature.h" +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { MatchCountExecutor::MatchCountExecutor(uint32_t fieldId, const IQueryEnvironment &env) : FeatureExecutor(), @@ -43,7 +43,7 @@ MatchCountExecutor::handle_bind_match_data(const MatchData &md) MatchCountBlueprint::MatchCountBlueprint() : Blueprint("matchCount"), - _field(NULL) + _field(nullptr) { } @@ -63,7 +63,7 @@ MatchCountBlueprint::setup(const IIndexEnvironment &, const ParameterList & para Blueprint::UP MatchCountBlueprint::createInstance() const { - return Blueprint::UP(new MatchCountBlueprint()); + return std::make_unique<MatchCountBlueprint>(); } FeatureExecutor & @@ -75,5 +75,4 @@ MatchCountBlueprint::createExecutor(const IQueryEnvironment & queryEnv, vespalib return stash.create<MatchCountExecutor>(_field->id(), queryEnv); } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/matchesfeature.cpp b/searchlib/src/vespa/searchlib/features/matchesfeature.cpp index f4788ee74c8..a99c2330ee3 100644 --- a/searchlib/src/vespa/searchlib/features/matchesfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/matchesfeature.cpp @@ -4,11 +4,12 @@ #include "utils.h" #include "valuefeature.h" #include <vespa/searchlib/fef/fieldinfo.h> +#include <vespa/vespalib/util/stash.h> + using namespace search::fef; -namespace search { -namespace features { +namespace search::features { MatchesExecutor::MatchesExecutor(uint32_t fieldId, const search::fef::IQueryEnvironment &env, @@ -47,7 +48,7 @@ MatchesExecutor::handle_bind_match_data(const MatchData &md) MatchesBlueprint::MatchesBlueprint() : Blueprint("matches"), - _field(NULL), + _field(nullptr), _termIdx(std::numeric_limits<uint32_t>::max()) { } @@ -73,7 +74,7 @@ MatchesBlueprint::setup(const IIndexEnvironment &, Blueprint::UP MatchesBlueprint::createInstance() const { - return Blueprint::UP(new MatchesBlueprint()); + return std::make_unique<MatchesBlueprint>(); } FeatureExecutor & @@ -89,5 +90,4 @@ MatchesBlueprint::createExecutor(const IQueryEnvironment & queryEnv, vespalib::S } } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/matchfeature.cpp b/searchlib/src/vespa/searchlib/features/matchfeature.cpp index 7210b8b67e9..f6843df1a2f 100644 --- a/searchlib/src/vespa/searchlib/features/matchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/matchfeature.cpp @@ -2,17 +2,16 @@ #include "matchfeature.h" #include "utils.h" -#include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/properties.h> -#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> + using namespace search::fef; using CollectionType = FieldInfo::CollectionType; -namespace search { -namespace features { +namespace search::features { MatchExecutor::MatchExecutor(const MatchParams & params) : FeatureExecutor(), @@ -46,9 +45,7 @@ MatchBlueprint::MatchBlueprint() : { } -MatchBlueprint::~MatchBlueprint() -{ -} +MatchBlueprint::~MatchBlueprint() = default; void MatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env, @@ -61,7 +58,7 @@ MatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env, Blueprint::UP MatchBlueprint::createInstance() const { - return Blueprint::UP(new MatchBlueprint()); + return std::make_unique<MatchBlueprint>(); } bool @@ -101,6 +98,4 @@ MatchBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &st return stash.create<MatchExecutor>(_params); } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp b/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp index 7865e32849f..5ede14130ec 100644 --- a/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp @@ -2,11 +2,11 @@ #include "native_dot_product_feature.h" #include "utils.h" +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { NativeDotProductExecutor::NativeDotProductExecutor(const search::fef::IQueryEnvironment &env) : FeatureExecutor(), @@ -80,5 +80,4 @@ NativeDotProductBlueprint::createExecutor(const IQueryEnvironment &queryEnv, ves } } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp b/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp index 5dda159f629..865ea9fc3c4 100644 --- a/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp @@ -7,11 +7,11 @@ #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/itablemanager.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { feature_t NativeAttributeMatchExecutor::calculateScore(const CachedTermData &td, const TermFieldMatchData &tfmd) @@ -115,7 +115,7 @@ NativeAttributeMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env, Blueprint::UP NativeAttributeMatchBlueprint::createInstance() const { - return Blueprint::UP(new NativeAttributeMatchBlueprint()); + return std::make_unique<NativeAttributeMatchBlueprint>(); } fef::ParameterDescriptions @@ -160,6 +160,4 @@ NativeAttributeMatchBlueprint::createExecutor(const IQueryEnvironment &env, vesp return NativeAttributeMatchExecutor::createExecutor(env, _params, stash); } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp b/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp index 089a8102d6e..2b6841750ad 100644 --- a/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp @@ -7,11 +7,11 @@ #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/itablemanager.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { const uint32_t NativeFieldMatchParam::NOT_DEF_FIELD_LENGTH(std::numeric_limits<uint32_t>::max()); @@ -95,9 +95,7 @@ NativeFieldMatchBlueprint::NativeFieldMatchBlueprint() : { } -NativeFieldMatchBlueprint::~NativeFieldMatchBlueprint() -{ -} +NativeFieldMatchBlueprint::~NativeFieldMatchBlueprint() = default; void NativeFieldMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env, @@ -110,7 +108,7 @@ NativeFieldMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env, Blueprint::UP NativeFieldMatchBlueprint::createInstance() const { - return Blueprint::UP(new NativeFieldMatchBlueprint()); + return std::make_unique<NativeFieldMatchBlueprint>(); } bool @@ -181,5 +179,4 @@ NativeFieldMatchBlueprint::createExecutor(const IQueryEnvironment &env, vespalib } } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp b/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp index a31d9207e05..2809417e382 100644 --- a/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp @@ -7,12 +7,12 @@ #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/itablemanager.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <map> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { feature_t NativeProximityExecutor::calculateScoreForField(const FieldSetup & fs, uint32_t docId) @@ -136,9 +136,7 @@ NativeProximityBlueprint::NativeProximityBlueprint() : { } -NativeProximityBlueprint::~NativeProximityBlueprint() -{ -} +NativeProximityBlueprint::~NativeProximityBlueprint() = default; void NativeProximityBlueprint::visitDumpFeatures(const IIndexEnvironment & env, @@ -151,7 +149,7 @@ NativeProximityBlueprint::visitDumpFeatures(const IIndexEnvironment & env, Blueprint::UP NativeProximityBlueprint::createInstance() const { - return Blueprint::UP(new NativeProximityBlueprint()); + return std::make_unique<NativeProximityBlueprint>(); } bool @@ -168,12 +166,12 @@ NativeProximityBlueprint::setup(const IIndexEnvironment & env, NativeProximityParam & param = _params.vector[fieldId]; param.field = true; if ((param.proximityTable = - util::lookupTable(env, getBaseName(), "proximityTable", info->name(), _defaultProximityBoost)) == NULL) + util::lookupTable(env, getBaseName(), "proximityTable", info->name(), _defaultProximityBoost)) == nullptr) { return false; } if ((param.revProximityTable = - util::lookupTable(env, getBaseName(), "reverseProximityTable", info->name(), _defaultRevProximityBoost)) == NULL) + util::lookupTable(env, getBaseName(), "reverseProximityTable", info->name(), _defaultRevProximityBoost)) == nullptr) { return false; } @@ -190,7 +188,7 @@ NativeProximityBlueprint::setup(const IIndexEnvironment & env, if (NativeRankBlueprint::useTableNormalization(env)) { const Table * fp = param.proximityTable; const Table * rp = param.revProximityTable; - if (fp != NULL && rp != NULL) { + if (fp != nullptr && rp != nullptr) { double value = (fp->max() * param.proximityImportance) + (rp->max() * (1 - param.proximityImportance)); _params.setMaxTableSums(fieldId, value); @@ -217,6 +215,4 @@ NativeProximityBlueprint::createExecutor(const IQueryEnvironment &env, vespalib: } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp b/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp index b519c4f4b7f..a980c265484 100644 --- a/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp @@ -4,6 +4,7 @@ #include "valuefeature.h" #include "utils.h" #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <sstream> #include <vespa/log/log.h> @@ -30,8 +31,7 @@ buildFeatureName(const vespalib::string & baseName, const search::features::Fiel } -namespace search { -namespace features { +namespace search::features { FieldWrapper::FieldWrapper(const IIndexEnvironment & env, const ParameterList & fields, @@ -93,7 +93,7 @@ NativeRankBlueprint::visitDumpFeatures(const IIndexEnvironment & env, Blueprint::UP NativeRankBlueprint::createInstance() const { - return Blueprint::UP(new NativeRankBlueprint()); + return std::make_unique<NativeRankBlueprint>(); } bool @@ -168,6 +168,4 @@ NativeRankBlueprint::useTableNormalization(const search::fef::IIndexEnvironment return (!(norm.found() && (norm.get() == vespalib::string("false")))); } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/nowfeature.cpp b/searchlib/src/vespa/searchlib/features/nowfeature.cpp index 074acb2e890..d6059592cf3 100644 --- a/searchlib/src/vespa/searchlib/features/nowfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nowfeature.cpp @@ -3,6 +3,7 @@ #include "nowfeature.h" #include <vespa/searchlib/fef/queryproperties.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <chrono> namespace search::features { diff --git a/searchlib/src/vespa/searchlib/features/proximityfeature.cpp b/searchlib/src/vespa/searchlib/features/proximityfeature.cpp index f625e30f378..daeb4af6569 100644 --- a/searchlib/src/vespa/searchlib/features/proximityfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/proximityfeature.cpp @@ -2,13 +2,11 @@ #include "proximityfeature.h" #include "utils.h" -#include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/itermdata.h> -#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> -namespace search { -namespace features { +namespace search::features { ProximityConfig::ProximityConfig() : fieldId(search::fef::IllegalHandle), @@ -139,7 +137,7 @@ ProximityBlueprint::setup(const search::fef::IIndexEnvironment &env, search::fef::Blueprint::UP ProximityBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new ProximityBlueprint()); + return std::make_unique<ProximityBlueprint>(); } search::fef::FeatureExecutor & @@ -148,4 +146,4 @@ ProximityBlueprint::createExecutor(const search::fef::IQueryEnvironment &env, ve return stash.create<ProximityExecutor>(env, _config); } -}} +} diff --git a/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp b/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp index b4b6a1b0eb4..56e8810e14b 100644 --- a/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp @@ -3,15 +3,14 @@ #include "querycompletenessfeature.h" #include "utils.h" #include <vespa/searchlib/fef/featurenamebuilder.h> -#include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/itermdata.h> +#include <vespa/vespalib/util/stash.h> #include <limits> #include <vespa/log/log.h> LOG_SETUP(".features.querycompleteness"); -namespace search { -namespace features { +namespace search::features { QueryCompletenessConfig::QueryCompletenessConfig() : fieldId(search::fef::IllegalHandle), @@ -106,7 +105,7 @@ QueryCompletenessBlueprint::setup(const search::fef::IIndexEnvironment &env, search::fef::Blueprint::UP QueryCompletenessBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new QueryCompletenessBlueprint()); + return std::make_unique<QueryCompletenessBlueprint>(); } search::fef::FeatureExecutor & @@ -115,4 +114,4 @@ QueryCompletenessBlueprint::createExecutor(const search::fef::IQueryEnvironment return stash.create<QueryCompletenessExecutor>(env, _config); } -}} +} diff --git a/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp b/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp index dfc4af059a1..cbaf9e97cb6 100644 --- a/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp @@ -4,15 +4,14 @@ #include "valuefeature.h" #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/fieldinfo.h> -#include <vespa/searchlib/fef/fieldtype.h> #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/itermdata.h> -#include <vespa/searchlib/fef/handle.h> +#include <vespa/vespalib/util/stash.h> + using namespace search::fef; -namespace search { -namespace features { +namespace search::features { QueryTermCountBlueprint::QueryTermCountBlueprint() : Blueprint("queryTermCount") @@ -30,7 +29,7 @@ QueryTermCountBlueprint::visitDumpFeatures(const IIndexEnvironment & env, Blueprint::UP QueryTermCountBlueprint::createInstance() const { - return Blueprint::UP(new QueryTermCountBlueprint()); + return std::make_unique<QueryTermCountBlueprint>(); } bool @@ -49,6 +48,4 @@ QueryTermCountBlueprint::createExecutor(const IQueryEnvironment &env, vespalib:: return stash.create<ValueExecutor>(values); } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp index dd0c67df45c..8d4af8fd88d 100644 --- a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp @@ -3,6 +3,7 @@ #include "random_normal_feature.h" #include "utils.h" #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <chrono> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp index bde0cedaed0..f3d33c7dc29 100644 --- a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp @@ -3,6 +3,7 @@ #include "random_normal_stable_feature.h" #include "utils.h" #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <vespa/log/log.h> LOG_SETUP(".features.randomnormalstablefeature"); @@ -41,7 +42,7 @@ RandomNormalStableBlueprint::visitDumpFeatures(const search::fef::IIndexEnvironm search::fef::Blueprint::UP RandomNormalStableBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new RandomNormalStableBlueprint()); + return std::make_unique<RandomNormalStableBlueprint>(); } bool @@ -75,5 +76,4 @@ RandomNormalStableBlueprint::createExecutor(const search::fef::IQueryEnvironment return stash.create<RandomNormalStableExecutor>(seed, _mean, _stddev); } - } diff --git a/searchlib/src/vespa/searchlib/features/randomfeature.cpp b/searchlib/src/vespa/searchlib/features/randomfeature.cpp index 18b0cf616d4..95daebd0452 100644 --- a/searchlib/src/vespa/searchlib/features/randomfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/randomfeature.cpp @@ -3,7 +3,9 @@ #include "randomfeature.h" #include "utils.h" #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <chrono> + #include <vespa/log/log.h> LOG_SETUP(".features.randomfeature"); @@ -75,5 +77,4 @@ RandomBlueprint::createExecutor(const fef::IQueryEnvironment &env, vespalib::Sta return stash.create<RandomExecutor>(seed, matchSeed); } - } diff --git a/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp b/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp index 61355581214..ea696c75eff 100644 --- a/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp @@ -2,6 +2,7 @@ #include "raw_score_feature.h" #include "utils.h" +#include <vespa/vespalib/util/stash.h> using namespace search::fef; diff --git a/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp b/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp index c27936332d2..436e9bc0a0c 100644 --- a/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp @@ -2,14 +2,11 @@ #include "reverseproximityfeature.h" #include "utils.h" -#include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> -#include <vespa/searchlib/fef/fieldtype.h> #include <vespa/searchlib/fef/itermdata.h> -#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> -namespace search { -namespace features { +namespace search::features { ReverseProximityConfig::ReverseProximityConfig() : fieldId(search::fef::IllegalHandle), @@ -126,7 +123,7 @@ ReverseProximityBlueprint::setup(const search::fef::IIndexEnvironment &env, search::fef::Blueprint::UP ReverseProximityBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new ReverseProximityBlueprint()); + return std::make_unique<ReverseProximityBlueprint>(); } search::fef::FeatureExecutor & @@ -135,4 +132,4 @@ ReverseProximityBlueprint::createExecutor(const search::fef::IQueryEnvironment & return stash.create<ReverseProximityExecutor>(env, _config); } -}} +} diff --git a/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp b/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp index 6c52b6edb76..aaff46af93e 100644 --- a/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp @@ -2,11 +2,11 @@ #include "subqueries_feature.h" #include "utils.h" +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { SubqueriesExecutor::SubqueriesExecutor(const IQueryEnvironment &env, uint32_t fieldId) @@ -59,5 +59,4 @@ SubqueriesBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib: return stash.create<SubqueriesExecutor>(queryEnv, _field->id()); } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp b/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp index a4ca8524140..7fd487d8bdd 100644 --- a/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp @@ -2,10 +2,10 @@ #include "term_field_md_feature.h" #include "utils.h" -#include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/itablemanager.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <cassert> using namespace search::fef; @@ -83,7 +83,7 @@ TermFieldMdBlueprint::visitDumpFeatures(const IIndexEnvironment &, Blueprint::UP TermFieldMdBlueprint::createInstance() const { - return Blueprint::UP(new TermFieldMdBlueprint()); + return std::make_unique<TermFieldMdBlueprint>(); } bool diff --git a/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp b/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp index fb38a49d6eb..e9f48421fcf 100644 --- a/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp @@ -5,11 +5,11 @@ #include "utils.h" #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { TermDistanceExecutor::TermDistanceExecutor(const IQueryEnvironment & env, @@ -61,7 +61,7 @@ TermDistanceBlueprint::visitDumpFeatures(const IIndexEnvironment &, Blueprint::UP TermDistanceBlueprint::createInstance() const { - return Blueprint::UP(new TermDistanceBlueprint()); + return std::make_unique<TermDistanceBlueprint>(); } bool @@ -97,6 +97,4 @@ TermDistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::St } } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp b/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp index 5990d62cb25..433bf6134b8 100644 --- a/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp @@ -6,6 +6,8 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/locale/c.h> +#include <vespa/vespalib/util/stash.h> + #include <vespa/log/log.h> LOG_SETUP(".features.termeditdistance"); @@ -219,7 +221,7 @@ TermEditDistanceBlueprint::setup(const search::fef::IIndexEnvironment &env, search::fef::Blueprint::UP TermEditDistanceBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new TermEditDistanceBlueprint()); + return std::make_unique<TermEditDistanceBlueprint>(); } search::fef::FeatureExecutor & diff --git a/searchlib/src/vespa/searchlib/features/termfeature.cpp b/searchlib/src/vespa/searchlib/features/termfeature.cpp index d6df25cc2b9..90540726227 100644 --- a/searchlib/src/vespa/searchlib/features/termfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/termfeature.cpp @@ -4,15 +4,14 @@ #include "utils.h" #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> -#include <vespa/searchlib/fef/fieldtype.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/itermdata.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { TermExecutor::TermExecutor(const search::fef::IQueryEnvironment &env, uint32_t termId) : @@ -21,7 +20,7 @@ TermExecutor::TermExecutor(const search::fef::IQueryEnvironment &env, _connectedness(util::lookupConnectedness(env, termId)), _significance(0) { - if (_termData != NULL) { + if (_termData != nullptr) { feature_t fallback = util::getSignificance(*_termData); _significance = util::lookupSignificance(env, termId, fallback); } @@ -30,7 +29,7 @@ TermExecutor::TermExecutor(const search::fef::IQueryEnvironment &env, void TermExecutor::execute(uint32_t) { - if (_termData == NULL) { // this query term is not present in the query + if (_termData == nullptr) { // this query term is not present in the query outputs().set_number(0, 0.0f); // connectedness outputs().set_number(1, 0.0f); // significance (1 - frequency) outputs().set_number(2, 0.0f); // weight @@ -76,7 +75,7 @@ TermBlueprint::setup(const search::fef::IIndexEnvironment &, search::fef::Blueprint::UP TermBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new TermBlueprint()); + return std::make_unique<TermBlueprint>(); } search::fef::FeatureExecutor & @@ -85,4 +84,4 @@ TermBlueprint::createExecutor(const search::fef::IQueryEnvironment &env, vespali return stash.create<TermExecutor>(env, _termId); } -}} +} diff --git a/searchlib/src/vespa/searchlib/features/terminfofeature.cpp b/searchlib/src/vespa/searchlib/features/terminfofeature.cpp index 4f32cda8c86..ca7a5d8e248 100644 --- a/searchlib/src/vespa/searchlib/features/terminfofeature.cpp +++ b/searchlib/src/vespa/searchlib/features/terminfofeature.cpp @@ -3,30 +3,28 @@ #include "terminfofeature.h" #include "valuefeature.h" #include <vespa/searchlib/fef/properties.h> -#include <vespa/searchlib/fef/fieldinfo.h> -#include <vespa/searchlib/fef/fieldtype.h> #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/itermdata.h> -#include <vespa/searchlib/fef/handle.h> +#include <vespa/vespalib/util/stash.h> -namespace search { -namespace features { +using namespace search::fef; +namespace search::features { TermInfoBlueprint::TermInfoBlueprint() - : search::fef::Blueprint("termInfo"), + : Blueprint("termInfo"), _termIdx(0) { } void -TermInfoBlueprint::visitDumpFeatures(const search::fef::IIndexEnvironment &, - search::fef::IDumpFeatureVisitor &) const +TermInfoBlueprint::visitDumpFeatures(const IIndexEnvironment &, + IDumpFeatureVisitor &) const { } bool -TermInfoBlueprint::setup(const search::fef::IIndexEnvironment &, - const search::fef::ParameterList & params) +TermInfoBlueprint::setup(const IIndexEnvironment &, + const ParameterList & params) { _termIdx = params[0].asInteger(); describeOutput("queryidx", "The index of the first term with the given " @@ -34,8 +32,8 @@ TermInfoBlueprint::setup(const search::fef::IIndexEnvironment &, return true; } -search::fef::FeatureExecutor & -TermInfoBlueprint::createExecutor(const search::fef::IQueryEnvironment &queryEnv, vespalib::Stash &stash) const +FeatureExecutor & +TermInfoBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stash &stash) const { feature_t queryIdx = -1.0; if (queryEnv.getNumTerms() > _termIdx) { @@ -46,5 +44,4 @@ TermInfoBlueprint::createExecutor(const search::fef::IQueryEnvironment &queryEnv return stash.create<ValueExecutor>(values); } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp b/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp index 3cdb20a16e5..a0e0a8759a0 100644 --- a/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp @@ -3,6 +3,7 @@ #include "text_similarity_feature.h" #include <vespa/searchlib/fef/itermdata.h> #include <vespa/searchlib/fef/featurenamebuilder.h> +#include <vespa/vespalib/util/stash.h> namespace search::features { @@ -194,7 +195,7 @@ TextSimilarityBlueprint::visitDumpFeatures(const fef::IIndexEnvironment &env, fef::Blueprint::UP TextSimilarityBlueprint::createInstance() const { - return Blueprint::UP(new TextSimilarityBlueprint()); + return std::make_unique<TextSimilarityBlueprint>(); } bool @@ -218,6 +219,4 @@ TextSimilarityBlueprint::createExecutor(const fef::IQueryEnvironment &env, vespa return stash.create<TextSimilarityExecutor>(env, _field_id); } -//----------------------------------------------------------------------------- - } diff --git a/searchlib/src/vespa/searchlib/features/valuefeature.cpp b/searchlib/src/vespa/searchlib/features/valuefeature.cpp index 339cc5431f1..2b91cf1688b 100644 --- a/searchlib/src/vespa/searchlib/features/valuefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/valuefeature.cpp @@ -2,12 +2,13 @@ #include "valuefeature.h" #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/stash.h> -namespace search { -namespace features { +using namespace search::fef; +namespace search::features { ValueExecutor::ValueExecutor(const std::vector<feature_t> & values) : - search::fef::FeatureExecutor(), + FeatureExecutor(), _values(values) { } @@ -27,22 +28,20 @@ SingleZeroValueExecutor::execute(uint32_t) } ValueBlueprint::ValueBlueprint() : - search::fef::Blueprint("value"), + Blueprint("value"), _values() { } -ValueBlueprint::~ValueBlueprint() {} +ValueBlueprint::~ValueBlueprint() = default; void -ValueBlueprint::visitDumpFeatures(const search::fef::IIndexEnvironment &, - search::fef::IDumpFeatureVisitor &) const +ValueBlueprint::visitDumpFeatures(const IIndexEnvironment &, IDumpFeatureVisitor &) const { } bool -ValueBlueprint::setup(const search::fef::IIndexEnvironment &, - const search::fef::ParameterList & params) +ValueBlueprint::setup(const IIndexEnvironment &, const ParameterList & params) { for (uint32_t i = 0; i < params.size(); ++i) { _values.push_back(params[i].asDouble()); @@ -56,13 +55,12 @@ ValueBlueprint::setup(const search::fef::IIndexEnvironment &, return true; } -search::fef::FeatureExecutor & -ValueBlueprint::createExecutor(const search::fef::IQueryEnvironment &queryEnv, vespalib::Stash &stash) const +FeatureExecutor & +ValueBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stash &stash) const { (void) queryEnv; return stash.create<ValueExecutor>(_values); } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp index cd2cd949a91..82a69009e82 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp +++ b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp @@ -85,9 +85,21 @@ struct Compiler : public Blueprint::DependencyHandler { } compile_error = true; } + fixup_feature_map(); return FeatureRef(); } + void fixup_feature_map() { + auto itr = feature_map.begin(); + while (itr != feature_map.end()) { + if (itr->second.executor >= spec_list.size()) { + itr = feature_map.erase(itr); + } else { + ++itr; + } + } + } + FeatureRef verify_type(const FeatureNameParser &parser, FeatureRef ref, Accept accept_type) { const auto &spec = spec_list[ref.executor]; bool is_object = spec.output_types[ref.output]; diff --git a/searchlib/src/vespa/searchlib/fef/rank_program.cpp b/searchlib/src/vespa/searchlib/fef/rank_program.cpp index d39be693806..bd8abe41afb 100644 --- a/searchlib/src/vespa/searchlib/fef/rank_program.cpp +++ b/searchlib/src/vespa/searchlib/fef/rank_program.cpp @@ -6,6 +6,9 @@ #include <algorithm> #include <cassert> +#include <vespa/log/log.h> +LOG_SETUP(".fef.rankprogram"); + using vespalib::Stash; namespace search::fef { @@ -42,7 +45,7 @@ struct OverrideVisitor : public IPropertiesVisitor { auto pos = feature_map.find(key); if (pos != feature_map.end()) { - overrides.push_back(Override(pos->second, vespalib::locale::c::strtod(values.get().c_str(), nullptr))); + overrides.emplace_back(pos->second, vespalib::locale::c::strtod(values.get().c_str(), nullptr)); } } }; @@ -175,6 +178,7 @@ RankProgram::setup(const MatchData &md, auto override_end = overrides.end(); const auto &specs = _resolver->getExecutorSpecs(); + _executors.reserve(specs.size()); for (uint32_t i = 0; i < specs.size(); ++i) { vespalib::ArrayRef<NumberOrObject> outputs = _hot_stash.create_array<NumberOrObject>(specs[i].output_types.size()); StashSelector stash(_hot_stash, _cold_stash); @@ -216,6 +220,8 @@ RankProgram::setup(const MatchData &md, } } assert(_executors.size() == specs.size()); + LOG(debug, "Num executors = %ld, hot stash = %ld, cold stash = %ld, match data fields = %d", + _executors.size(), _hot_stash.count_used(), _cold_stash.count_used(), md.getNumTermFields()); } FeatureResolver diff --git a/searchlib/src/vespa/searchlib/fef/rank_program.h b/searchlib/src/vespa/searchlib/fef/rank_program.h index 0e5c390162a..aa6f77abce4 100644 --- a/searchlib/src/vespa/searchlib/fef/rank_program.h +++ b/searchlib/src/vespa/searchlib/fef/rank_program.h @@ -9,8 +9,8 @@ #include "feature_resolver.h" #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/array.h> +#include <vespa/vespalib/util/stash.h> #include <set> -#include <vector> namespace search::fef { diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp index ee88be8ad00..88f4a07d95d 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp @@ -12,7 +12,7 @@ class VisitorAdapter : public search::fef::IDumpFeatureVisitor { search::fef::BlueprintResolver &_resolver; public: - VisitorAdapter(search::fef::BlueprintResolver &resolver) + explicit VisitorAdapter(search::fef::BlueprintResolver &resolver) : _resolver(resolver) {} void visitDumpFeature(const vespalib::string &name) override { _resolver.addSeed(name); @@ -60,7 +60,8 @@ RankSetup::RankSetup(const BlueprintFactory &factory, const IIndexEnvironment &i _diversityCutoffFactor(10.0), _diversityCutoffStrategy("loose"), _softTimeoutEnabled(false), - _softTimeoutTailCost(0.1) + _softTimeoutTailCost(0.1), + _softTimeoutFactor(0.5) { } RankSetup::~RankSetup() = default; @@ -71,13 +72,13 @@ RankSetup::configure() setFirstPhaseRank(rank::FirstPhase::lookup(_indexEnv.getProperties())); setSecondPhaseRank(rank::SecondPhase::lookup(_indexEnv.getProperties())); std::vector<vespalib::string> summaryFeatures = summary::Feature::lookup(_indexEnv.getProperties()); - for (uint32_t i = 0; i < summaryFeatures.size(); ++i) { - addSummaryFeature(summaryFeatures[i]); + for (const auto & feature : summaryFeatures) { + addSummaryFeature(feature); } setIgnoreDefaultRankFeatures(dump::IgnoreDefaultFeatures::check(_indexEnv.getProperties())); std::vector<vespalib::string> dumpFeatures = dump::Feature::lookup(_indexEnv.getProperties()); - for (uint32_t i = 0; i < dumpFeatures.size(); ++i) { - addDumpFeature(dumpFeatures[i]); + for (const auto & feature : dumpFeatures) { + addDumpFeature(feature); } split_unpacking_iterators(matching::SplitUnpackingIterators::check(_indexEnv.getProperties())); delay_unpacking_iterators(matching::DelayUnpackingIterators::check(_indexEnv.getProperties())); @@ -159,15 +160,15 @@ RankSetup::compile() _compileError = true; } } - for (uint32_t i = 0; i < _summaryFeatures.size(); ++i) { - _summary_resolver->addSeed(_summaryFeatures[i]); + for (const auto & feature :_summaryFeatures) { + _summary_resolver->addSeed(feature); } if (!_ignoreDefaultRankFeatures) { VisitorAdapter adapter(*_dumpResolver); _factory.visitDumpFeatures(_indexEnv, adapter); } - for (uint32_t i = 0; i < _dumpFeatures.size(); ++i) { - _dumpResolver->addSeed(_dumpFeatures[i]); + for (const auto & feature : _dumpFeatures) { + _dumpResolver->addSeed(feature); } _indexEnv.hintFeatureMotivation(IIndexEnvironment::RANK); _compileError |= !_first_phase_resolver->compile(); diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.h b/searchlib/src/vespa/searchlib/fef/ranksetup.h index d543794b347..e1cd78d41a9 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.h +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.h @@ -397,10 +397,10 @@ public: // them to be ready to use. Also keep in mind that creating a rank // program is cheap while setting it up is more expensive. - RankProgram::UP create_first_phase_program() const { return RankProgram::UP(new RankProgram(_first_phase_resolver)); } - RankProgram::UP create_second_phase_program() const { return RankProgram::UP(new RankProgram(_second_phase_resolver)); } - RankProgram::UP create_summary_program() const { return RankProgram::UP(new RankProgram(_summary_resolver)); } - RankProgram::UP create_dump_program() const { return RankProgram::UP(new RankProgram(_dumpResolver)); } + RankProgram::UP create_first_phase_program() const { return std::make_unique<RankProgram>(_first_phase_resolver); } + RankProgram::UP create_second_phase_program() const { return std::make_unique<RankProgram>(_second_phase_resolver); } + RankProgram::UP create_summary_program() const { return std::make_unique<RankProgram>(_summary_resolver); } + RankProgram::UP create_dump_program() const { return std::make_unique<RankProgram>(_dumpResolver); } /** * Here you can do some preprocessing. State must be stored in the IObjectStore. diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp index 31e99ef9953..8be0961f999 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp @@ -2,11 +2,10 @@ #include "cfgvalue.h" #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stash.h> #include <sstream> -namespace search { -namespace fef { -namespace test { +namespace search::fef::test { CfgValueBlueprint::CfgValueBlueprint() : Blueprint("test_cfgvalue"), @@ -14,9 +13,7 @@ CfgValueBlueprint::CfgValueBlueprint() : { } -CfgValueBlueprint::~CfgValueBlueprint() -{ -} +CfgValueBlueprint::~CfgValueBlueprint() = default; void CfgValueBlueprint::visitDumpFeatures(const IIndexEnvironment &indexEnv, IDumpFeatureVisitor &visitor) const @@ -59,6 +56,4 @@ CfgValueBlueprint::createExecutor(const IQueryEnvironment & queryEnv, vespalib:: return stash.create<search::features::ValueExecutor>(_values); } -} // namespace test -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp index 86754c2c22d..017b916ad76 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp @@ -1,11 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "chain.h" +#include <vespa/vespalib/util/stash.h> #include <sstream> -namespace search { -namespace fef { -namespace test { +namespace search::fef::test { ChainExecutor::ChainExecutor() : FeatureExecutor() @@ -67,6 +66,4 @@ ChainBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stas return stash.create<ChainExecutor>(); } -} // namespace test -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp index d9ec8b13e57..6f8ebd57fb0 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp @@ -3,6 +3,7 @@ #include "double.h" #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/stash.h> #include <cassert> namespace search::fef::test { diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp index 4b4b10c4d25..1e57d6252ee 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp @@ -4,6 +4,7 @@ #include <vespa/searchlib/features/valuefeature.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/locale/c.h> +#include <vespa/vespalib/util/stash.h> #include <sstream> namespace search::fef::test { @@ -32,7 +33,7 @@ QueryBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stas { std::vector<feature_t> values; std::string val = queryEnv.getProperties().lookup(_key).get("0.0"); - values.push_back(vespalib::locale::c::strtod(val.data(), NULL)); + values.push_back(vespalib::locale::c::strtod(val.data(), nullptr)); return stash.create<search::features::ValueExecutor>(values); } diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp index c1b8b940245..c5871b23e77 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp @@ -2,10 +2,9 @@ #include "staticrank.h" #include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/vespalib/util/stash.h> -namespace search { -namespace fef { -namespace test { +namespace search::fef::test { StaticRankExecutor::StaticRankExecutor(const search::attribute::IAttributeVector * attribute) : FeatureExecutor(), @@ -17,7 +16,7 @@ void StaticRankExecutor::execute(uint32_t docId) { search::attribute::FloatContent staticRank; - if (_attribute != NULL) { + if (_attribute != nullptr) { staticRank.allocate(_attribute->getMaxValueCount()); staticRank.fill(*_attribute, docId); } @@ -31,9 +30,7 @@ StaticRankBlueprint::StaticRankBlueprint() : { } -StaticRankBlueprint::~StaticRankBlueprint() -{ -} +StaticRankBlueprint::~StaticRankBlueprint() = default; bool StaticRankBlueprint::setup(const IIndexEnvironment & indexEnv, const StringVector & params) @@ -54,6 +51,4 @@ StaticRankBlueprint::createExecutor(const IQueryEnvironment & queryEnv, vespalib return stash.create<StaticRankExecutor>(av); } -} // namespace test -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp index b5025d53cbd..4362a7f0860 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp @@ -2,10 +2,9 @@ #include "sum.h" #include <vespa/searchlib/fef/featurenamebuilder.h> +#include <vespa/vespalib/util/stash.h> -namespace search { -namespace fef { -namespace test { +namespace search::fef::test { void SumExecutor::execute(uint32_t) @@ -73,6 +72,4 @@ SumBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stash return stash.create<SumExecutor>(); } -} // namespace test -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp index 7b3876fada0..e30b4893e15 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "unbox.h" +#include <vespa/vespalib/util/stash.h> namespace search::fef::test { diff --git a/searchlib/src/vespa/searchlib/fef/test/test_features.cpp b/searchlib/src/vespa/searchlib/fef/test/test_features.cpp index f924acd65de..c32776dc88d 100644 --- a/searchlib/src/vespa/searchlib/fef/test/test_features.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/test_features.cpp @@ -3,6 +3,8 @@ #include <vespa/vespalib/testkit/test_kit.h> #include "test_features.h" #include <vespa/vespalib/locale/c.h> +#include <vespa/vespalib/util/stash.h> + using vespalib::eval::DoubleValue; using vespalib::eval::ValueType; diff --git a/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt b/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt index de2919443ff..0dcb0393473 100644 --- a/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt @@ -32,6 +32,7 @@ vespa_add_library(searchlib_queryeval OBJECT nearest_neighbor_blueprint.cpp nearest_neighbor_iterator.cpp nearsearch.cpp + nns_index_iterator.cpp orsearch.cpp predicate_blueprint.cpp predicate_search.cpp diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp index 8be6263221a..d4aa2aaa1d7 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp @@ -3,6 +3,7 @@ #include "emptysearch.h" #include "nearest_neighbor_blueprint.h" #include "nearest_neighbor_iterator.h" +#include "nns_index_iterator.h" #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/eval/tensor/dense/dense_tensor_view.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> @@ -17,20 +18,51 @@ NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& f _attr_tensor(attr_tensor), _query_tensor(std::move(query_tensor)), _target_num_hits(target_num_hits), - _distance_heap(target_num_hits) + _distance_heap(target_num_hits), + _found_hits() { - setEstimate(HitEstimate(_attr_tensor.getNumDocs(), false)); + uint32_t est_hits = _attr_tensor.getNumDocs(); + if (_attr_tensor.nearest_neighbor_index()) { + est_hits = std::min(target_num_hits, est_hits); + } + setEstimate(HitEstimate(est_hits, false)); } NearestNeighborBlueprint::~NearestNeighborBlueprint() = default; +void +NearestNeighborBlueprint::perform_top_k() +{ + auto nns_index = _attr_tensor.nearest_neighbor_index(); + if (nns_index) { + auto lhs_type = _query_tensor->fast_type(); + auto rhs_type = _attr_tensor.getTensorType(); + // XXX deal with different cell types later + if (lhs_type == rhs_type) { + auto lhs = _query_tensor->cellsRef(); + uint32_t k = _target_num_hits; + uint32_t explore_k = k + 100; // XXX hardcoded for now + _found_hits = nns_index->find_top_k(k, lhs, explore_k); + } + } +} + +void +NearestNeighborBlueprint::fetchPostings(const ExecuteInfo &execInfo) { + if (execInfo.isStrict()) { + perform_top_k(); + } +} + std::unique_ptr<SearchIterator> NearestNeighborBlueprint::createLeafSearch(const search::fef::TermFieldMatchDataArray& tfmda, bool strict) const { assert(tfmda.size() == 1); fef::TermFieldMatchData &tfmd = *tfmda[0]; // always search in only one field + if (strict && ! _found_hits.empty()) { + return NnsIndexIterator::create(tfmd, _found_hits); + } const vespalib::tensor::DenseTensorView &qT = *_query_tensor; - return NearestNeighborIterator::create(strict, tfmd, qT, _attr_tensor, _distance_heap); } diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h index 019f8e31842..ab4413c487a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h @@ -3,6 +3,7 @@ #include "blueprint.h" #include "nearest_neighbor_distance_heap.h" +#include <vespa/searchlib/tensor/nearest_neighbor_index.h> namespace vespalib::tensor { class DenseTensorView; } namespace search::tensor { class DenseTensorAttribute; } @@ -21,7 +22,9 @@ private: std::unique_ptr<vespalib::tensor::DenseTensorView> _query_tensor; uint32_t _target_num_hits; mutable NearestNeighborDistanceHeap _distance_heap; + std::vector<search::tensor::NearestNeighborIndex::Neighbor> _found_hits; + void perform_top_k(); public: NearestNeighborBlueprint(const queryeval::FieldSpec& field, const tensor::DenseTensorAttribute& attr_tensor, @@ -38,6 +41,7 @@ public: bool strict) const override; void visitMembers(vespalib::ObjectVisitor& visitor) const override; bool always_needs_unpack() const override; + void fetchPostings(const ExecuteInfo &execInfo) override; }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp new file mode 100644 index 00000000000..18e0213e092 --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp @@ -0,0 +1,68 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "nns_index_iterator.h" +#include <vespa/searchlib/tensor/nearest_neighbor_index.h> +#include <cmath> + +using Neighbor = search::tensor::NearestNeighborIndex::Neighbor; + +namespace search::queryeval { + +/** + * Search iterator for K nearest neighbor matching, + * where the actual search is done up front and this class + * just iterates over a vector held by the blueprint. + **/ +class NeighborVectorIterator : public NnsIndexIterator +{ +private: + fef::TermFieldMatchData &_tfmd; + const std::vector<Neighbor> &_hits; + uint32_t _idx; + double _last_sq_dist; +public: + NeighborVectorIterator(fef::TermFieldMatchData &tfmd, + const std::vector<Neighbor> &hits) + : _tfmd(tfmd), + _hits(hits), + _idx(0), + _last_sq_dist(0.0) + {} + + void initRange(uint32_t begin_id, uint32_t end_id) override { + SearchIterator::initRange(begin_id, end_id); + _idx = 0; + } + + void doSeek(uint32_t docId) override { + while (_idx < _hits.size()) { + uint32_t hit_id = _hits[_idx].docid; + if (hit_id < docId) { + ++_idx; + } else if (hit_id < getEndId()) { + setDocId(hit_id); + _last_sq_dist = _hits[_idx].distance; + return; + } else { + _idx = _hits.size(); + } + } + setAtEnd(); + } + + void doUnpack(uint32_t docId) override { + _tfmd.setRawScore(docId, sqrt(_last_sq_dist)); + } + + Trinary is_strict() const override { return Trinary::True; } +}; + +std::unique_ptr<NnsIndexIterator> +NnsIndexIterator::create( + fef::TermFieldMatchData &tfmd, + const std::vector<Neighbor> &hits) +{ + return std::make_unique<NeighborVectorIterator>(tfmd, hits); +} + +} // namespace diff --git a/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h new file mode 100644 index 00000000000..9ffd0df94eb --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h @@ -0,0 +1,20 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "searchiterator.h" +#include <vespa/searchlib/fef/termfieldmatchdata.h> +#include <vespa/searchlib/tensor/nearest_neighbor_index.h> + +namespace search::queryeval { + +class NnsIndexIterator : public SearchIterator +{ +public: + using Hit = search::tensor::NearestNeighborIndex::Neighbor; + static std::unique_ptr<NnsIndexIterator> create( + fef::TermFieldMatchData &tfmd, + const std::vector<Hit> &hits); +}; + +} // namespace diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt index 9175168248c..0bdcd53af77 100644 --- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt @@ -1,16 +1,18 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchlib_tensor OBJECT SOURCES + default_nearest_neighbor_index_factory.cpp dense_tensor_attribute.cpp dense_tensor_attribute_saver.cpp dense_tensor_store.cpp generic_tensor_attribute.cpp + generic_tensor_attribute_saver.cpp generic_tensor_store.cpp hnsw_index.cpp imported_tensor_attribute_vector.cpp imported_tensor_attribute_vector_read_guard.cpp + nearest_neighbor_index.cpp tensor_attribute.cpp - generic_tensor_attribute_saver.cpp tensor_store.cpp DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp new file mode 100644 index 00000000000..68efe6417c0 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp @@ -0,0 +1,51 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "default_nearest_neighbor_index_factory.h" +#include "distance_functions.h" +#include "hnsw_index.h" +#include "random_level_generator.h" +#include <vespa/searchcommon/attribute/config.h> + +namespace search::tensor { + +using vespalib::eval::ValueType; + +namespace { + +class LevelZeroGenerator : public RandomLevelGenerator { + uint32_t max_level() override { return 0; } +}; + +DistanceFunction::UP +make_distance_function(ValueType::CellType cell_type) +{ + if (cell_type == ValueType::CellType::FLOAT) { + return std::make_unique<SquaredEuclideanDistance<float>>(); + } else { + return std::make_unique<SquaredEuclideanDistance<double>>(); + } +} + +RandomLevelGenerator::UP +make_random_level_generator() +{ + // TODO: Make generator that results in hierarchical graph. + return std::make_unique<LevelZeroGenerator>(); +} + +} + +std::unique_ptr<NearestNeighborIndex> +DefaultNearestNeighborIndexFactory::make(const DocVectorAccess& vectors, + vespalib::eval::ValueType::CellType cell_type, + const search::attribute::HnswIndexParams& params) const +{ + HnswIndex::Config cfg(params.max_links_per_node() * 2, + params.max_links_per_node(), + params.neighbors_to_explore_at_insert(), + true); + return std::make_unique<HnswIndex>(vectors, make_distance_function(cell_type), make_random_level_generator(), cfg); +} + +} + diff --git a/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h new file mode 100644 index 00000000000..ea784efdb51 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h @@ -0,0 +1,19 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "nearest_neighbor_index_factory.h" + +namespace search::tensor { + +/** + * Factory that instantiates the production hnsw index. + */ +class DefaultNearestNeighborIndexFactory : public NearestNeighborIndexFactory { +public: + std::unique_ptr<NearestNeighborIndex> make(const DocVectorAccess& vectors, + vespalib::eval::ValueType::CellType cell_type, + const search::attribute::HnswIndexParams& params) const override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index a2b9f136ed9..171340e07f1 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -2,6 +2,7 @@ #include "dense_tensor_attribute.h" #include "dense_tensor_attribute_saver.h" +#include "nearest_neighbor_index.h" #include "tensor_attribute.hpp" #include <vespa/eval/tensor/tensor.h> #include <vespa/eval/tensor/dense/mutable_dense_tensor_view.h> @@ -55,11 +56,23 @@ TensorReader::is_present() { } -DenseTensorAttribute::DenseTensorAttribute(vespalib::stringref baseFileName, - const Config &cfg) +void +DenseTensorAttribute::consider_remove_from_index(DocId docid) +{ + if (_index && _refVector[docid].valid()) { + _index->remove_document(docid); + } +} + +DenseTensorAttribute::DenseTensorAttribute(vespalib::stringref baseFileName, const Config& cfg, + const NearestNeighborIndexFactory& index_factory) : TensorAttribute(baseFileName, cfg, _denseTensorStore), - _denseTensorStore(cfg.tensorType()) + _denseTensorStore(cfg.tensorType()), + _index() { + if (cfg.hnsw_index_params().has_value()) { + _index = index_factory.make(*this, cfg.tensorType().cell_type(), cfg.hnsw_index_params().value()); + } } @@ -69,12 +82,23 @@ DenseTensorAttribute::~DenseTensorAttribute() _tensorStore.clearHoldLists(); } +uint32_t +DenseTensorAttribute::clearDoc(DocId docId) +{ + consider_remove_from_index(docId); + return TensorAttribute::clearDoc(docId); +} + void DenseTensorAttribute::setTensor(DocId docId, const Tensor &tensor) { checkTensorType(tensor); + consider_remove_from_index(docId); EntryRef ref = _denseTensorStore.setTensor(tensor); setTensorRef(docId, ref); + if (_index) { + _index->add_document(docId); + } } @@ -120,6 +144,11 @@ DenseTensorAttribute::onLoad() auto raw = _denseTensorStore.allocRawBuffer(); tensorReader.readTensor(raw.data, _denseTensorStore.getBufSize()); _refVector.push_back(raw.ref); + if (_index) { + // This ensures that get_vector() (via getTensor()) is able to find the newly added tensor. + setCommittedDocIdLimit(lid + 1); + _index->add_document(lid); + } } else { _refVector.push_back(EntryRef()); } @@ -154,4 +183,12 @@ DenseTensorAttribute::getVersion() const return DENSE_TENSOR_ATTRIBUTE_VERSION; } +vespalib::tensor::TypedCells +DenseTensorAttribute::get_vector(uint32_t docid) const +{ + MutableDenseTensorView tensor_view(_denseTensorStore.type()); + getTensor(docid, tensor_view); + return tensor_view.cellsRef(); +} + } diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h index 593741cef39..f9a8a81b56b 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h @@ -2,35 +2,47 @@ #pragma once -#include "tensor_attribute.h" +#include "default_nearest_neighbor_index_factory.h" #include "dense_tensor_store.h" +#include "doc_vector_access.h" +#include "tensor_attribute.h" +#include <memory> -namespace vespalib { namespace tensor { class MutableDenseTensorView; }} +namespace vespalib::tensor { class MutableDenseTensorView; } -namespace search { +namespace search::tensor { -namespace tensor { +class NearestNeighborIndex; /** * Attribute vector class used to store dense tensors for all * documents in memory. */ -class DenseTensorAttribute : public TensorAttribute -{ +class DenseTensorAttribute : public TensorAttribute, public DocVectorAccess { +private: DenseTensorStore _denseTensorStore; + std::unique_ptr<NearestNeighborIndex> _index; + + void consider_remove_from_index(DocId docid); + public: - DenseTensorAttribute(vespalib::stringref baseFileName, const Config &cfg); + DenseTensorAttribute(vespalib::stringref baseFileName, const Config& cfg, + const NearestNeighborIndexFactory& index_factory = DefaultNearestNeighborIndexFactory()); virtual ~DenseTensorAttribute(); - virtual void setTensor(DocId docId, const Tensor &tensor) override; - virtual std::unique_ptr<Tensor> getTensor(DocId docId) const override; - virtual void getTensor(DocId docId, vespalib::tensor::MutableDenseTensorView &tensor) const override; - virtual bool onLoad() override; - virtual std::unique_ptr<AttributeSaver> onInitSave(vespalib::stringref fileName) override; - virtual void compactWorst() override; - virtual uint32_t getVersion() const override; + // Implements TensorAttribute + uint32_t clearDoc(DocId docId) override; + void setTensor(DocId docId, const Tensor &tensor) override; + std::unique_ptr<Tensor> getTensor(DocId docId) const override; + void getTensor(DocId docId, vespalib::tensor::MutableDenseTensorView &tensor) const override; + bool onLoad() override; + std::unique_ptr<AttributeSaver> onInitSave(vespalib::stringref fileName) override; + void compactWorst() override; + uint32_t getVersion() const override; + + // Implements DocVectorAccess + vespalib::tensor::TypedCells get_vector(uint32_t docid) const override; + + const NearestNeighborIndex* nearest_neighbor_index() const { return _index.get(); } }; - -} // namespace search::tensor - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function.h b/searchlib/src/vespa/searchlib/tensor/distance_function.h index 8dfb77ddccb..b682824c805 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_function.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_function.h @@ -2,6 +2,8 @@ #pragma once +#include <memory> + namespace vespalib::tensor { struct TypedCells; } namespace search::tensor { @@ -14,6 +16,7 @@ namespace search::tensor { */ class DistanceFunction { public: + using UP = std::unique_ptr<DistanceFunction>; virtual ~DistanceFunction() {} virtual double calc(const vespalib::tensor::TypedCells& lhs, const vespalib::tensor::TypedCells& rhs) const = 0; }; diff --git a/searchlib/src/vespa/searchlib/tensor/distance_functions.h b/searchlib/src/vespa/searchlib/tensor/distance_functions.h index 1e8727e92aa..494d1a859b6 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_functions.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_functions.h @@ -3,6 +3,7 @@ #pragma once #include "distance_function.h" +#include <vespa/eval/tensor/dense/typed_cells.h> namespace search::tensor { diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index be53b758841..54779408b37 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -44,7 +44,7 @@ HnswIndex::max_links_for_level(uint32_t level) const uint32_t HnswIndex::make_node_for_document(uint32_t docid) { - uint32_t max_level = _level_generator.max_level(); + uint32_t max_level = _level_generator->max_level(); // TODO: Add capping on num_levels uint32_t num_levels = max_level + 1; // Note: The level array instance lives as long as the document is present in the index. @@ -136,7 +136,7 @@ HnswIndex::select_neighbors(const HnswCandidateVector& neighbors, uint32_t max_l } void -HnswIndex::connect_new_node(uint32_t docid, const LinkArray& neighbors, uint32_t level) +HnswIndex::connect_new_node(uint32_t docid, const LinkArrayRef &neighbors, uint32_t level) { set_link_array(docid, level, neighbors); for (uint32_t neighbor_docid : neighbors) { @@ -170,11 +170,11 @@ double HnswIndex::calc_distance(const TypedCells& lhs, uint32_t rhs_docid) const { auto rhs = get_vector(rhs_docid); - return _distance_func.calc(lhs, rhs); + return _distance_func->calc(lhs, rhs); } HnswCandidate -HnswIndex::find_nearest_in_layer(const TypedCells& input, const HnswCandidate& entry_point, uint32_t level) +HnswIndex::find_nearest_in_layer(const TypedCells& input, const HnswCandidate& entry_point, uint32_t level) const { HnswCandidate nearest = entry_point; bool keep_searching = true; @@ -192,7 +192,7 @@ HnswIndex::find_nearest_in_layer(const TypedCells& input, const HnswCandidate& e } void -HnswIndex::search_layer(const TypedCells& input, uint32_t neighbors_to_find, FurthestPriQ& best_neighbors, uint32_t level) +HnswIndex::search_layer(const TypedCells& input, uint32_t neighbors_to_find, FurthestPriQ& best_neighbors, uint32_t level) const { NearestPriQ candidates; // TODO: Add proper handling of visited set. @@ -227,11 +227,11 @@ HnswIndex::search_layer(const TypedCells& input, uint32_t neighbors_to_find, Fur } } -HnswIndex::HnswIndex(const DocVectorAccess& vectors, const DistanceFunction& distance_func, - RandomLevelGenerator& level_generator, const Config& cfg) +HnswIndex::HnswIndex(const DocVectorAccess& vectors, DistanceFunction::UP distance_func, + RandomLevelGenerator::UP level_generator, const Config& cfg) : _vectors(vectors), - _distance_func(distance_func), - _level_generator(level_generator), + _distance_func(std::move(distance_func)), + _level_generator(std::move(level_generator)), _cfg(cfg), _node_refs(), _nodes(make_default_node_store_config()), @@ -310,24 +310,32 @@ HnswIndex::remove_document(uint32_t docid) _node_refs[docid].store_release(invalid); } -std::vector<uint32_t> -HnswIndex::find_top_k(uint32_t k, TypedCells vector, uint32_t explore_k) +struct NeighborsByDocId { + bool operator() (const NearestNeighborIndex::Neighbor &lhs, + const NearestNeighborIndex::Neighbor &rhs) + { + return (lhs.docid < rhs.docid); + } +}; + +std::vector<NearestNeighborIndex::Neighbor> +HnswIndex::find_top_k(uint32_t k, TypedCells vector, uint32_t explore_k) const { - std::vector<uint32_t> result; + std::vector<Neighbor> result; FurthestPriQ candidates = top_k_candidates(vector, std::max(k, explore_k)); while (candidates.size() > k) { candidates.pop(); } result.reserve(candidates.size()); for (const HnswCandidate & hit : candidates.peek()) { - result.emplace_back(hit.docid); + result.emplace_back(hit.docid, hit.distance); } - std::sort(result.begin(), result.end()); + std::sort(result.begin(), result.end(), NeighborsByDocId()); return result; } FurthestPriQ -HnswIndex::top_k_candidates(const TypedCells &vector, uint32_t k) +HnswIndex::top_k_candidates(const TypedCells &vector, uint32_t k) const { FurthestPriQ best_neighbors; if (_entry_level < 0) { @@ -363,5 +371,28 @@ HnswIndex::get_node(uint32_t docid) const return HnswNode(result); } +void +HnswIndex::set_node(uint32_t docid, const HnswNode &node) +{ + _node_refs.ensure_size(docid + 1, AtomicEntryRef()); + // A document cannot be added twice. + assert(!_node_refs[docid].load_acquire().valid()); + + // make new node + size_t num_levels = node.size(); + assert(num_levels > 0); + LevelArray levels(num_levels, AtomicEntryRef()); + auto node_ref = _nodes.add(levels); + _node_refs[docid].store_release(node_ref); + + for (size_t level = 0; level < num_levels; ++level) { + connect_new_node(docid, node.level(level), level); + } + int max_level = num_levels - 1; + if (_entry_level < max_level) { + _entry_docid = docid; + _entry_level = max_level; + } } +} diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h index 814148072ca..a8129032c11 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -2,10 +2,12 @@ #pragma once +#include "distance_function.h" #include "doc_vector_access.h" #include "hnsw_index_utils.h" #include "hnsw_node.h" #include "nearest_neighbor_index.h" +#include "random_level_generator.h" #include <vespa/eval/tensor/dense/typed_cells.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/vespalib/datastore/array_store.h> @@ -15,9 +17,6 @@ namespace search::tensor { -class DistanceFunction; -class RandomLevelGenerator; - /** * Implementation of a hierarchical navigable small world graph (HNSW) * that is used for approximate K-nearest neighbor search. @@ -82,8 +81,8 @@ protected: using TypedCells = vespalib::tensor::TypedCells; const DocVectorAccess& _vectors; - const DistanceFunction& _distance_func; - RandomLevelGenerator& _level_generator; + DistanceFunction::UP _distance_func; + RandomLevelGenerator::UP _level_generator; Config _cfg; NodeRefVector _node_refs; NodeStore _nodes; @@ -111,7 +110,7 @@ protected: LinkArray select_neighbors_heuristic(const HnswCandidateVector& neighbors, uint32_t max_links) const; LinkArray select_neighbors_simple(const HnswCandidateVector& neighbors, uint32_t max_links) const; LinkArray select_neighbors(const HnswCandidateVector& neighbors, uint32_t max_links) const; - void connect_new_node(uint32_t docid, const LinkArray& neighbors, uint32_t level); + void connect_new_node(uint32_t docid, const LinkArrayRef &neighbors, uint32_t level); void remove_link_to(uint32_t remove_from, uint32_t remove_id, uint32_t level); inline TypedCells get_vector(uint32_t docid) const { @@ -124,18 +123,20 @@ protected: /** * Performs a greedy search in the given layer to find the candidate that is nearest the input vector. */ - HnswCandidate find_nearest_in_layer(const TypedCells& input, const HnswCandidate& entry_point, uint32_t level); - void search_layer(const TypedCells& input, uint32_t neighbors_to_find, FurthestPriQ& found_neighbors, uint32_t level); + HnswCandidate find_nearest_in_layer(const TypedCells& input, const HnswCandidate& entry_point, uint32_t level) const; + void search_layer(const TypedCells& input, uint32_t neighbors_to_find, FurthestPriQ& found_neighbors, uint32_t level) const; public: - HnswIndex(const DocVectorAccess& vectors, const DistanceFunction& distance_func, - RandomLevelGenerator& level_generator, const Config& cfg); + HnswIndex(const DocVectorAccess& vectors, DistanceFunction::UP distance_func, + RandomLevelGenerator::UP level_generator, const Config& cfg); ~HnswIndex() override; + const Config& config() const { return _cfg; } + void add_document(uint32_t docid) override; void remove_document(uint32_t docid) override; - std::vector<uint32_t> find_top_k(uint32_t k, TypedCells vector, uint32_t explore_k) override; - FurthestPriQ top_k_candidates(const TypedCells &vector, uint32_t k); + std::vector<Neighbor> find_top_k(uint32_t k, TypedCells vector, uint32_t explore_k) const override; + FurthestPriQ top_k_candidates(const TypedCells &vector, uint32_t k) const; // TODO: Add support for generation handling and cleanup (transfer_hold_lists, trim_hold_lists) @@ -144,8 +145,7 @@ public: // Should only be used by unit tests. HnswNode get_node(uint32_t docid) const; - - // TODO: Implement set_node() as well for use in unit tests. + void set_node(uint32_t docid, const HnswNode &node); }; } diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.cpp b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.cpp new file mode 100644 index 00000000000..f31230af381 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.cpp @@ -0,0 +1,3 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "nearest_neighbor_index.h" diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h index 2ae322fe76e..f933af0147e 100644 --- a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h +++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h @@ -13,10 +13,20 @@ namespace search::tensor { */ class NearestNeighborIndex { public: + struct Neighbor { + uint32_t docid; + double distance; + Neighbor(uint32_t id, double dist) + : docid(id), distance(dist) + {} + Neighbor() : docid(0), distance(0.0) {} + }; virtual ~NearestNeighborIndex() {} virtual void add_document(uint32_t docid) = 0; virtual void remove_document(uint32_t docid) = 0; - virtual std::vector<uint32_t> find_top_k(uint32_t k, vespalib::tensor::TypedCells vector, uint32_t explore_k) = 0; + virtual std::vector<Neighbor> find_top_k(uint32_t k, + vespalib::tensor::TypedCells vector, + uint32_t explore_k) const = 0; }; } diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_factory.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_factory.h new file mode 100644 index 00000000000..c09403df5e0 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_factory.h @@ -0,0 +1,26 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/value_type.h> +#include <memory> + +namespace search::attribute { class HnswIndexParams; } + +namespace search::tensor { + +class DocVectorAccess; +class NearestNeighborIndex; + +/** + * Factory interface used to instantiate an index used for (approximate) nearest neighbor search. + */ +class NearestNeighborIndexFactory { +public: + virtual ~NearestNeighborIndexFactory() {} + virtual std::unique_ptr<NearestNeighborIndex> make(const DocVectorAccess& vectors, + vespalib::eval::ValueType::CellType cell_type, + const search::attribute::HnswIndexParams& params) const = 0; +}; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/random_level_generator.h b/searchlib/src/vespa/searchlib/tensor/random_level_generator.h index 0fcac977d9d..0f4c7c34445 100644 --- a/searchlib/src/vespa/searchlib/tensor/random_level_generator.h +++ b/searchlib/src/vespa/searchlib/tensor/random_level_generator.h @@ -2,6 +2,8 @@ #pragma once +#include <memory> + namespace search::tensor { /** @@ -9,6 +11,7 @@ namespace search::tensor { */ class RandomLevelGenerator { public: + using UP = std::unique_ptr<RandomLevelGenerator>; virtual ~RandomLevelGenerator() {} virtual uint32_t max_level() = 0; }; diff --git a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp index 9019a212f3f..dff3acc5b89 100644 --- a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp +++ b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp @@ -246,6 +246,7 @@ TEST_F(MatchedElementsFilterTest, filters_elements_in_array_field_value_when_inp expect_filtered("array_in_doc", {0, 1, 2}, "[{'name':'a','weight':3}," "{'name':'b','weight':5}," "{'name':'c','weight':7}]"); + expect_filtered("array_in_doc", {0, 1, 100}, "[]"); } TEST_F(MatchedElementsFilterTest, struct_field_mapper_is_setup_for_array_field_value) @@ -276,6 +277,7 @@ TEST_F(MatchedElementsFilterTest, filters_elements_in_map_field_value_when_input expect_filtered("map_in_doc", {0, 1, 2}, "[{'key':'a','value':{'name':'a','weight':3}}," "{'key':'b','value':{'name':'b','weight':5}}," "{'key':'c','value':{'name':'c','weight':7}}]"); + expect_filtered("map_in_doc", {0, 1, 100}, "[]"); } TEST_F(MatchedElementsFilterTest, struct_field_mapper_is_setup_for_map_field_value) diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp index 7368a199569..ada14bf17f5 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp @@ -389,9 +389,11 @@ private: MapFieldValueInserter map_inserter(_inserter, _tokenize); if (filter_matching_elements()) { assert(v.has_no_erased_keys()); - for (uint32_t id_to_keep : (*_matching_elems)) { - auto entry = v[id_to_keep]; - map_inserter.insert_entry(*entry.first, *entry.second); + if (!_matching_elems->empty() && _matching_elems->back() < v.size()) { + for (uint32_t id_to_keep : (*_matching_elems)) { + auto entry = v[id_to_keep]; + map_inserter.insert_entry(*entry.first, *entry.second); + } } } else { for (const auto &entry : v) { @@ -406,8 +408,10 @@ private: ArrayInserter ai(a); SlimeFiller conv(ai, _tokenize); if (filter_matching_elements()) { - for (uint32_t id_to_keep : (*_matching_elems)) { - value[id_to_keep].accept(conv); + if (!_matching_elems->empty() && _matching_elems->back() < value.size()) { + for (uint32_t id_to_keep : (*_matching_elems)) { + value[id_to_keep].accept(conv); + } } } else { for (const FieldValue &fv : value) { diff --git a/security-tools/pom.xml b/security-tools/pom.xml index 38b14ce957f..195e2d06311 100644 --- a/security-tools/pom.xml +++ b/security-tools/pom.xml @@ -57,6 +57,7 @@ <exclude>META-INF/*.SF</exclude> <exclude>META-INF/*.DSA</exclude> <exclude>META-INF/*.RSA</exclude> + <exclude>META-INF/versions/*/module-info.class</exclude> </excludes> </filter> </filters> diff --git a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java index 367d7b9dd83..c314d17e018 100644 --- a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java +++ b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java @@ -54,6 +54,9 @@ public class Main { MixedMode mixedMode = TransportSecurityUtils.getInsecureMixedMode(envVars); if (options.isPresent() && mixedMode != MixedMode.PLAINTEXT_CLIENT_MIXED_SERVER) { outputVariables.put(OutputVariable.TLS_ENABLED, "1"); + if (options.get().isHostnameValidationDisabled()) { + outputVariables.put(OutputVariable.DISABLE_HOSTNAME_VALIDATION, "1"); + } options.get().getCaCertificatesFile() .ifPresent(caCertFile -> outputVariables.put(OutputVariable.CA_CERTIFICATE, caCertFile.toString())); options.get().getCertificatesFile() diff --git a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/OutputVariable.java b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/OutputVariable.java index dd248d05aac..9a90a145f30 100644 --- a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/OutputVariable.java +++ b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/OutputVariable.java @@ -10,7 +10,8 @@ enum OutputVariable { TLS_ENABLED("VESPA_TLS_ENABLED", "Set to '1' if TLS is enabled in Vespa"), CA_CERTIFICATE("VESPA_TLS_CA_CERT", "Path to CA certificates file"), CERTIFICATE("VESPA_TLS_CERT", "Path to certificate file"), - PRIVATE_KEY("VESPA_TLS_PRIVATE_KEY", "Path to private key file"); + PRIVATE_KEY("VESPA_TLS_PRIVATE_KEY", "Path to private key file"), + DISABLE_HOSTNAME_VALIDATION("VESPA_TLS_HOSTNAME_VALIDATION_DISABLED", "Set to '1' if TLS hostname validation is disabled"); private final String variableName; private final String description; diff --git a/security-tools/src/main/sh/vespa-curl-wrapper b/security-tools/src/main/sh/vespa-curl-wrapper index e286e121f64..b4fd9224a8a 100755 --- a/security-tools/src/main/sh/vespa-curl-wrapper +++ b/security-tools/src/main/sh/vespa-curl-wrapper @@ -88,6 +88,11 @@ then CURL_PARAMETERS=("${CURL_PARAMETERS[@]/http:/https:}") fi +if [ -n "${VESPA_TLS_HOSTNAME_VALIDATION_DISABLED}" ] +then + CURL_PARAMETERS=("--insecure" "${CURL_PARAMETERS[@]}") +fi + if [ -n "${VESPA_TLS_CA_CERT}" ] then CURL_PARAMETERS=("--cacert" "${VESPA_TLS_CA_CERT}" "${CURL_PARAMETERS[@]}") diff --git a/security-tools/src/test/java/com/yahoo/vespa/security/tool/securityenv/MainTest.java b/security-tools/src/test/java/com/yahoo/vespa/security/tool/securityenv/MainTest.java index b563ebd14f4..45626820f4d 100644 --- a/security-tools/src/test/java/com/yahoo/vespa/security/tool/securityenv/MainTest.java +++ b/security-tools/src/test/java/com/yahoo/vespa/security/tool/securityenv/MainTest.java @@ -106,6 +106,7 @@ public class MainTest { TransportSecurityOptions options = new TransportSecurityOptions.Builder() .withCertificates(Paths.get("/path/to/certificate"), Paths.get("/path/to/key")) .withCaCertificates(Paths.get("/path/to/cacerts")) + .withHostnameValidationDisabled(true) .build(); Path configFile = tmpFolder.newFile().toPath(); options.toJsonFile(configFile); diff --git a/security-tools/src/test/resources/bash-output.txt b/security-tools/src/test/resources/bash-output.txt index c07c667af47..182dc177d42 100644 --- a/security-tools/src/test/resources/bash-output.txt +++ b/security-tools/src/test/resources/bash-output.txt @@ -2,3 +2,4 @@ VESPA_TLS_ENABLED="1"; export VESPA_TLS_ENABLED; VESPA_TLS_CA_CERT="/path/to/cacerts"; export VESPA_TLS_CA_CERT; VESPA_TLS_CERT="/path/to/certificate"; export VESPA_TLS_CERT; VESPA_TLS_PRIVATE_KEY="/path/to/key"; export VESPA_TLS_PRIVATE_KEY; +VESPA_TLS_HOSTNAME_VALIDATION_DISABLED="1"; export VESPA_TLS_HOSTNAME_VALIDATION_DISABLED; diff --git a/security-tools/src/test/resources/csh-output.txt b/security-tools/src/test/resources/csh-output.txt index 2b6716de92b..2e6cd886c26 100644 --- a/security-tools/src/test/resources/csh-output.txt +++ b/security-tools/src/test/resources/csh-output.txt @@ -2,3 +2,4 @@ setenv VESPA_TLS_ENABLED "1"; setenv VESPA_TLS_CA_CERT "/path/to/cacerts"; setenv VESPA_TLS_CERT "/path/to/certificate"; setenv VESPA_TLS_PRIVATE_KEY "/path/to/key"; +setenv VESPA_TLS_HOSTNAME_VALIDATION_DISABLED "1"; diff --git a/security-tools/src/test/resources/expected-help-output.txt b/security-tools/src/test/resources/expected-help-output.txt index 7d125fe15a2..33ad3b6d232 100644 --- a/security-tools/src/test/resources/expected-help-output.txt +++ b/security-tools/src/test/resources/expected-help-output.txt @@ -9,3 +9,5 @@ The output may include the following variables: - 'VESPA_TLS_CA_CERT': Path to CA certificates file - 'VESPA_TLS_CERT': Path to certificate file - 'VESPA_TLS_PRIVATE_KEY': Path to private key file + - 'VESPA_TLS_HOSTNAME_VALIDATION_DISABLED': Set to '1' if TLS hostname +validation is disabled diff --git a/security-tools/src/test/resources/no-security-output.txt b/security-tools/src/test/resources/no-security-output.txt index 3467f1316b5..257a2747ee2 100644 --- a/security-tools/src/test/resources/no-security-output.txt +++ b/security-tools/src/test/resources/no-security-output.txt @@ -2,3 +2,4 @@ unset VESPA_TLS_ENABLED; unset VESPA_TLS_CA_CERT; unset VESPA_TLS_CERT; unset VESPA_TLS_PRIVATE_KEY; +unset VESPA_TLS_HOSTNAME_VALIDATION_DISABLED; diff --git a/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java b/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java index d2b98fd20d9..f3932c84a17 100644 --- a/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java +++ b/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java @@ -35,6 +35,7 @@ public class SslContextBuilder { private TrustManagerFactory trustManagerFactory = TrustManagerUtils::createDefaultX509TrustManager; private KeyManagerFactory keyManagerFactory = KeyManagerUtils::createDefaultX509KeyManager; private X509ExtendedKeyManager keyManager; + private X509ExtendedTrustManager trustManager; public SslContextBuilder() {} @@ -121,15 +122,25 @@ public class SslContextBuilder { return this; } + /** + * Note: Callee is responsible for configuring the trust manager. + * Any truststore configured by {@link #withTrustStore(KeyStore)} or the other overloads will be ignored. + */ + public SslContextBuilder withTrustManager(X509ExtendedTrustManager trustManager) { + this.trustManager = trustManager; + return this; + } + public SSLContext build() { try { SSLContext sslContext = SSLContext.getInstance(TlsContext.SSL_CONTEXT_VERSION); - TrustManager[] trustManagers = new TrustManager[] { trustManagerFactory.createTrustManager(trustStoreSupplier.get()) }; + X509ExtendedTrustManager trustManager = this.trustManager != null + ? this.trustManager + : trustManagerFactory.createTrustManager(trustStoreSupplier.get()); X509ExtendedKeyManager keyManager = this.keyManager != null ? this.keyManager : keyManagerFactory.createKeyManager(keyStoreSupplier.get(), keyStorePassword); - KeyManager[] keyManagers = new KeyManager[] {keyManager}; - sslContext.init(keyManagers, trustManagers, null); + sslContext.init(new KeyManager[] {keyManager}, new TrustManager[] {trustManager}, null); return sslContext; } catch (GeneralSecurityException e) { throw new RuntimeException(e); diff --git a/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java index f746480b126..28854c59b2c 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java @@ -12,11 +12,9 @@ import com.yahoo.security.tls.policy.AuthorizedPeers; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLParameters; -import javax.net.ssl.X509ExtendedTrustManager; import java.io.IOException; import java.io.UncheckedIOException; import java.lang.ref.WeakReference; -import java.nio.file.Files; import java.nio.file.Path; import java.security.KeyStore; import java.time.Duration; @@ -110,12 +108,14 @@ public class ConfigFileBasedTlsContext implements TlsContext { MutableX509TrustManager mutableTrustManager, MutableX509KeyManager mutableKeyManager, PeerAuthentication peerAuthentication) { + + HostnameVerification hostnameVerification = options.isHostnameValidationDisabled() ? HostnameVerification.DISABLED : HostnameVerification.ENABLED; + PeerAuthorizerTrustManager authorizerTrustManager = options.getAuthorizedPeers() + .map(authorizedPeers -> new PeerAuthorizerTrustManager(authorizedPeers, mode, hostnameVerification, mutableTrustManager)) + .orElseGet(() -> new PeerAuthorizerTrustManager(new AuthorizedPeers(com.yahoo.vespa.jdk8compat.Set.of()), AuthorizationMode.DISABLE, hostnameVerification, mutableTrustManager)); SSLContext sslContext = new SslContextBuilder() .withKeyManager(mutableKeyManager) - .withTrustManagerFactory( - ignoredTruststore -> options.getAuthorizedPeers() - .map(authorizedPeers -> (X509ExtendedTrustManager) new PeerAuthorizerTrustManager(authorizedPeers, mode, mutableTrustManager)) - .orElseGet(() -> new PeerAuthorizerTrustManager(new AuthorizedPeers(com.yahoo.vespa.jdk8compat.Set.of()), AuthorizationMode.DISABLE, mutableTrustManager))) + .withTrustManager(authorizerTrustManager) .build(); List<String> acceptedCiphers = options.getAcceptedCiphers(); Set<String> ciphers = acceptedCiphers.isEmpty() ? TlsContext.ALLOWED_CIPHER_SUITES : new HashSet<>(acceptedCiphers); diff --git a/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java index c3f10a464a5..def3e49be4d 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java @@ -34,8 +34,9 @@ public class DefaultTlsContext implements TlsContext { List<X509Certificate> caCertificates, AuthorizedPeers authorizedPeers, AuthorizationMode mode, - PeerAuthentication peerAuthentication) { - this(createSslContext(certificates, privateKey, caCertificates, authorizedPeers, mode), peerAuthentication); + PeerAuthentication peerAuthentication, + HostnameVerification hostnameVerification) { + this(createSslContext(certificates, privateKey, caCertificates, authorizedPeers, mode, hostnameVerification), peerAuthentication); } public DefaultTlsContext(SSLContext sslContext, PeerAuthentication peerAuthentication) { @@ -120,7 +121,8 @@ public class DefaultTlsContext implements TlsContext { PrivateKey privateKey, List<X509Certificate> caCertificates, AuthorizedPeers authorizedPeers, - AuthorizationMode mode) { + AuthorizationMode mode, + HostnameVerification hostnameVerification) { SslContextBuilder builder = new SslContextBuilder(); if (!certificates.isEmpty()) { builder.withKeyStore(privateKey, certificates); @@ -129,12 +131,12 @@ public class DefaultTlsContext implements TlsContext { builder.withTrustStore(caCertificates); } if (authorizedPeers != null) { - builder.withTrustManagerFactory(truststore -> new PeerAuthorizerTrustManager(authorizedPeers, mode, truststore)); + builder.withTrustManagerFactory(truststore -> new PeerAuthorizerTrustManager(authorizedPeers, mode, hostnameVerification, truststore)); } else { - builder.withTrustManagerFactory(truststore -> new PeerAuthorizerTrustManager(new AuthorizedPeers(com.yahoo.vespa.jdk8compat.Set.of()), AuthorizationMode.DISABLE, truststore)); + builder.withTrustManagerFactory(truststore -> new PeerAuthorizerTrustManager( + new AuthorizedPeers(com.yahoo.vespa.jdk8compat.Set.of()), AuthorizationMode.DISABLE, hostnameVerification, truststore)); } return builder.build(); } - } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/HostnameVerification.java b/security-utils/src/main/java/com/yahoo/security/tls/HostnameVerification.java new file mode 100644 index 00000000000..a41edc6dc44 --- /dev/null +++ b/security-utils/src/main/java/com/yahoo/security/tls/HostnameVerification.java @@ -0,0 +1,7 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.security.tls; + +/** + * @author bjorncs + */ +public enum HostnameVerification { ENABLED, DISABLED } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManager.java b/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManager.java index 3ddd0861f39..03358190e8a 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManager.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManager.java @@ -3,6 +3,7 @@ package com.yahoo.security.tls.authz; import com.yahoo.security.X509CertificateUtils; import com.yahoo.security.tls.AuthorizationMode; +import com.yahoo.security.tls.HostnameVerification; import com.yahoo.security.tls.TrustManagerUtils; import com.yahoo.security.tls.policy.AuthorizedPeers; @@ -14,7 +15,6 @@ import java.net.Socket; import java.security.KeyStore; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; -import java.util.Objects; import java.util.Optional; import java.util.logging.Logger; @@ -33,15 +33,23 @@ public class PeerAuthorizerTrustManager extends X509ExtendedTrustManager { private final PeerAuthorizer authorizer; private final X509ExtendedTrustManager defaultTrustManager; private final AuthorizationMode mode; + private final HostnameVerification hostnameVerification; - public PeerAuthorizerTrustManager(AuthorizedPeers authorizedPeers, AuthorizationMode mode, X509ExtendedTrustManager defaultTrustManager) { + public PeerAuthorizerTrustManager(AuthorizedPeers authorizedPeers, + AuthorizationMode mode, + HostnameVerification hostnameVerification, + X509ExtendedTrustManager defaultTrustManager) { this.authorizer = new PeerAuthorizer(authorizedPeers); this.mode = mode; + this.hostnameVerification = hostnameVerification; this.defaultTrustManager = defaultTrustManager; } - public PeerAuthorizerTrustManager(AuthorizedPeers authorizedPeers, AuthorizationMode mode, KeyStore truststore) { - this(authorizedPeers, mode, TrustManagerUtils.createDefaultX509TrustManager(truststore)); + public PeerAuthorizerTrustManager(AuthorizedPeers authorizedPeers, + AuthorizationMode mode, + HostnameVerification hostnameVerification, + KeyStore truststore) { + this(authorizedPeers, mode, hostnameVerification, TrustManagerUtils.createDefaultX509TrustManager(truststore)); } @Override @@ -58,28 +66,26 @@ public class PeerAuthorizerTrustManager extends X509ExtendedTrustManager { @Override public void checkClientTrusted(X509Certificate[] chain, String authType, Socket socket) throws CertificateException { - overrideHostnameVerification(socket); defaultTrustManager.checkClientTrusted(chain, authType, socket); authorizePeer(chain[0], authType, true, null); } @Override public void checkServerTrusted(X509Certificate[] chain, String authType, Socket socket) throws CertificateException { - overrideHostnameVerification(socket); + overrideHostnameVerificationForClient(socket); defaultTrustManager.checkServerTrusted(chain, authType, socket); authorizePeer(chain[0], authType, false, null); } @Override public void checkClientTrusted(X509Certificate[] chain, String authType, SSLEngine sslEngine) throws CertificateException { - overrideHostnameVerification(sslEngine); defaultTrustManager.checkClientTrusted(chain, authType, sslEngine); authorizePeer(chain[0], authType, true, sslEngine); } @Override public void checkServerTrusted(X509Certificate[] chain, String authType, SSLEngine sslEngine) throws CertificateException { - overrideHostnameVerification(sslEngine); + overrideHostnameVerificationForClient(sslEngine); defaultTrustManager.checkServerTrusted(chain, authType, sslEngine); authorizePeer(chain[0], authType, false, sslEngine); } @@ -121,31 +127,44 @@ public class PeerAuthorizerTrustManager extends X509ExtendedTrustManager { certificate.getSubjectX500Principal(), X509CertificateUtils.getSubjectAlternativeNames(certificate), authType, isVerifyingClient); } - private static void overrideHostnameVerification(SSLEngine engine) { + private void overrideHostnameVerificationForClient(SSLEngine engine) { SSLParameters params = engine.getSSLParameters(); - if (overrideHostnameVerification(params)) { + if (overrideHostnameVerificationForClient(params)) { engine.setSSLParameters(params); } } - private static void overrideHostnameVerification(Socket socket) { + private void overrideHostnameVerificationForClient(Socket socket) { if (socket instanceof SSLSocket) { SSLSocket sslSocket = (SSLSocket) socket; SSLParameters params = sslSocket.getSSLParameters(); - if (overrideHostnameVerification(params)) { + if (overrideHostnameVerificationForClient(params)) { sslSocket.setSSLParameters(params); } } } - // Disable the default hostname verification that is performed by underlying trust manager when 'HTTPS' is used as endpoint identification algorithm. - // Some http clients, notably the new http client in Java 11, does not allow user configuration of the endpoint algorithm or custom HostnameVerifier. - private static boolean overrideHostnameVerification(SSLParameters params) { - if (Objects.equals("HTTPS", params.getEndpointIdentificationAlgorithm())) { - params.setEndpointIdentificationAlgorithm(""); - return true; + // Overrides the endpoint identification algorithm specified in the ssl parameters of the ssl engine/socket. + // The underlying trust manager will perform hostname verification if endpoint identification algorithm is set to 'HTTPS'. + // Returns true if the parameter instance was modified + private boolean overrideHostnameVerificationForClient(SSLParameters params) { + String configuredAlgorithm = params.getEndpointIdentificationAlgorithm(); + switch (hostnameVerification) { + case ENABLED: + if (!"HTTPS".equals(configuredAlgorithm)) { + params.setEndpointIdentificationAlgorithm("HTTPS"); + return true; + } + return false; + case DISABLED: + if (configuredAlgorithm != null && !configuredAlgorithm.isEmpty()) { + params.setEndpointIdentificationAlgorithm(""); // disable any configured endpoint identification algorithm + return true; + } + return false; + default: + throw new IllegalStateException("Unknown host verification type: " + hostnameVerification); } - return false; } } diff --git a/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java b/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java index 727a64ae934..00928187f55 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java @@ -46,7 +46,9 @@ public class DefaultTlsContextTest { singletonList(new RequiredPeerCredential(RequiredPeerCredential.Field.CN, new HostGlobPattern("dummy")))))); DefaultTlsContext tlsContext = - new DefaultTlsContext(singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers, AuthorizationMode.ENFORCE, PeerAuthentication.NEED); + new DefaultTlsContext( + singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers, + AuthorizationMode.ENFORCE, PeerAuthentication.NEED, HostnameVerification.ENABLED); SSLEngine sslEngine = tlsContext.createSslEngine(); assertThat(sslEngine).isNotNull(); diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java index f559e9336c8..1cfb70560b8 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java @@ -4,9 +4,9 @@ package com.yahoo.vespa.service.duper; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.log.LogLevel; +import com.yahoo.vespa.service.monitor.DuperModelListener; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -22,12 +22,16 @@ public class DuperModel { private final Map<ApplicationId, ApplicationInfo> applications = new TreeMap<>(); private final List<DuperModelListener> listeners = new ArrayList<>(); + private boolean isComplete = false; public void registerListener(DuperModelListener listener) { applications.values().forEach(listener::applicationActivated); listeners.add(listener); } + public void setCompleteness(boolean isComplete) { this.isComplete = isComplete; } + public boolean isComplete() { return isComplete; } + public boolean contains(ApplicationId applicationId) { return applications.containsKey(applicationId); } @@ -47,6 +51,6 @@ public class DuperModel { public List<ApplicationInfo> getApplicationInfos() { logger.log(LogLevel.DEBUG, "Applications in duper model: " + applications.values().size()); - return Collections.unmodifiableList(new ArrayList<>(applications.values())); + return List.copyOf(applications.values()); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java index 885368810a8..15c461c7f59 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java @@ -12,6 +12,8 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.service.monitor.DuperModelInfraApi; +import com.yahoo.vespa.service.monitor.DuperModelListener; +import com.yahoo.vespa.service.monitor.DuperModelProvider; import com.yahoo.vespa.service.monitor.InfraApplicationApi; import java.util.ArrayList; @@ -27,7 +29,7 @@ import java.util.stream.Stream; /** * @author hakonhall */ -public class DuperModelManager implements DuperModelInfraApi { +public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi { // Infrastructure applications static final ControllerHostApplication controllerHostApplication = new ControllerHostApplication(); @@ -45,6 +47,8 @@ public class DuperModelManager implements DuperModelInfraApi { // The set of active infrastructure ApplicationInfo. Not all are necessarily in the DuperModel for historical reasons. private final Set<ApplicationId> activeInfraInfos = new HashSet<>(10); + private boolean superModelIsComplete = false; + private boolean infraApplicationsIsComplete = false; @Inject public DuperModelManager(ConfigserverConfig configServerConfig, FlagSource flagSource, SuperModelProvider superModelProvider) { @@ -53,7 +57,7 @@ public class DuperModelManager implements DuperModelInfraApi { superModelProvider, new DuperModel(), flagSource, SystemName.from(configServerConfig.system())); } - /** For testing */ + /** Non-private for testing */ DuperModelManager(boolean multitenant, boolean isController, SuperModelProvider superModelProvider, DuperModel duperModel, FlagSource flagSource, SystemName system) { this.duperModel = duperModel; @@ -86,6 +90,16 @@ public class DuperModelManager implements DuperModelInfraApi { duperModel.remove(applicationId); } } + + @Override + public void notifyOfCompleteness(SuperModel superModel) { + synchronized (monitor) { + if (!superModelIsComplete) { + superModelIsComplete = true; + maybeSetDuperModelAsComplete(); + } + } + } }); } @@ -93,6 +107,7 @@ public class DuperModelManager implements DuperModelInfraApi { * Synchronously call {@link DuperModelListener#applicationActivated(ApplicationInfo) listener.applicationActivated()} * for each currently active application, and forward future changes. */ + @Override public void registerListener(DuperModelListener listener) { synchronized (monitor) { duperModel.registerListener(listener); @@ -148,9 +163,25 @@ public class DuperModelManager implements DuperModelInfraApi { } } + @Override + public void infraApplicationsIsNowComplete() { + synchronized (monitor) { + if (!infraApplicationsIsComplete) { + infraApplicationsIsComplete = true; + maybeSetDuperModelAsComplete(); + } + } + } + public List<ApplicationInfo> getApplicationInfos() { synchronized (monitor) { return duperModel.getApplicationInfos(); } } + + private void maybeSetDuperModelAsComplete() { + if (superModelIsComplete && infraApplicationsIsComplete) { + duperModel.setCompleteness(true); + } + } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java index 3cc7010e209..d6e15f6af4e 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java @@ -92,6 +92,10 @@ public class HealthMonitorManager implements MonitorManager, HealthMonitorApi { } @Override + public void bootstrapComplete() { + } + + @Override public ServiceStatusInfo getStatus(ApplicationId applicationId, ClusterId clusterId, ServiceType serviceType, diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java index dd781a02cef..a7579d3f0da 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.manager; -import com.yahoo.vespa.service.duper.DuperModelListener; +import com.yahoo.vespa.service.monitor.DuperModelListener; import com.yahoo.vespa.service.monitor.ServiceStatusProvider; /** diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java index 3490ad4a5d2..2aacc3eadac 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java @@ -51,4 +51,8 @@ public class UnionMonitorManager implements MonitorManager { slobrokMonitorManager.applicationRemoved(id); healthMonitorManager.applicationRemoved(id); } + + @Override + public void bootstrapComplete() { + } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java index d08bba2bd3d..f9e47b6b80a 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java @@ -27,4 +27,7 @@ public interface DuperModelInfraApi { /** Update the DuperModel: A supported infrastructure application has been removed or is not active. */ void infraApplicationRemoved(ApplicationId applicationId); + + /** All infra applications that are supposed to activate on config server bootstrap has been activated. */ + void infraApplicationsIsNowComplete(); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelListener.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelListener.java index a969b6c3f40..f664e5246ca 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelListener.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelListener.java @@ -1,34 +1,45 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.duper; +package com.yahoo.vespa.service.monitor; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.SuperModel; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.service.duper.DuperModel; /** * Interface for listening for changes to the {@link DuperModel}. * - * @author hakon + * @author hakonhall */ public interface DuperModelListener { /** * An application has been activated: * * <ul> - * <li>A synthetic application like the config server application has been added/"activated" + * <li>A synthetic application like the config server application has been added/activated * <li>A super model application has been activated (see * {@link com.yahoo.config.model.api.SuperModelListener#applicationActivated(SuperModel, ApplicationInfo) * SuperModelListener} * </ul> * - * No other threads will concurrently call any methods on this interface. + * <p>No other threads will concurrently call any methods on this interface.</p> */ void applicationActivated(ApplicationInfo application); /** * Application has been removed. * - * No other threads will concurrently call any methods on this interface. + * <p>No other threads will concurrently call any methods on this interface.</p> */ void applicationRemoved(ApplicationId id); + + /** + * During bootstrap of the config server, a number of applications are activated before + * resuming normal operations: The normal "tenant" application (making the super model) and + * the relevant infrastructure applications. Once all of these have been activated, this method + * will be invoked. + * + * <p>No other threads will concurrently call any methods on this interface.</p> + */ + void bootstrapComplete(); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelProvider.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelProvider.java new file mode 100644 index 00000000000..a90fa418054 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelProvider.java @@ -0,0 +1,6 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor; + +public interface DuperModelProvider { + void registerListener(DuperModelListener listener); +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java index e3ea48ca9fe..e7a8d33ea14 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java @@ -73,6 +73,10 @@ public class SlobrokMonitorManagerImpl implements SlobrokApi, MonitorManager { } @Override + public void bootstrapComplete() { + } + + @Override public List<Mirror.Entry> lookup(ApplicationId id, String pattern) { synchronized (monitor) { SlobrokMonitor slobrokMonitor = slobrokMonitors.get(id); diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java index 31fd266649a..dc90035be71 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.service.duper; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.service.monitor.DuperModelListener; import org.junit.Before; import org.junit.Test; diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java index 0f7c0dde357..3fb10f1f24e 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java @@ -47,7 +47,7 @@ public class ExampleModel { Map<ApplicationId, ApplicationInfo> applicationInfos = new HashMap<>(); applicationInfos.put(applicationInfo.getApplicationId(), applicationInfo); - return new SuperModel(applicationInfos); + return new SuperModel(applicationInfos, true); } public static ApplicationBuilder createApplication(String tenant, diff --git a/storage/src/tests/distributor/garbagecollectiontest.cpp b/storage/src/tests/distributor/garbagecollectiontest.cpp index 65c1ac726b5..776cfc14d84 100644 --- a/storage/src/tests/distributor/garbagecollectiontest.cpp +++ b/storage/src/tests/distributor/garbagecollectiontest.cpp @@ -3,6 +3,7 @@ #include <vespa/storageapi/message/removelocation.h> #include <vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h> #include <vespa/storage/distributor/idealstatemanager.h> +#include <vespa/storage/distributor/idealstatemetricsset.h> #include <tests/distributor/distributortestutil.h> #include <vespa/storage/distributor/distributor.h> #include <vespa/document/test/make_document_bucket.h> @@ -35,11 +36,13 @@ struct GarbageCollectionOperationTest : Test, DistributorTestUtil { } // FIXME fragile to assume that send order == node index, but that's the way it currently works - void reply_to_nth_request(GarbageCollectionOperation& op, size_t n, uint32_t bucket_info_checksum) { + void reply_to_nth_request(GarbageCollectionOperation& op, size_t n, + uint32_t bucket_info_checksum, uint32_t n_docs_removed) { auto msg = _sender.command(n); assert(msg->getType() == api::MessageType::REMOVELOCATION); std::shared_ptr<api::StorageReply> reply(msg->makeReply()); auto& gc_reply = dynamic_cast<api::RemoveLocationReply&>(*reply); + gc_reply.set_documents_removed(n_docs_removed); gc_reply.setBucketInfo(api::BucketInfo(bucket_info_checksum, 90, 500)); op.receive(_sender, reply); @@ -56,6 +59,13 @@ struct GarbageCollectionOperationTest : Test, DistributorTestUtil { << entry->getNode(i)->getBucketInfo(); } } + + uint32_t gc_removed_documents_metric() { + auto metric_base = getIdealStateManager().getMetrics().operations[IdealStateOperation::GARBAGE_COLLECTION]; + auto gc_metrics = std::dynamic_pointer_cast<GcMetricSet>(metric_base); + assert(gc_metrics); + return gc_metrics->documents_removed.getValue(); + } }; TEST_F(GarbageCollectionOperationTest, simple) { @@ -63,29 +73,34 @@ TEST_F(GarbageCollectionOperationTest, simple) { op->start(_sender, framework::MilliSecTime(0)); ASSERT_EQ(2, _sender.commands().size()); + EXPECT_EQ(0u, gc_removed_documents_metric()); for (uint32_t i = 0; i < 2; ++i) { std::shared_ptr<api::StorageCommand> msg = _sender.command(i); ASSERT_EQ(msg->getType(), api::MessageType::REMOVELOCATION); auto& tmp = dynamic_cast<api::RemoveLocationCommand&>(*msg); EXPECT_EQ("music.date < 34", tmp.getDocumentSelection()); - reply_to_nth_request(*op, i, 777 + i); + reply_to_nth_request(*op, i, 777 + i, 50); } ASSERT_NO_FATAL_FAILURE(assert_bucket_db_contains({api::BucketInfo(777, 90, 500), api::BucketInfo(778, 90, 500)}, 34)); + EXPECT_EQ(50u, gc_removed_documents_metric()); } TEST_F(GarbageCollectionOperationTest, replica_bucket_info_not_added_to_db_until_all_replies_received) { auto op = create_op(); op->start(_sender, framework::MilliSecTime(0)); ASSERT_EQ(2, _sender.commands().size()); + EXPECT_EQ(0u, gc_removed_documents_metric()); // Respond to 1st request. Should _not_ cause bucket info to be merged into the database yet - reply_to_nth_request(*op, 0, 1234); + reply_to_nth_request(*op, 0, 1234, 70); ASSERT_NO_FATAL_FAILURE(assert_bucket_db_contains({api::BucketInfo(250, 50, 300), api::BucketInfo(250, 50, 300)}, 0)); // Respond to 2nd request. This _should_ cause bucket info to be merged into the database. - reply_to_nth_request(*op, 1, 4567); + reply_to_nth_request(*op, 1, 4567, 60); ASSERT_NO_FATAL_FAILURE(assert_bucket_db_contains({api::BucketInfo(1234, 90, 500), api::BucketInfo(4567, 90, 500)}, 34)); + + EXPECT_EQ(70u, gc_removed_documents_metric()); // Use max of received metrics } TEST_F(GarbageCollectionOperationTest, gc_bucket_info_does_not_overwrite_later_sequenced_bucket_info_writes) { @@ -93,10 +108,10 @@ TEST_F(GarbageCollectionOperationTest, gc_bucket_info_does_not_overwrite_later_s op->start(_sender, framework::MilliSecTime(0)); ASSERT_EQ(2, _sender.commands().size()); - reply_to_nth_request(*op, 0, 1234); + reply_to_nth_request(*op, 0, 1234, 0); // Change to replica on node 0 happens after GC op, but before GC info is merged into the DB. Must not be lost. insertBucketInfo(op->getBucketId(), 0, 7777, 100, 2000); - reply_to_nth_request(*op, 1, 4567); + reply_to_nth_request(*op, 1, 4567, 0); // Bucket info for node 0 is that of the later sequenced operation, _not_ from the earlier GC op. ASSERT_NO_FATAL_FAILURE(assert_bucket_db_contains({api::BucketInfo(7777, 100, 2000), api::BucketInfo(4567, 90, 500)}, 34)); } diff --git a/storage/src/tests/persistence/processalltest.cpp b/storage/src/tests/persistence/processalltest.cpp index 8c0f8853d2d..83f243ed1b2 100644 --- a/storage/src/tests/persistence/processalltest.cpp +++ b/storage/src/tests/persistence/processalltest.cpp @@ -23,11 +23,15 @@ TEST_F(ProcessAllHandlerTest, remove_location) { api::RemoveLocationCommand removeLocation("id.user == 4", makeDocumentBucket(bucketId)); ProcessAllHandler handler(getEnv(), getPersistenceProvider()); spi::Context context(documentapi::LoadType::DEFAULT, 0, 0); - handler.handleRemoveLocation(removeLocation, context); + auto tracker = handler.handleRemoveLocation(removeLocation, context); EXPECT_EQ("DocEntry(1234, 1, id:mail:testdoctype1:n=4:3619.html)\n" "DocEntry(2345, 1, id:mail:testdoctype1:n=4:4008.html)\n", dumpBucket(bucketId)); + + auto reply = std::dynamic_pointer_cast<api::RemoveLocationReply>(tracker->getReply()); + ASSERT_TRUE(reply.get() != nullptr); + EXPECT_EQ(2u, reply->documents_removed()); } TEST_F(ProcessAllHandlerTest, remove_location_document_subset) { @@ -44,7 +48,7 @@ TEST_F(ProcessAllHandlerTest, remove_location_document_subset) { api::RemoveLocationCommand removeLocation("testdoctype1.headerval % 2 == 0", makeDocumentBucket(bucketId)); spi::Context context(documentapi::LoadType::DEFAULT, 0, 0); - handler.handleRemoveLocation(removeLocation, context); + auto tracker = handler.handleRemoveLocation(removeLocation, context); EXPECT_EQ("DocEntry(100, 1, id:mail:testdoctype1:n=4:3619.html)\n" "DocEntry(101, 0, Doc(id:mail:testdoctype1:n=4:33113.html))\n" @@ -57,6 +61,10 @@ TEST_F(ProcessAllHandlerTest, remove_location_document_subset) { "DocEntry(108, 1, id:mail:testdoctype1:n=4:42967.html)\n" "DocEntry(109, 0, Doc(id:mail:testdoctype1:n=4:6925.html))\n", dumpBucket(bucketId)); + + auto reply = std::dynamic_pointer_cast<api::RemoveLocationReply>(tracker->getReply()); + ASSERT_TRUE(reply.get() != nullptr); + EXPECT_EQ(5u, reply->documents_removed()); } TEST_F(ProcessAllHandlerTest, remove_location_throws_exception_on_unknown_doc_type) { diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp index 61e67b40f44..c211e775326 100644 --- a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp +++ b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp @@ -4,8 +4,7 @@ #include <vespa/storage/distributor/distributormetricsset.h> #include <vespa/storage/distributor/idealstatemetricsset.h> -namespace storage { -namespace distributor { +namespace storage::distributor { BucketDBMetricUpdater::Stats::Stats() : _docCount(0), @@ -27,9 +26,7 @@ BucketDBMetricUpdater::BucketDBMetricUpdater() { } -BucketDBMetricUpdater::~BucketDBMetricUpdater() -{ -} +BucketDBMetricUpdater::~BucketDBMetricUpdater() = default; void BucketDBMetricUpdater::resetStats() @@ -148,5 +145,4 @@ BucketDBMetricUpdater::reset() resetStats(); } -} // distributor -} // storage +} // storage::distributor diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h index 6e15ee03d12..7ef8479866f 100644 --- a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h +++ b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h @@ -7,12 +7,9 @@ #include <unordered_map> -namespace storage { +namespace storage::distributor { class DistributorMetricSet; - -namespace distributor { - class IdealStateMetricSet; class BucketDBMetricUpdater { @@ -107,5 +104,4 @@ private: void resetStats(); }; -} // distributor -} // storage +} // storage::distributor diff --git a/storage/src/vespa/storage/distributor/distributorinterface.h b/storage/src/vespa/storage/distributor/distributorinterface.h index aba58e112dc..b17bcd56d19 100644 --- a/storage/src/vespa/storage/distributor/distributorinterface.h +++ b/storage/src/vespa/storage/distributor/distributorinterface.h @@ -12,10 +12,10 @@ namespace storage::api { class MergeBucketReply; } namespace storage::lib { class ClusterStateBundle; } namespace storage { class DistributorConfiguration; - class DistributorMetricSet; } namespace storage::distributor { +class DistributorMetricSet; class PendingMessageTracker; class DistributorInterface : public DistributorMessageSender diff --git a/storage/src/vespa/storage/distributor/distributormetricsset.cpp b/storage/src/vespa/storage/distributor/distributormetricsset.cpp index 244406ca6fb..98e96f9294f 100644 --- a/storage/src/vespa/storage/distributor/distributormetricsset.cpp +++ b/storage/src/vespa/storage/distributor/distributormetricsset.cpp @@ -3,7 +3,7 @@ #include <vespa/metrics/loadmetric.hpp> #include <vespa/metrics/summetric.hpp> -namespace storage { +namespace storage::distributor { using metrics::MetricSet; diff --git a/storage/src/vespa/storage/distributor/distributormetricsset.h b/storage/src/vespa/storage/distributor/distributormetricsset.h index 1e4730b8de6..b5be72e8c14 100644 --- a/storage/src/vespa/storage/distributor/distributormetricsset.h +++ b/storage/src/vespa/storage/distributor/distributormetricsset.h @@ -7,7 +7,7 @@ #include <vespa/metrics/metrics.h> #include <vespa/documentapi/loadtypes/loadtypeset.h> -namespace storage { +namespace storage::distributor { class DistributorMetricSet : public metrics::MetricSet { diff --git a/storage/src/vespa/storage/distributor/externaloperationhandler.h b/storage/src/vespa/storage/distributor/externaloperationhandler.h index 96875a3644a..60cad15a791 100644 --- a/storage/src/vespa/storage/distributor/externaloperationhandler.h +++ b/storage/src/vespa/storage/distributor/externaloperationhandler.h @@ -13,11 +13,11 @@ namespace storage { -class DistributorMetricSet; class PersistenceOperationMetricSet; namespace distributor { +class DistributorMetricSet; class Distributor; class MaintenanceOperationGenerator; class DirectDispatchSender; diff --git a/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp b/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp index d72f4a80ef4..fd193ad6fd8 100644 --- a/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp +++ b/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp @@ -6,7 +6,7 @@ namespace storage { namespace distributor { OperationMetricSet::OperationMetricSet(const std::string& name, metrics::Metric::Tags tags, const std::string& description, MetricSet* owner) - : MetricSet(name, tags, description, owner), + : MetricSet(name, std::move(tags), description, owner), pending("pending", {{"logdefault"},{"yamasdefault"}}, "The number of operations pending", this), @@ -16,14 +16,25 @@ OperationMetricSet::OperationMetricSet(const std::string& name, metrics::Metric: failed("done_failed", {{"logdefault"},{"yamasdefault"}}, "The number of operations that failed", this) -{ } +{} -OperationMetricSet::~OperationMetricSet() { } +OperationMetricSet::~OperationMetricSet() = default; + +GcMetricSet::GcMetricSet(const std::string& name, metrics::Metric::Tags tags, const std::string& description, MetricSet* owner) + : OperationMetricSet(name, std::move(tags), description, owner), + documents_removed("documents_removed", + {{"logdefault"},{"yamasdefault"}}, + "Number of documents removed by GC operations", this) +{} + +GcMetricSet::~GcMetricSet() = default; void IdealStateMetricSet::createOperationMetrics() { typedef IdealStateOperation ISO; operations.resize(ISO::OPERATION_COUNT); + // Note: naked new is used instead of make_shared due to the latter not being + // able to properly transitively deduce the types for the tag initializer lists. operations[ISO::DELETE_BUCKET] = std::shared_ptr<OperationMetricSet>( new OperationMetricSet("delete_bucket", {{"logdefault"},{"yamasdefault"}}, @@ -45,9 +56,9 @@ IdealStateMetricSet::createOperationMetrics() { {{"logdefault"},{"yamasdefault"}}, "Operations to set active/ready state for bucket copies", this)); operations[ISO::GARBAGE_COLLECTION] = std::shared_ptr<OperationMetricSet>( - new OperationMetricSet("garbage_collection", - {{"logdefault"},{"yamasdefault"}}, - "Operations to garbage collect data from buckets", this)); + new GcMetricSet("garbage_collection", + {{"logdefault"},{"yamasdefault"}}, + "Operations to garbage collect data from buckets", this)); } IdealStateMetricSet::IdealStateMetricSet() @@ -81,7 +92,7 @@ IdealStateMetricSet::IdealStateMetricSet() createOperationMetrics(); } -IdealStateMetricSet::~IdealStateMetricSet() { } +IdealStateMetricSet::~IdealStateMetricSet() = default; void IdealStateMetricSet::setPendingOperations(const std::vector<uint64_t>& newMetrics) { for (uint32_t i = 0; i < IdealStateOperation::OPERATION_COUNT; i++) { diff --git a/storage/src/vespa/storage/distributor/idealstatemetricsset.h b/storage/src/vespa/storage/distributor/idealstatemetricsset.h index 7bb472b4a2c..2679da17598 100644 --- a/storage/src/vespa/storage/distributor/idealstatemetricsset.h +++ b/storage/src/vespa/storage/distributor/idealstatemetricsset.h @@ -16,13 +16,21 @@ public: metrics::LongCountMetric failed; OperationMetricSet(const std::string& name, metrics::Metric::Tags tags, const std::string& description, MetricSet* owner); - ~OperationMetricSet(); + ~OperationMetricSet() override; +}; + +struct GcMetricSet : OperationMetricSet { + metrics::LongCountMetric documents_removed; + + GcMetricSet(const std::string& name, metrics::Metric::Tags tags, + const std::string& description, MetricSet* owner); + ~GcMetricSet() override; }; class IdealStateMetricSet : public metrics::MetricSet { public: - std::vector<std::shared_ptr<OperationMetricSet> > operations; + std::vector<std::shared_ptr<OperationMetricSet>> operations; metrics::LongValueMetric idealstate_diff; metrics::LongValueMetric buckets_toofewcopies; metrics::LongValueMetric buckets_toomanycopies; @@ -35,7 +43,7 @@ public: void createOperationMetrics(); IdealStateMetricSet(); - ~IdealStateMetricSet(); + ~IdealStateMetricSet() override; void setPendingOperations(const std::vector<uint64_t>& newMetrics); }; diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp index c674add80f7..fc127c2e0eb 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp @@ -2,6 +2,7 @@ #include "garbagecollectionoperation.h" #include <vespa/storage/distributor/idealstatemanager.h> +#include <vespa/storage/distributor/idealstatemetricsset.h> #include <vespa/storage/distributor/distributor.h> #include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/storageapi/message/removelocation.h> @@ -9,19 +10,18 @@ #include <vespa/log/log.h> LOG_SETUP(".distributor.operation.idealstate.remove"); -using namespace storage::distributor; +namespace storage::distributor { GarbageCollectionOperation::GarbageCollectionOperation(const std::string& clusterName, const BucketAndNodes& nodes) : IdealStateOperation(nodes), _tracker(clusterName), - _replica_info() + _replica_info(), + _max_documents_removed(0) {} GarbageCollectionOperation::~GarbageCollectionOperation() = default; -void -GarbageCollectionOperation::onStart(DistributorMessageSender& sender) -{ +void GarbageCollectionOperation::onStart(DistributorMessageSender& sender) { BucketDatabase::Entry entry = _bucketSpace->getBucketDatabase().get(getBucketId()); std::vector<uint16_t> nodes = entry->getNodes(); @@ -43,7 +43,7 @@ GarbageCollectionOperation::onStart(DistributorMessageSender& sender) void GarbageCollectionOperation::onReceive(DistributorMessageSender&, - const std::shared_ptr<api::StorageReply>& reply) + const std::shared_ptr<api::StorageReply>& reply) { auto* rep = dynamic_cast<api::RemoveLocationReply*>(reply.get()); assert(rep != nullptr); @@ -53,6 +53,7 @@ GarbageCollectionOperation::onReceive(DistributorMessageSender&, if (!rep->getResult().failed()) { _replica_info.emplace_back(_manager->getDistributorComponent().getUniqueTimestamp(), node, rep->getBucketInfo()); + _max_documents_removed = std::max(_max_documents_removed, rep->documents_removed()); } else { _ok = false; } @@ -61,6 +62,7 @@ GarbageCollectionOperation::onReceive(DistributorMessageSender&, if (_ok) { merge_received_bucket_info_into_db(); } + update_gc_metrics(); done(); } } @@ -76,8 +78,16 @@ void GarbageCollectionOperation::merge_received_bucket_info_into_db() { } } +void GarbageCollectionOperation::update_gc_metrics() { + auto metric_base = _manager->getMetrics().operations[IdealStateOperation::GARBAGE_COLLECTION]; + auto gc_metrics = std::dynamic_pointer_cast<GcMetricSet>(metric_base); + assert(gc_metrics); + gc_metrics->documents_removed.inc(_max_documents_removed); +} + bool -GarbageCollectionOperation::shouldBlockThisOperation(uint32_t, uint8_t) const -{ +GarbageCollectionOperation::shouldBlockThisOperation(uint32_t, uint8_t) const { return true; } + +} diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h index 47ea11bb328..28de9592a63 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h @@ -26,8 +26,10 @@ protected: MessageTracker _tracker; private: std::vector<BucketCopy> _replica_info; + uint32_t _max_documents_removed; void merge_received_bucket_info_into_db(); + void update_gc_metrics(); }; } diff --git a/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h b/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h index d951d7ceba2..1299fdad2ad 100644 --- a/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h +++ b/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h @@ -16,7 +16,7 @@ class PersistenceFailuresMetricSet : public metrics::MetricSet { public: explicit PersistenceFailuresMetricSet(metrics::MetricSet* owner); - ~PersistenceFailuresMetricSet(); + ~PersistenceFailuresMetricSet() override; metrics::SumMetric<metrics::LongCountMetric> sum; metrics::LongCountMetric notready; @@ -44,7 +44,7 @@ public: PersistenceFailuresMetricSet failures; PersistenceOperationMetricSet(const std::string& name, metrics::MetricSet* owner = nullptr); - ~PersistenceOperationMetricSet(); + ~PersistenceOperationMetricSet() override; MetricSet * clone(std::vector<Metric::UP>& ownerList, CopyType copyType, metrics::MetricSet* owner, bool includeUnused) const override; diff --git a/storage/src/vespa/storage/persistence/processallhandler.cpp b/storage/src/vespa/storage/persistence/processallhandler.cpp index 8c951a9f50d..5b94a3da027 100644 --- a/storage/src/vespa/storage/persistence/processallhandler.cpp +++ b/storage/src/vespa/storage/persistence/processallhandler.cpp @@ -23,6 +23,7 @@ public: spi::PersistenceProvider& _provider; const spi::Bucket& _bucket; spi::Context& _context; + uint32_t _n_removed; UnrevertableRemoveEntryProcessor( spi::PersistenceProvider& provider, @@ -30,7 +31,9 @@ public: spi::Context& context) : _provider(provider), _bucket(bucket), - _context(context) {} + _context(context), + _n_removed(0) + {} void process(spi::DocEntry& entry) override { spi::RemoveResult removeResult = _provider.remove( @@ -45,13 +48,14 @@ public: << removeResult.getErrorMessage(); throw std::runtime_error(ss.str()); } + ++_n_removed; } }; class StatEntryProcessor : public BucketProcessor::EntryProcessor { public: std::ostream& ost; - StatEntryProcessor(std::ostream& o) + explicit StatEntryProcessor(std::ostream& o) : ost(o) {}; void process(spi::DocEntry& e) override { @@ -97,7 +101,9 @@ ProcessAllHandler::handleRemoveLocation(api::RemoveLocationCommand& cmd, context); spi::Result result = _spi.flush(bucket, context); uint32_t code = _env.convertErrorCode(result); - if (code != 0) { + if (code == 0) { + tracker->setReply(std::make_shared<api::RemoveLocationReply>(cmd, processor._n_removed)); + } else { tracker->fail(code, result.getErrorMessage()); } diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp index 2f959e40e2a..2e5eb115844 100644 --- a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp +++ b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp @@ -522,8 +522,15 @@ TEST_P(StorageProtocolTest, remove_location) { EXPECT_EQ("id.group == \"mygroup\"", cmd2->getDocumentSelection()); EXPECT_EQ(_bucket, cmd2->getBucket()); - auto reply = std::make_shared<RemoveLocationReply>(*cmd2); + uint32_t n_docs_removed = 12345; + auto reply = std::make_shared<RemoveLocationReply>(*cmd2, n_docs_removed); auto reply2 = copyReply(reply); + if (GetParam().getMajor() == 7) { + // Statistics are only available for protobuf-enabled version. + EXPECT_EQ(n_docs_removed, reply2->documents_removed()); + } else { + EXPECT_EQ(0, reply2->documents_removed()); + } } TEST_P(StorageProtocolTest, create_visitor) { diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto b/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto index 810f88f588f..12dbaf59146 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto +++ b/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto @@ -90,7 +90,12 @@ message RemoveLocationRequest { bytes document_selection = 2; } +message RemoveLocationStats { + uint32 documents_removed = 1; +} + message RemoveLocationResponse { BucketInfo bucket_info = 1; BucketId remapped_bucket_id = 2; + RemoveLocationStats stats = 3; } diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp index 9751fd1be98..90c8d1c7d2a 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp @@ -643,7 +643,9 @@ void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveLocationComma } void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveLocationReply& msg) const { - encode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, msg, no_op_encode); + encode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, msg, [&](auto& res) { + res.mutable_stats()->set_documents_removed(msg.documents_removed()); + }); } api::StorageCommand::UP ProtocolSerialization7::onDecodeRemoveLocationCommand(BBuf& buf) const { @@ -653,8 +655,11 @@ api::StorageCommand::UP ProtocolSerialization7::onDecodeRemoveLocationCommand(BB } api::StorageReply::UP ProtocolSerialization7::onDecodeRemoveLocationReply(const SCmd& cmd, BBuf& buf) const { - return decode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, [&]([[maybe_unused]] auto& res) { - return std::make_unique<api::RemoveLocationReply>(static_cast<const api::RemoveLocationCommand&>(cmd)); + return decode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, [&](auto& res) { + uint32_t documents_removed = (res.has_stats() ? res.stats().documents_removed() : 0u); + return std::make_unique<api::RemoveLocationReply>( + static_cast<const api::RemoveLocationCommand&>(cmd), + documents_removed); }); } diff --git a/storageapi/src/vespa/storageapi/message/removelocation.cpp b/storageapi/src/vespa/storageapi/message/removelocation.cpp index b53584601ef..49c9d22f5ee 100644 --- a/storageapi/src/vespa/storageapi/message/removelocation.cpp +++ b/storageapi/src/vespa/storageapi/message/removelocation.cpp @@ -25,8 +25,9 @@ RemoveLocationCommand::print(std::ostream& out, bool verbose, const std::string& BucketInfoCommand::print(out, verbose, indent); } -RemoveLocationReply::RemoveLocationReply(const RemoveLocationCommand& cmd) - : BucketInfoReply(cmd) +RemoveLocationReply::RemoveLocationReply(const RemoveLocationCommand& cmd, uint32_t docs_removed) + : BucketInfoReply(cmd), + _documents_removed(docs_removed) { } diff --git a/storageapi/src/vespa/storageapi/message/removelocation.h b/storageapi/src/vespa/storageapi/message/removelocation.h index 46555497035..812cc8c413b 100644 --- a/storageapi/src/vespa/storageapi/message/removelocation.h +++ b/storageapi/src/vespa/storageapi/message/removelocation.h @@ -11,7 +11,7 @@ class RemoveLocationCommand : public BucketInfoCommand { public: RemoveLocationCommand(vespalib::stringref documentSelection, const document::Bucket &bucket); - ~RemoveLocationCommand(); + ~RemoveLocationCommand() override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; const vespalib::string& getDocumentSelection() const { return _documentSelection; } @@ -22,8 +22,13 @@ private: class RemoveLocationReply : public BucketInfoReply { + uint32_t _documents_removed; public: - RemoveLocationReply(const RemoveLocationCommand& cmd); + explicit RemoveLocationReply(const RemoveLocationCommand& cmd, uint32_t docs_removed = 0); + void set_documents_removed(uint32_t docs_removed) noexcept { + _documents_removed = docs_removed; + } + uint32_t documents_removed() const noexcept { return _documents_removed; } DECLARE_STORAGEREPLY(RemoveLocationReply, onRemoveLocationReply) }; diff --git a/vbench/src/vbench/vbench/vbench.cpp b/vbench/src/vbench/vbench/vbench.cpp index 4f6efadfbdd..58854af705e 100644 --- a/vbench/src/vbench/vbench/vbench.cpp +++ b/vbench/src/vbench/vbench/vbench.cpp @@ -29,11 +29,13 @@ CryptoEngine::SP setup_crypto(const vespalib::slime::Inspector &tls) { if (!tls.valid()) { return std::make_shared<vespalib::NullCryptoEngine>(); } - vespalib::net::tls::TransportSecurityOptions - tls_opts(maybe_load(tls["ca-certificates"]), - maybe_load(tls["certificates"]), - maybe_load(tls["private-key"])); - return std::make_shared<vespalib::TlsCryptoEngine>(tls_opts); + auto ts_builder = vespalib::net::tls::TransportSecurityOptions::Params(). + ca_certs_pem(maybe_load(tls["ca-certificates"])). + cert_chain_pem(maybe_load(tls["certificates"])). + private_key_pem(maybe_load(tls["private-key"])). + authorized_peers(vespalib::net::tls::AuthorizedPeers::allow_all_authenticated()). + disable_hostname_validation(true); // TODO configurable or default false! + return std::make_shared<vespalib::TlsCryptoEngine>(vespalib::net::tls::TransportSecurityOptions(std::move(ts_builder))); } } // namespace vbench::<unnamed> diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java index 0e202d1f348..59953fbe002 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java @@ -70,7 +70,6 @@ public class Runner { Optional.ofNullable(commandLineArgs.getFile()), commandLineArgs.getAddRootElementToXml()); - int intervalOfLogging = commandLineArgs.getVerbose() ? commandLineArgs.getWhenVerboseEnabledPrintMessageForEveryXDocuments() : Integer.MAX_VALUE; @@ -86,13 +85,15 @@ public class Runner { if (commandLineArgs.getVerbose()) { System.err.println(feedClient.getStatsAsJson()); - double fileSizeMb = ((double) new File(commandLineArgs.getFile()).length()) / 1024.0 / 1024.0; double transferTimeSec = ((double) sendTotalTimeMs) / 1000.0; - System.err.println("Sent " + fileSizeMb + " MB in " + transferTimeSec + " seconds."); - System.err.println("Speed: " + ((fileSizeMb / transferTimeSec) * 8.0) + " Mbits/sec, + HTTP overhead " + - "(not taking compression into account)"); if (transferTimeSec > 0) { - System.err.printf("Docs/sec %.3f%n\n", numSent.get() / transferTimeSec); + System.err.printf("Docs/sec %.3f%n", numSent.get() / transferTimeSec); + } + if (commandLineArgs.getFile() != null) { + double fileSizeMb = ((double) new File(commandLineArgs.getFile()).length()) / 1024.0 / 1024.0; + System.err.println("Sent " + fileSizeMb + " MB in " + transferTimeSec + " seconds."); + System.err.println("Speed: " + ((fileSizeMb / transferTimeSec) * 8.0) + " Mbits/sec, + HTTP overhead " + + "(not taking compression into account)"); } } callback.printProgress(); diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java index ed079442440..90f7a76b356 100644 --- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java @@ -61,8 +61,7 @@ public class TestRunnerHandler extends LoggingRequestHandler { private HttpResponse handleGET(HttpRequest request) { String path = request.getUri().getPath(); - // TODO: Migrate to /tester/v1/log when /tester/v1/log2 is not in use anymore (and remove /tester/v1/log2) - if (path.equals("/tester/v1/log") || path.equals("/tester/v1/log2")) { + if (path.equals("/tester/v1/log")) { return new SlimeJsonResponse(logToSlime(testRunner.getLog(request.hasProperty("after") ? Long.parseLong(request.getProperty("after")) : -1))); @@ -91,7 +90,7 @@ public class TestRunnerHandler extends LoggingRequestHandler { path = path.substring(0, path.length() - 1); int lastSlash = path.lastIndexOf("/"); if (lastSlash < 0) return path; - return path.substring(lastSlash + 1, path.length()); + return path.substring(lastSlash + 1); } static Slime logToSlime(Collection<LogRecord> log) { diff --git a/vespabase/src/rhel-prestart.sh b/vespabase/src/rhel-prestart.sh index 3a770cdd785..d84122898a3 100755 --- a/vespabase/src/rhel-prestart.sh +++ b/vespabase/src/rhel-prestart.sh @@ -96,32 +96,32 @@ fixdir () { # BEGIN directory fixups -fixdir root wheel 1777 logs -fixdir root wheel 1777 tmp -fixdir root wheel 1777 var/run -fixdir ${VESPA_USER} wheel 1777 var/crash -fixdir ${VESPA_USER} wheel 1777 logs/vespa -fixdir ${VESPA_USER} wheel 1777 tmp/vespa -fixdir ${VESPA_USER} wheel 755 var -fixdir ${VESPA_USER} wheel 755 libexec/vespa/plugins/qrs -fixdir ${VESPA_USER} wheel 755 logs/vespa/configserver -fixdir ${VESPA_USER} wheel 755 logs/vespa/qrs -fixdir ${VESPA_USER} wheel 755 logs/vespa/search -fixdir ${VESPA_USER} wheel 755 var/db/vespa -fixdir ${VESPA_USER} wheel 755 var/db/vespa/tmp -fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server -fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server/serverdb -fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server/serverdb/tenants -fixdir ${VESPA_USER} wheel 755 var/db/vespa/filedistribution -fixdir ${VESPA_USER} wheel 755 var/db/vespa/index -fixdir ${VESPA_USER} wheel 755 var/db/vespa/logcontrol -fixdir ${VESPA_USER} wheel 755 var/db/vespa/search -fixdir ${VESPA_USER} wheel 755 var/jdisc_container -fixdir ${VESPA_USER} wheel 755 var/vespa -fixdir ${VESPA_USER} wheel 755 var/vespa/application -fixdir ${VESPA_USER} wheel 755 var/vespa/bundlecache -fixdir ${VESPA_USER} wheel 755 var/vespa/bundlecache/configserver -fixdir ${VESPA_USER} wheel 755 var/vespa/cache/config/ +fixdir root root 1777 logs +fixdir root root 1777 tmp +fixdir root root 1777 var/run +fixdir ${VESPA_USER} root 1777 var/crash +fixdir ${VESPA_USER} root 1777 logs/vespa +fixdir ${VESPA_USER} root 1777 tmp/vespa +fixdir root root 755 var +fixdir ${VESPA_USER} root 755 libexec/vespa/plugins/qrs +fixdir ${VESPA_USER} root 755 logs/vespa/configserver +fixdir ${VESPA_USER} root 755 logs/vespa/qrs +fixdir ${VESPA_USER} root 755 logs/vespa/search +fixdir ${VESPA_USER} root 755 var/db/vespa +fixdir ${VESPA_USER} root 755 var/db/vespa/tmp +fixdir ${VESPA_USER} root 755 var/db/vespa/config_server +fixdir ${VESPA_USER} root 755 var/db/vespa/config_server/serverdb +fixdir ${VESPA_USER} root 755 var/db/vespa/config_server/serverdb/tenants +fixdir ${VESPA_USER} root 755 var/db/vespa/filedistribution +fixdir ${VESPA_USER} root 755 var/db/vespa/index +fixdir ${VESPA_USER} root 755 var/db/vespa/logcontrol +fixdir ${VESPA_USER} root 755 var/db/vespa/search +fixdir ${VESPA_USER} root 755 var/jdisc_container +fixdir ${VESPA_USER} root 755 var/vespa +fixdir ${VESPA_USER} root 755 var/vespa/application +fixdir ${VESPA_USER} root 755 var/vespa/bundlecache +fixdir ${VESPA_USER} root 755 var/vespa/bundlecache/configserver +fixdir ${VESPA_USER} root 755 var/vespa/cache/config/ if [ "${VESPA_UNPRIVILEGED}" != yes ]; then chown -hR ${VESPA_USER} logs/vespa diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/Http.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/Http.pm index 2dbf475f2a7..d907e89fa54 100644 --- a/vespaclient/src/perl/lib/Yahoo/Vespa/Http.pm +++ b/vespaclient/src/perl/lib/Yahoo/Vespa/Http.pm @@ -100,7 +100,10 @@ sub initialize { # () my $tls_enabled = $ENV{'VESPA_TLS_ENABLED'}; if (defined $tls_enabled and $tls_enabled eq '1') { $BROWSER->ssl_opts( SSL_version => 'TLSv12'); - $BROWSER->ssl_opts( verify_hostname => 0); + my $hostname_verification_disabled = $ENV{'VESPA_TLS_HOSTNAME_VALIDATION_DISABLED'}; + if (defined $hostname_verification_disabled and $hostname_verification_disabled eq '1') { + $BROWSER->ssl_opts( verify_hostname => 0); + } $BROWSER->ssl_opts( SSL_cipher_list => 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-CHACHA20-POLY1305:TLS13-AES-128-GCM-SHA256:TLS13-AES-256-GCM-SHA384:TLS13-CHACHA20-POLY1305-SHA256' ); } if (defined $ENV{'VESPA_TLS_CA_CERT'}) { diff --git a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp index 78838ce2cd2..54c8c19fc64 100644 --- a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp +++ b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp @@ -57,11 +57,23 @@ void print_decode_result(const char* mode, const DecodeResult& res) { decode_state_to_str(res.state)); } +TransportSecurityOptions ts_from_pems(vespalib::stringref ca_certs_pem, + vespalib::stringref cert_chain_pem, + vespalib::stringref private_key_pem) +{ + auto ts_builder = TransportSecurityOptions::Params(). + ca_certs_pem(ca_certs_pem). + cert_chain_pem(cert_chain_pem). + private_key_pem(private_key_pem). + authorized_peers(AuthorizedPeers::allow_all_authenticated()); + return TransportSecurityOptions(std::move(ts_builder)); +} + struct Fixture { TransportSecurityOptions tls_opts; std::shared_ptr<TlsContext> tls_ctx; - std::unique_ptr<CryptoCodec> client; - std::unique_ptr<CryptoCodec> server; + std::unique_ptr<OpenSslCryptoCodecImpl> client; + std::unique_ptr<OpenSslCryptoCodecImpl> server; SmartBuffer client_to_server; SmartBuffer server_to_client; @@ -77,16 +89,21 @@ struct Fixture { static TransportSecurityOptions create_options_without_own_peer_cert() { auto source_opts = vespalib::test::make_tls_options_for_testing(); - return TransportSecurityOptions(source_opts.ca_certs_pem(), "", ""); + return ts_from_pems(source_opts.ca_certs_pem(), "", ""); } - static std::unique_ptr<CryptoCodec> create_openssl_codec( - const TransportSecurityOptions& opts, CryptoCodec::Mode mode) { + static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec( + const TransportSecurityOptions& opts, CryptoCodec::Mode mode, const SocketSpec& peer_spec) { auto ctx = TlsContext::create_default_context(opts, AuthorizationMode::Enforce); - return create_openssl_codec(ctx, mode); + return create_openssl_codec(ctx, mode, peer_spec); + } + + static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec( + const TransportSecurityOptions& opts, CryptoCodec::Mode mode) { + return create_openssl_codec(opts, mode, SocketSpec::invalid); } - static std::unique_ptr<CryptoCodec> create_openssl_codec( + static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec( const TransportSecurityOptions& opts, std::shared_ptr<CertificateVerificationCallback> cert_verify_callback, CryptoCodec::Mode mode) { @@ -94,21 +111,30 @@ struct Fixture { return create_openssl_codec(ctx, mode); } - static std::unique_ptr<CryptoCodec> create_openssl_codec( - const std::shared_ptr<TlsContext>& ctx, CryptoCodec::Mode mode) { + static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec( + const std::shared_ptr<TlsContext>& ctx, CryptoCodec::Mode mode, const SocketSpec& peer_spec) { auto ctx_impl = std::dynamic_pointer_cast<impl::OpenSslTlsContextImpl>(ctx); - return std::make_unique<impl::OpenSslCryptoCodecImpl>(std::move(ctx_impl), SocketAddress(), mode); + if (mode == CryptoCodec::Mode::Client) { + return OpenSslCryptoCodecImpl::make_client_codec(std::move(ctx_impl), peer_spec, SocketAddress()); + } else { + return OpenSslCryptoCodecImpl::make_server_codec(std::move(ctx_impl), SocketAddress()); + } } - EncodeResult do_encode(CryptoCodec& codec, Output& buffer, vespalib::stringref plaintext) { + static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec( + const std::shared_ptr<TlsContext>& ctx, CryptoCodec::Mode mode) { + return create_openssl_codec(ctx, mode, SocketSpec::invalid); + } + + static EncodeResult do_encode(CryptoCodec& codec, Output& buffer, vespalib::stringref plaintext) { auto out = buffer.reserve(codec.min_encode_buffer_size()); auto enc_res = codec.encode(plaintext.data(), plaintext.size(), out.data, out.size); buffer.commit(enc_res.bytes_produced); return enc_res; } - DecodeResult do_decode(CryptoCodec& codec, Input& buffer, vespalib::string& out, - size_t max_bytes_produced, size_t max_bytes_consumed) { + static DecodeResult do_decode(CryptoCodec& codec, Input& buffer, vespalib::string& out, + size_t max_bytes_produced, size_t max_bytes_consumed) { auto in = buffer.obtain(); out.resize(max_bytes_produced); auto to_consume = std::min(in.size, max_bytes_consumed); @@ -382,13 +408,13 @@ l9pLv1vrujrPEC78cyIQe2x55wf3pRoaDg== -----END EC PRIVATE KEY-----)"; TEST_F("client with certificate signed by untrusted CA is rejected by server", Fixture) { - TransportSecurityOptions client_opts(unknown_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem); + auto client_opts = ts_from_pems(unknown_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem); f.client = f.create_openssl_codec(client_opts, CryptoCodec::Mode::Client); EXPECT_FALSE(f.handshake()); } TEST_F("server with certificate signed by untrusted CA is rejected by client", Fixture) { - TransportSecurityOptions server_opts(unknown_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem); + auto server_opts = ts_from_pems(unknown_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem); f.server = f.create_openssl_codec(server_opts, CryptoCodec::Mode::Server); EXPECT_FALSE(f.handshake()); } @@ -396,8 +422,8 @@ TEST_F("server with certificate signed by untrusted CA is rejected by client", F TEST_F("Can specify multiple trusted CA certs in transport options", Fixture) { auto& base_opts = f.tls_opts; auto multi_ca_pem = base_opts.ca_certs_pem() + "\n" + unknown_ca_pem; - TransportSecurityOptions multi_ca_using_ca_1(multi_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem); - TransportSecurityOptions multi_ca_using_ca_2(multi_ca_pem, base_opts.cert_chain_pem(), base_opts.private_key_pem()); + auto multi_ca_using_ca_1 = ts_from_pems(multi_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem); + auto multi_ca_using_ca_2 = ts_from_pems(multi_ca_pem, base_opts.cert_chain_pem(), base_opts.private_key_pem()); // Let client be signed by CA 1, server by CA 2. Both have the two CAs in their trust store // so this should allow for a successful handshake. f.client = f.create_openssl_codec(multi_ca_using_ca_1, CryptoCodec::Mode::Client); @@ -446,7 +472,7 @@ struct CertFixture : Fixture { return {std::move(cert), std::move(key)}; } - static std::unique_ptr<CryptoCodec> create_openssl_codec_with_authz_mode( + static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec_with_authz_mode( const TransportSecurityOptions& opts, std::shared_ptr<CertificateVerificationCallback> cert_verify_callback, CryptoCodec::Mode codec_mode, @@ -455,33 +481,52 @@ struct CertFixture : Fixture { return create_openssl_codec(ctx, codec_mode); } + TransportSecurityOptions::Params ts_builder_from(const CertKeyWrapper& ck) const { + return TransportSecurityOptions::Params(). + ca_certs_pem(root_ca.cert->to_pem()). + cert_chain_pem(ck.cert->to_pem()). + private_key_pem(ck.key->private_to_pem()); + } + void reset_client_with_cert_opts(const CertKeyWrapper& ck, AuthorizedPeers authorized) { - TransportSecurityOptions client_opts(root_ca.cert->to_pem(), ck.cert->to_pem(), - ck.key->private_to_pem(), std::move(authorized)); - client = create_openssl_codec(client_opts, CryptoCodec::Mode::Client); + auto ts_params = ts_builder_from(ck).authorized_peers(std::move(authorized)); + client = create_openssl_codec(TransportSecurityOptions(std::move(ts_params)), CryptoCodec::Mode::Client); } void reset_client_with_cert_opts(const CertKeyWrapper& ck, std::shared_ptr<CertificateVerificationCallback> cert_cb) { - TransportSecurityOptions client_opts(root_ca.cert->to_pem(), ck.cert->to_pem(), ck.key->private_to_pem()); - client = create_openssl_codec(client_opts, std::move(cert_cb), CryptoCodec::Mode::Client); + auto ts_params = ts_builder_from(ck).authorized_peers(AuthorizedPeers::allow_all_authenticated()); + client = create_openssl_codec(TransportSecurityOptions(std::move(ts_params)), + std::move(cert_cb), CryptoCodec::Mode::Client); } void reset_server_with_cert_opts(const CertKeyWrapper& ck, AuthorizedPeers authorized) { - TransportSecurityOptions server_opts(root_ca.cert->to_pem(), ck.cert->to_pem(), - ck.key->private_to_pem(), std::move(authorized)); - server = create_openssl_codec(server_opts, CryptoCodec::Mode::Server); + auto ts_params = ts_builder_from(ck).authorized_peers(std::move(authorized)); + server = create_openssl_codec(TransportSecurityOptions(std::move(ts_params)), CryptoCodec::Mode::Server); } void reset_server_with_cert_opts(const CertKeyWrapper& ck, std::shared_ptr<CertificateVerificationCallback> cert_cb) { - TransportSecurityOptions server_opts(root_ca.cert->to_pem(), ck.cert->to_pem(), ck.key->private_to_pem()); - server = create_openssl_codec(server_opts, std::move(cert_cb), CryptoCodec::Mode::Server); + auto ts_params = ts_builder_from(ck).authorized_peers(AuthorizedPeers::allow_all_authenticated()); + server = create_openssl_codec(TransportSecurityOptions(std::move(ts_params)), + std::move(cert_cb), CryptoCodec::Mode::Server); } void reset_server_with_cert_opts(const CertKeyWrapper& ck, std::shared_ptr<CertificateVerificationCallback> cert_cb, AuthorizationMode authz_mode) { - TransportSecurityOptions server_opts(root_ca.cert->to_pem(), ck.cert->to_pem(), ck.key->private_to_pem()); - server = create_openssl_codec_with_authz_mode(server_opts, std::move(cert_cb), CryptoCodec::Mode::Server, authz_mode); + auto ts_params = ts_builder_from(ck).authorized_peers(AuthorizedPeers::allow_all_authenticated()); + server = create_openssl_codec_with_authz_mode(TransportSecurityOptions(std::move(ts_params)), + std::move(cert_cb), CryptoCodec::Mode::Server, authz_mode); + } + + void reset_client_with_peer_spec(const CertKeyWrapper& ck, + const SocketSpec& peer_spec, + bool disable_hostname_validation = false) + { + auto ts_params = ts_builder_from(ck). + authorized_peers(AuthorizedPeers::allow_all_authenticated()). + disable_hostname_validation(disable_hostname_validation); + client = create_openssl_codec(TransportSecurityOptions(std::move(ts_params)), + CryptoCodec::Mode::Client, peer_spec); } }; @@ -537,7 +582,7 @@ TEST_F("Exception during verification callback processing breaks handshake", Cer EXPECT_FALSE(f.handshake()); } -TEST_F("certificate verification callback observes CN and DNS SANs", CertFixture) { +TEST_F("Certificate verification callback observes CN and DNS SANs", CertFixture) { auto ck = f.create_ca_issued_peer_cert( {{"rockets.wile.example.com"}}, {{"DNS:crash.wile.example.com"}, {"DNS:burn.wile.example.com"}}); @@ -556,7 +601,7 @@ TEST_F("certificate verification callback observes CN and DNS SANs", CertFixture EXPECT_EQUAL("burn.wile.example.com", creds.dns_sans[1]); } -TEST_F("last occurring CN is given to verification callback if multiple CNs are present", CertFixture) { +TEST_F("Last occurring CN is given to verification callback if multiple CNs are present", CertFixture) { auto ck = f.create_ca_issued_peer_cert( {{"foo.wile.example.com"}, {"bar.wile.example.com"}, {"baz.wile.example.com"}}, {}); @@ -646,6 +691,51 @@ TEST_F("Disabled insecure authorization mode ignores verification result", CertF EXPECT_TRUE(f.handshake()); } +void reset_peers_with_client_peer_spec(CertFixture& f, + const SocketSpec& peer_spec, + bool disable_hostname_validation = false) +{ + auto client_ck = f.create_ca_issued_peer_cert({"hello.world.example.com"}, {}); + f.reset_client_with_peer_spec(client_ck, peer_spec, disable_hostname_validation); + // Since hostname validation is enabled by default, providing a peer spec also + // means that we must have a valid server name to present back (or the handshake fails). + auto server_ck = f.create_ca_issued_peer_cert({}, {{"DNS:*.example.com"}}); + f.reset_server_with_cert_opts(server_ck, AuthorizedPeers::allow_all_authenticated()); +} + +TEST_F("Client does not send SNI extension if hostname not provided in spec", CertFixture) { + reset_peers_with_client_peer_spec(f, SocketSpec::invalid); + + ASSERT_TRUE(f.handshake()); + auto maybe_sni = f.server->client_provided_sni_extension(); + EXPECT_FALSE(maybe_sni.has_value()); +} + +TEST_F("Client sends SNI extension with hostname provided in spec", CertFixture) { + reset_peers_with_client_peer_spec(f, SocketSpec::from_host_port("sni-test.example.com", 12345)); + + ASSERT_TRUE(f.handshake()); + auto maybe_sni = f.server->client_provided_sni_extension(); + ASSERT_TRUE(maybe_sni.has_value()); + EXPECT_EQUAL("sni-test.example.com", *maybe_sni); +} + +TEST_F("Client hostname validation passes handshake if server hostname matches certificate", CertFixture) { + reset_peers_with_client_peer_spec(f, SocketSpec::from_host_port("server-must-be-under.example.com", 12345), false); + EXPECT_TRUE(f.handshake()); +} + +TEST_F("Client hostname validation fails handshake if server hostname mismatches certificate", CertFixture) { + // Wildcards only apply to a single level, so this should fail as the server only has a cert for *.example.com + reset_peers_with_client_peer_spec(f, SocketSpec::from_host_port("nested.name.example.com", 12345), false); + EXPECT_FALSE(f.handshake()); +} + +TEST_F("Mismatching server cert vs hostname does not fail if hostname validation is disabled", CertFixture) { + reset_peers_with_client_peer_spec(f, SocketSpec::from_host_port("a.very.nested.name.example.com", 12345), true); + EXPECT_TRUE(f.handshake()); +} + TEST_F("Failure statistics are incremented on authorization failures", CertFixture) { reset_peers_with_server_authz_mode(f, AuthorizationMode::Enforce); auto server_before = ConnectionStatistics::get(true).snapshot(); diff --git a/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp b/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp index a54e2f29aa1..00459a4e69c 100644 --- a/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp +++ b/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp @@ -155,6 +155,47 @@ TEST("accepted cipher list is populated if specified") { EXPECT_EQUAL("bar", ciphers[1]); } +// FIXME this is temporary until we know enabling it by default won't break the world! +TEST("hostname validation is DISABLED by default when creating options from config file") { + const char* json = R"({"files":{"private-key":"dummy_privkey.txt", + "certificates":"dummy_certs.txt", + "ca-certificates":"dummy_ca_certs.txt"}})"; + EXPECT_TRUE(read_options_from_json_string(json)->disable_hostname_validation()); +} + +TEST("TransportSecurityOptions builder does not disable hostname validation by default") { + auto ts_builder = vespalib::net::tls::TransportSecurityOptions::Params(). + ca_certs_pem("foo"). + cert_chain_pem("bar"). + private_key_pem("fantonald"); + TransportSecurityOptions ts_opts(std::move(ts_builder)); + EXPECT_FALSE(ts_opts.disable_hostname_validation()); +} + +TEST("hostname validation can be explicitly disabled") { + const char* json = R"({"files":{"private-key":"dummy_privkey.txt", + "certificates":"dummy_certs.txt", + "ca-certificates":"dummy_ca_certs.txt"}, + "disable-hostname-validation": true})"; + EXPECT_TRUE(read_options_from_json_string(json)->disable_hostname_validation()); +} + +TEST("hostname validation can be explicitly enabled") { + const char* json = R"({"files":{"private-key":"dummy_privkey.txt", + "certificates":"dummy_certs.txt", + "ca-certificates":"dummy_ca_certs.txt"}, + "disable-hostname-validation": false})"; + EXPECT_FALSE(read_options_from_json_string(json)->disable_hostname_validation()); +} + +TEST("unknown fields are ignored at parse-time") { + const char* json = R"({"files":{"private-key":"dummy_privkey.txt", + "certificates":"dummy_certs.txt", + "ca-certificates":"dummy_ca_certs.txt"}, + "flipper-the-dolphin": "*weird dolphin noises*"})"; + EXPECT_TRUE(read_options_from_json_string(json).get() != nullptr); // And no exception thrown. +} + // TODO test parsing of multiple policies TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/vespa/vespalib/data/slime/object_value.h b/vespalib/src/vespa/vespalib/data/slime/object_value.h index 377bf5bd37f..651f3a156d2 100644 --- a/vespalib/src/vespa/vespalib/data/slime/object_value.h +++ b/vespalib/src/vespa/vespalib/data/slime/object_value.h @@ -12,8 +12,7 @@ #include <vespa/vespalib/stllike/vector_map.h> #include <vespa/vespalib/util/stash.h> -namespace vespalib { -namespace slime { +namespace vespalib::slime { /** * Class representing a collection of unordered values that can be @@ -32,7 +31,7 @@ private: Cursor &setIfUnset(SymbolInserter &symbol, const ValueFactory &input) { Value *&pos = _fields[symbol.insert()]; - if (pos != 0) { + if (pos != nullptr) { return *NixValue::invalid(); } pos = input.create(_stash); @@ -40,7 +39,7 @@ private: } Value *lookup(const SymbolLookup &symbol) const { - SymbolValueMap::const_iterator pos = _fields.find(symbol.lookup()); + auto pos = _fields.find(symbol.lookup()); if (pos == _fields.end()) { return NixValue::invalid(); } @@ -81,9 +80,7 @@ public: Cursor &setObject(Memory name) override; Symbol resolve(Memory symbol_name) override; - ~ObjectValue() { } + ~ObjectValue() override = default; }; -} // namespace vespalib::slime -} // namespace vespalib - +} diff --git a/vespalib/src/vespa/vespalib/geo/zcurve.cpp b/vespalib/src/vespa/vespalib/geo/zcurve.cpp index b2dc02759dc..00ce7ee18c6 100644 --- a/vespalib/src/vespa/vespalib/geo/zcurve.cpp +++ b/vespalib/src/vespa/vespalib/geo/zcurve.cpp @@ -4,8 +4,7 @@ #include <vespa/vespalib/util/priority_queue.h> #include <vespa/vespalib/util/fiddle.h> -namespace vespalib { -namespace geo { +namespace vespalib::geo { namespace { @@ -182,4 +181,3 @@ ZCurve::decodeSlow(int64_t enc, int32_t *xp, int32_t *yp) } } -} diff --git a/vespalib/src/vespa/vespalib/geo/zcurve.h b/vespalib/src/vespa/vespalib/geo/zcurve.h index 2a05c4c7744..bd76b78ea23 100644 --- a/vespalib/src/vespa/vespalib/geo/zcurve.h +++ b/vespalib/src/vespa/vespalib/geo/zcurve.h @@ -6,8 +6,7 @@ #include <cassert> #include <vector> -namespace vespalib { -namespace geo { +namespace vespalib::geo { /** * @brief Utility methods for a Z-curve (Morton-order) encoder and decoder. @@ -31,7 +30,7 @@ public: public: BoundingBox(int32_t minx, int32_t maxx, int32_t miny, int32_t maxy); - ~BoundingBox() { } + ~BoundingBox() = default; int64_t getzMinx() const { return _zMinx; } int64_t getzMaxx() const { return _zMaxx; } @@ -221,5 +220,3 @@ public: }; } -} - diff --git a/vespalib/src/vespa/vespalib/net/crypto_engine.h b/vespalib/src/vespa/vespalib/net/crypto_engine.h index 4deacf9a6c7..71511b8a552 100644 --- a/vespalib/src/vespa/vespalib/net/crypto_engine.h +++ b/vespalib/src/vespa/vespalib/net/crypto_engine.h @@ -19,6 +19,8 @@ class SocketSpec; **/ struct CryptoEngine { using SP = std::shared_ptr<CryptoEngine>; + virtual bool use_tls_when_client() const = 0; + virtual bool always_use_tls_when_server() const = 0; virtual CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) = 0; virtual CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) = 0; virtual ~CryptoEngine(); @@ -29,6 +31,8 @@ struct CryptoEngine { * Crypto engine without encryption. **/ struct NullCryptoEngine : public CryptoEngine { + bool use_tls_when_client() const override { return false; } + bool always_use_tls_when_server() const override { return false; } CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override; }; @@ -39,6 +43,8 @@ struct NullCryptoEngine : public CryptoEngine { * from TLS. **/ struct XorCryptoEngine : public CryptoEngine { + bool use_tls_when_client() const override { return false; } + bool always_use_tls_when_server() const override { return false; } CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override; }; diff --git a/vespalib/src/vespa/vespalib/net/socket_spec.cpp b/vespalib/src/vespa/vespalib/net/socket_spec.cpp index 06682086670..d1dd81a454a 100644 --- a/vespalib/src/vespa/vespalib/net/socket_spec.cpp +++ b/vespalib/src/vespa/vespalib/net/socket_spec.cpp @@ -41,7 +41,7 @@ SocketSpec::address(bool server) const return SocketAddress(); } -SocketSpec SocketSpec::invalid; +const SocketSpec SocketSpec::invalid; SocketSpec::SocketSpec(const vespalib::string &spec) : SocketSpec() diff --git a/vespalib/src/vespa/vespalib/net/socket_spec.h b/vespalib/src/vespa/vespalib/net/socket_spec.h index 01af382d638..4e3dddf6814 100644 --- a/vespalib/src/vespa/vespalib/net/socket_spec.h +++ b/vespalib/src/vespa/vespalib/net/socket_spec.h @@ -24,7 +24,7 @@ private: : _type(type), _node(node), _port(port) {} SocketAddress address(bool server) const; public: - static SocketSpec invalid; + static const SocketSpec invalid; explicit SocketSpec(const vespalib::string &spec); vespalib::string spec() const; SocketSpec replace_host(const vespalib::string &new_host) const; diff --git a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp index c425ab75ce8..bdb2402adbc 100644 --- a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp @@ -99,6 +99,18 @@ CryptoSocket::UP AutoReloadingTlsCryptoEngine::create_server_crypto_socket(Socke return acquire_current_engine()->create_server_crypto_socket(std::move(socket)); } +bool +AutoReloadingTlsCryptoEngine::use_tls_when_client() const +{ + return acquire_current_engine()->use_tls_when_client(); +} + +bool +AutoReloadingTlsCryptoEngine::always_use_tls_when_server() const +{ + return acquire_current_engine()->always_use_tls_when_server(); +} + std::unique_ptr<TlsCryptoSocket> AutoReloadingTlsCryptoEngine::create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) { return acquire_current_engine()->create_tls_client_crypto_socket(std::move(socket), spec); diff --git a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h index e268cbc8f1a..1b80b782daf 100644 --- a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h +++ b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h @@ -47,6 +47,8 @@ public: CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override; + bool use_tls_when_client() const override; + bool always_use_tls_when_server() const override; std::unique_ptr<TlsCryptoSocket> create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; std::unique_ptr<TlsCryptoSocket> create_tls_server_crypto_socket(SocketHandle socket) override; }; diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.cpp b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.cpp index c54990b3782..d3ac975d90a 100644 --- a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.cpp @@ -6,12 +6,23 @@ namespace vespalib::net::tls { -std::unique_ptr<CryptoCodec> CryptoCodec::create_default_codec( - std::shared_ptr<TlsContext> ctx, const SocketAddress& peer_address, Mode mode) +std::unique_ptr<CryptoCodec> +CryptoCodec::create_default_client_codec(std::shared_ptr<TlsContext> ctx, + const SocketSpec& peer_spec, + const SocketAddress& peer_address) { auto ctx_impl = std::dynamic_pointer_cast<impl::OpenSslTlsContextImpl>(ctx); // only takes by const ref assert(ctx_impl); - return std::make_unique<impl::OpenSslCryptoCodecImpl>(std::move(ctx_impl), peer_address, mode); + return impl::OpenSslCryptoCodecImpl::make_client_codec(std::move(ctx_impl), peer_spec, peer_address); +} + +std::unique_ptr<CryptoCodec> +CryptoCodec::create_default_server_codec(std::shared_ptr<TlsContext> ctx, + const SocketAddress& peer_address) +{ + auto ctx_impl = std::dynamic_pointer_cast<impl::OpenSslTlsContextImpl>(ctx); // only takes by const ref + assert(ctx_impl); + return impl::OpenSslCryptoCodecImpl::make_server_codec(std::move(ctx_impl), peer_address); } } diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h index 5d9684461d7..787485b47be 100644 --- a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h +++ b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h @@ -4,6 +4,8 @@ #include <vespa/vespalib/net/socket_address.h> #include <memory> +namespace vespalib { class SocketSpec; } + namespace vespalib::net::tls { struct HandshakeResult { @@ -179,9 +181,13 @@ public: * * Throws CryptoException if resources cannot be allocated for the codec. */ - static std::unique_ptr<CryptoCodec> create_default_codec(std::shared_ptr<TlsContext> ctx, - const SocketAddress& peer_address, - Mode mode); + static std::unique_ptr<CryptoCodec> + create_default_client_codec(std::shared_ptr<TlsContext> ctx, + const SocketSpec& peer_spec, + const SocketAddress& peer_address); + static std::unique_ptr<CryptoCodec> + create_default_server_codec(std::shared_ptr<TlsContext> ctx, + const SocketAddress& peer_address); }; } diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp index 5315754d53a..6a79caa8264 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp @@ -172,9 +172,11 @@ void log_ssl_error(const char* source, const SocketAddress& peer_address, int ss } // anon ns OpenSslCryptoCodecImpl::OpenSslCryptoCodecImpl(std::shared_ptr<OpenSslTlsContextImpl> ctx, + const SocketSpec& peer_spec, const SocketAddress& peer_address, Mode mode) : _ctx(std::move(ctx)), + _peer_spec(peer_spec), _peer_address(peer_address), _ssl(::SSL_new(_ctx->native_context())), _mode(mode), @@ -219,6 +221,8 @@ OpenSslCryptoCodecImpl::OpenSslCryptoCodecImpl(std::shared_ptr<OpenSslTlsContext _output_bio = tmp_output_bio.release(); if (_mode == Mode::Client) { ::SSL_set_connect_state(_ssl.get()); + enable_hostname_validation_if_requested(); + set_server_name_indication_extension(); } else { ::SSL_set_accept_state(_ssl.get()); } @@ -230,6 +234,59 @@ OpenSslCryptoCodecImpl::OpenSslCryptoCodecImpl(std::shared_ptr<OpenSslTlsContext OpenSslCryptoCodecImpl::~OpenSslCryptoCodecImpl() = default; +std::unique_ptr<OpenSslCryptoCodecImpl> +OpenSslCryptoCodecImpl::make_client_codec(std::shared_ptr<OpenSslTlsContextImpl> ctx, + const SocketSpec& peer_spec, + const SocketAddress& peer_address) +{ + // Naked new due to private ctor + return std::unique_ptr<OpenSslCryptoCodecImpl>( + new OpenSslCryptoCodecImpl(std::move(ctx), peer_spec, peer_address, Mode::Client)); +} +std::unique_ptr<OpenSslCryptoCodecImpl> +OpenSslCryptoCodecImpl::make_server_codec(std::shared_ptr<OpenSslTlsContextImpl> ctx, + const SocketAddress& peer_address) +{ + // Naked new due to private ctor + return std::unique_ptr<OpenSslCryptoCodecImpl>( + new OpenSslCryptoCodecImpl(std::move(ctx), SocketSpec::invalid, peer_address, Mode::Server)); +} + +void OpenSslCryptoCodecImpl::enable_hostname_validation_if_requested() { + if (_peer_spec.valid() && !_ctx->transport_security_options().disable_hostname_validation()) { + auto* verify_param = SSL_get0_param(_ssl.get()); // Internal ptr, no refcount bump or alloc. We must not free. + LOG_ASSERT(verify_param != nullptr); + vespalib::string host = _peer_spec.host(); + if (X509_VERIFY_PARAM_set1_host(verify_param, host.c_str(), host.size()) != 1) { + throw CryptoException("X509_VERIFY_PARAM_set1_host() failed"); + } + // TODO should we set expected IP based on peer address as well? + } +} + +void OpenSslCryptoCodecImpl::set_server_name_indication_extension() { + if (_peer_spec.valid()) { + vespalib::string host = _peer_spec.host(); + // OpenSSL tries to cast const char* to void* in a macro, even on 1.1.1. GCC is not overly impressed, + // so to satiate OpenSSL's quirks we pre-cast away the constness. + auto* host_cstr_that_trusts_openssl_not_to_mess_up = const_cast<char*>(host.c_str()); + if (SSL_set_tlsext_host_name(_ssl.get(), host_cstr_that_trusts_openssl_not_to_mess_up) != 1) { + throw CryptoException("SSL_set_tlsext_host_name() failed"); + } + } +} + +std::optional<vespalib::string> OpenSslCryptoCodecImpl::client_provided_sni_extension() const { + if ((_mode != Mode::Server) || (SSL_get_servername_type(_ssl.get()) != TLSEXT_NAMETYPE_host_name)) { + return {}; + } + const char* sni_host_raw = SSL_get_servername(_ssl.get(), TLSEXT_NAMETYPE_host_name); + if (sni_host_raw == nullptr) { + return {}; + } + return vespalib::string(sni_host_raw); +} + HandshakeResult OpenSslCryptoCodecImpl::handshake(const char* from_peer, size_t from_peer_buf_size, char* to_peer, size_t to_peer_buf_size) noexcept { @@ -428,3 +485,5 @@ EncodeResult OpenSslCryptoCodecImpl::half_close(char* ciphertext, size_t ciphert // External references: // [0] http://openssl.6102.n7.nabble.com/nonblocking-implementation-question-tp1728p1732.html // [1] https://github.com/grpc/grpc/blob/master/src/core/tsi/ssl_transport_security.cc +// [2] https://wiki.openssl.org/index.php/Hostname_validation +// [3] https://wiki.openssl.org/index.php/SSL/TLS_Client diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h index 14200de449a..ec8df853c16 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h +++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h @@ -3,6 +3,7 @@ #include "openssl_typedefs.h" #include <vespa/vespalib/net/socket_address.h> +#include <vespa/vespalib/net/socket_spec.h> #include <vespa/vespalib/net/tls/transport_security_options.h> #include <vespa/vespalib/net/tls/crypto_codec.h> #include <memory> @@ -46,17 +47,23 @@ class OpenSslCryptoCodecImpl : public CryptoCodec { // The context maintains shared verification callback state, so it must be // kept alive explictly for at least as long as any codecs. std::shared_ptr<OpenSslTlsContextImpl> _ctx; + SocketSpec _peer_spec; SocketAddress _peer_address; - SslPtr _ssl; - ::BIO* _input_bio; // Owned by _ssl - ::BIO* _output_bio; // Owned by _ssl - Mode _mode; + SslPtr _ssl; + ::BIO* _input_bio; // Owned by _ssl + ::BIO* _output_bio; // Owned by _ssl + Mode _mode; std::optional<DeferredHandshakeParams> _deferred_handshake_params; - std::optional<HandshakeResult> _deferred_handshake_result; + std::optional<HandshakeResult> _deferred_handshake_result; public: - OpenSslCryptoCodecImpl(std::shared_ptr<OpenSslTlsContextImpl> ctx, const SocketAddress& peer_address, Mode mode); ~OpenSslCryptoCodecImpl() override; + static std::unique_ptr<OpenSslCryptoCodecImpl> make_client_codec(std::shared_ptr<OpenSslTlsContextImpl> ctx, + const SocketSpec& peer_spec, + const SocketAddress& peer_address); + static std::unique_ptr<OpenSslCryptoCodecImpl> make_server_codec(std::shared_ptr<OpenSslTlsContextImpl> ctx, + const SocketAddress& peer_address); + /* * From RFC 8449 (Record Size Limit Extension for TLS), section 1: * "TLS versions 1.2 [RFC5246] and earlier permit senders to @@ -89,7 +96,20 @@ public: EncodeResult half_close(char* ciphertext, size_t ciphertext_size) noexcept override; const SocketAddress& peer_address() const noexcept { return _peer_address; } + /* + * If a client has sent a SNI extension field as part of the handshake, + * returns the raw string representation of this. It only makes sense to + * call this for codecs in server mode. + */ + std::optional<vespalib::string> client_provided_sni_extension() const; private: + OpenSslCryptoCodecImpl(std::shared_ptr<OpenSslTlsContextImpl> ctx, + const SocketSpec& peer_spec, + const SocketAddress& peer_address, + Mode mode); + + void enable_hostname_validation_if_requested(); + void set_server_name_indication_extension(); HandshakeResult do_handshake_and_consume_peer_input_bytes() noexcept; DecodeResult drain_and_produce_plaintext_from_ssl(char* plaintext, size_t plaintext_size) noexcept; // Precondition: read_result < 0 diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h index 147a770bc8f..ece7d094c54 100644 --- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h +++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h @@ -28,6 +28,8 @@ public: : _null_engine(std::make_shared<NullCryptoEngine>()), _tls_engine(std::move(tls_engine)), _use_tls_when_client(use_tls_when_client) {} + bool use_tls_when_client() const override { return _use_tls_when_client; } + bool always_use_tls_when_server() const override { return false; } CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override; }; diff --git a/vespalib/src/vespa/vespalib/net/tls/peer_policies.h b/vespalib/src/vespa/vespalib/net/tls/peer_policies.h index 44cbe70fd92..c558708de8f 100644 --- a/vespalib/src/vespa/vespalib/net/tls/peer_policies.h +++ b/vespalib/src/vespa/vespalib/net/tls/peer_policies.h @@ -64,19 +64,24 @@ public: class AuthorizedPeers { // A peer will be authorized iff it matches _one or more_ policies. std::vector<PeerPolicy> _peer_policies; - bool _allow_all_if_empty = false; + bool _allow_all_if_empty; explicit AuthorizedPeers(bool allow_all_if_empty) : _peer_policies(), _allow_all_if_empty(allow_all_if_empty) {} public: - AuthorizedPeers() = default; + AuthorizedPeers() : _peer_policies(), _allow_all_if_empty(false) {} explicit AuthorizedPeers(std::vector<PeerPolicy> peer_policies_) : _peer_policies(std::move(peer_policies_)), _allow_all_if_empty(false) {} + AuthorizedPeers(const AuthorizedPeers&) = default; + AuthorizedPeers& operator=(const AuthorizedPeers&) = default; + AuthorizedPeers(AuthorizedPeers&&) noexcept = default; + AuthorizedPeers& operator=(AuthorizedPeers&&) noexcept = default; + static AuthorizedPeers allow_all_authenticated() { return AuthorizedPeers(true); } diff --git a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp index d0475f3e88d..99862e83dbf 100644 --- a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp @@ -12,18 +12,16 @@ TlsCryptoEngine::TlsCryptoEngine(net::tls::TransportSecurityOptions tls_opts, ne } std::unique_ptr<TlsCryptoSocket> -TlsCryptoEngine::create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &) +TlsCryptoEngine::create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &peer_spec) { - auto mode = net::tls::CryptoCodec::Mode::Client; - auto codec = net::tls::CryptoCodec::create_default_codec(_tls_ctx, SocketAddress::peer_address(socket.get()), mode); + auto codec = net::tls::CryptoCodec::create_default_client_codec(_tls_ctx, peer_spec, SocketAddress::peer_address(socket.get())); return std::make_unique<net::tls::CryptoCodecAdapter>(std::move(socket), std::move(codec)); } std::unique_ptr<TlsCryptoSocket> TlsCryptoEngine::create_tls_server_crypto_socket(SocketHandle socket) { - auto mode = net::tls::CryptoCodec::Mode::Server; - auto codec = net::tls::CryptoCodec::create_default_codec(_tls_ctx, SocketAddress::peer_address(socket.get()), mode); + auto codec = net::tls::CryptoCodec::create_default_server_codec(_tls_ctx, SocketAddress::peer_address(socket.get())); return std::make_unique<net::tls::CryptoCodecAdapter>(std::move(socket), std::move(codec)); } diff --git a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h index 5e760cf5585..444a817b357 100644 --- a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h +++ b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h @@ -27,6 +27,8 @@ public: net::tls::AuthorizationMode authz_mode = net::tls::AuthorizationMode::Enforce); std::unique_ptr<TlsCryptoSocket> create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; std::unique_ptr<TlsCryptoSocket> create_tls_server_crypto_socket(SocketHandle socket) override; + bool use_tls_when_client() const override { return true; } + bool always_use_tls_when_server() const override { return true; } CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override { return create_tls_client_crypto_socket(std::move(socket), spec); } diff --git a/vespalib/src/vespa/vespalib/net/tls/transport_security_options.cpp b/vespalib/src/vespa/vespalib/net/tls/transport_security_options.cpp index c9a5fa5a6e9..47b0e1e0a43 100644 --- a/vespalib/src/vespa/vespalib/net/tls/transport_security_options.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/transport_security_options.cpp @@ -11,41 +11,57 @@ TransportSecurityOptions::TransportSecurityOptions(Params params) _cert_chain_pem(std::move(params._cert_chain_pem)), _private_key_pem(std::move(params._private_key_pem)), _authorized_peers(std::move(params._authorized_peers)), - _accepted_ciphers(std::move(params._accepted_ciphers)) + _accepted_ciphers(std::move(params._accepted_ciphers)), + _disable_hostname_validation(params._disable_hostname_validation) { } TransportSecurityOptions::TransportSecurityOptions(vespalib::string ca_certs_pem, vespalib::string cert_chain_pem, - vespalib::string private_key_pem) + vespalib::string private_key_pem, + AuthorizedPeers authorized_peers, + bool disable_hostname_validation) : _ca_certs_pem(std::move(ca_certs_pem)), _cert_chain_pem(std::move(cert_chain_pem)), _private_key_pem(std::move(private_key_pem)), - _authorized_peers(AuthorizedPeers::allow_all_authenticated()) + _authorized_peers(std::move(authorized_peers)), + _disable_hostname_validation(disable_hostname_validation) { } -TransportSecurityOptions::TransportSecurityOptions(vespalib::string ca_certs_pem, - vespalib::string cert_chain_pem, - vespalib::string private_key_pem, - AuthorizedPeers authorized_peers) - : _ca_certs_pem(std::move(ca_certs_pem)), - _cert_chain_pem(std::move(cert_chain_pem)), - _private_key_pem(std::move(private_key_pem)), - _authorized_peers(std::move(authorized_peers)) -{ +TransportSecurityOptions TransportSecurityOptions::copy_without_private_key() const { + return TransportSecurityOptions(_ca_certs_pem, _cert_chain_pem, "", + _authorized_peers, _disable_hostname_validation); } void secure_memzero(void* buf, size_t size) noexcept { OPENSSL_cleanse(buf, size); } -TransportSecurityOptions::Params::Params() = default; +TransportSecurityOptions::Params::Params() + : _ca_certs_pem(), + _cert_chain_pem(), + _private_key_pem(), + _authorized_peers(), + _accepted_ciphers(), + _disable_hostname_validation(false) +{ +} TransportSecurityOptions::Params::~Params() { secure_memzero(&_private_key_pem[0], _private_key_pem.size()); } +TransportSecurityOptions::Params::Params(const Params&) = default; + +TransportSecurityOptions::Params& +TransportSecurityOptions::Params::operator=(const TransportSecurityOptions::Params&) = default; + +TransportSecurityOptions::Params::Params(Params&&) noexcept = default; + +TransportSecurityOptions::Params& +TransportSecurityOptions::Params::operator=(TransportSecurityOptions::Params&&) noexcept = default; + TransportSecurityOptions::~TransportSecurityOptions() { secure_memzero(&_private_key_pem[0], _private_key_pem.size()); } diff --git a/vespalib/src/vespa/vespalib/net/tls/transport_security_options.h b/vespalib/src/vespa/vespalib/net/tls/transport_security_options.h index 39abb1ce4de..e0a48fc6cf5 100644 --- a/vespalib/src/vespa/vespalib/net/tls/transport_security_options.h +++ b/vespalib/src/vespa/vespalib/net/tls/transport_security_options.h @@ -14,18 +14,22 @@ class TransportSecurityOptions { vespalib::string _private_key_pem; AuthorizedPeers _authorized_peers; std::vector<vespalib::string> _accepted_ciphers; + bool _disable_hostname_validation; public: - TransportSecurityOptions() = default; - struct Params { vespalib::string _ca_certs_pem; vespalib::string _cert_chain_pem; vespalib::string _private_key_pem; AuthorizedPeers _authorized_peers; std::vector<vespalib::string> _accepted_ciphers; + bool _disable_hostname_validation; Params(); ~Params(); + Params(const Params&); + Params& operator=(const Params&); + Params(Params&&) noexcept; + Params& operator=(Params&&) noexcept; Params& ca_certs_pem(vespalib::stringref pem) { _ca_certs_pem = pem; return *this; } Params& cert_chain_pem(vespalib::stringref pem) { _cert_chain_pem = pem; return *this; } @@ -35,19 +39,14 @@ public: _accepted_ciphers = std::move(ciphers); return *this; } + Params& disable_hostname_validation(bool disable) { + _disable_hostname_validation = disable; + return *this; + } }; explicit TransportSecurityOptions(Params params); - TransportSecurityOptions(vespalib::string ca_certs_pem, - vespalib::string cert_chain_pem, - vespalib::string private_key_pem); - - TransportSecurityOptions(vespalib::string ca_certs_pem, - vespalib::string cert_chain_pem, - vespalib::string private_key_pem, - AuthorizedPeers authorized_peers); - ~TransportSecurityOptions(); const vespalib::string& ca_certs_pem() const noexcept { return _ca_certs_pem; } @@ -55,10 +54,16 @@ public: const vespalib::string& private_key_pem() const noexcept { return _private_key_pem; } const AuthorizedPeers& authorized_peers() const noexcept { return _authorized_peers; } - TransportSecurityOptions copy_without_private_key() const { - return TransportSecurityOptions(_ca_certs_pem, _cert_chain_pem, "", _authorized_peers); - } + TransportSecurityOptions copy_without_private_key() const; const std::vector<vespalib::string>& accepted_ciphers() const noexcept { return _accepted_ciphers; } + bool disable_hostname_validation() const noexcept { return _disable_hostname_validation; } + +private: + TransportSecurityOptions(vespalib::string ca_certs_pem, + vespalib::string cert_chain_pem, + vespalib::string private_key_pem, + AuthorizedPeers authorized_peers, + bool disable_hostname_validation); }; // Zeroes out `size` bytes in `buf` in a way that shall never be optimized diff --git a/vespalib/src/vespa/vespalib/net/tls/transport_security_options_reading.cpp b/vespalib/src/vespa/vespalib/net/tls/transport_security_options_reading.cpp index 6b6e65bef8a..80caa15e8b2 100644 --- a/vespalib/src/vespa/vespalib/net/tls/transport_security_options_reading.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/transport_security_options_reading.cpp @@ -123,6 +123,12 @@ std::unique_ptr<TransportSecurityOptions> load_from_input(Input& input) { auto priv_key = load_file_referenced_by_field(files, "private-key"); auto authorized_peers = parse_authorized_peers(root["authorized-peers"]); auto accepted_ciphers = parse_accepted_ciphers(root["accepted-ciphers"]); + // FIXME this is temporary until we know it won't break a bunch of things! + // It's still possible to explicitly enable hostname validation by setting this to false. + bool disable_hostname_validation = true; + if (root["disable-hostname-validation"].valid()) { + disable_hostname_validation = root["disable-hostname-validation"].asBool(); + } auto options = std::make_unique<TransportSecurityOptions>( TransportSecurityOptions::Params() @@ -130,7 +136,8 @@ std::unique_ptr<TransportSecurityOptions> load_from_input(Input& input) { .cert_chain_pem(certs) .private_key_pem(priv_key) .authorized_peers(std::move(authorized_peers)) - .accepted_ciphers(std::move(accepted_ciphers))); + .accepted_ciphers(std::move(accepted_ciphers)) + .disable_hostname_validation(disable_hostname_validation)); secure_memzero(&priv_key[0], priv_key.size()); return options; } diff --git a/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp b/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp index dcd2ced8036..b31f2830976 100644 --- a/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp +++ b/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp @@ -70,7 +70,13 @@ namespace vespalib::test { SocketSpec local_spec("tcp/localhost:123"); vespalib::net::tls::TransportSecurityOptions make_tls_options_for_testing() { - return vespalib::net::tls::TransportSecurityOptions(ca_pem, cert_pem, key_pem); + auto ts_builder = vespalib::net::tls::TransportSecurityOptions::Params(). + ca_certs_pem(ca_pem). + cert_chain_pem(cert_pem). + private_key_pem(key_pem). + authorized_peers(vespalib::net::tls::AuthorizedPeers::allow_all_authenticated()). + disable_hostname_validation(true); // FIXME this is to avoid mass breakage of TLS'd networking tests. + return vespalib::net::tls::TransportSecurityOptions(std::move(ts_builder)); } } // namespace vespalib::test diff --git a/vespalib/src/vespa/vespalib/util/stash.cpp b/vespalib/src/vespa/vespalib/util/stash.cpp index 18ed0e56e27..836654c2fb2 100644 --- a/vespalib/src/vespa/vespalib/util/stash.cpp +++ b/vespalib/src/vespa/vespalib/util/stash.cpp @@ -59,14 +59,14 @@ Stash::do_alloc(size_t size) } } -Stash::Stash(size_t chunk_size) +Stash::Stash(size_t chunk_size) noexcept : _chunks(nullptr), _cleanup(nullptr), _chunk_size(std::max(size_t(4096), chunk_size)) { } -Stash::Stash(Stash &&rhs) +Stash::Stash(Stash &&rhs) noexcept : _chunks(rhs._chunks), _cleanup(rhs._cleanup), _chunk_size(rhs._chunk_size) @@ -76,7 +76,7 @@ Stash::Stash(Stash &&rhs) } Stash & -Stash::operator=(Stash &&rhs) +Stash::operator=(Stash &&rhs) noexcept { stash::run_cleanup(_cleanup); stash::free_chunks(_chunks); diff --git a/vespalib/src/vespa/vespalib/util/stash.h b/vespalib/src/vespa/vespalib/util/stash.h index aa1441aa0bb..c5e8631ca9e 100644 --- a/vespalib/src/vespa/vespalib/util/stash.h +++ b/vespalib/src/vespa/vespalib/util/stash.h @@ -14,19 +14,19 @@ struct Cleanup { explicit Cleanup(Cleanup *next_in) noexcept : next(next_in) {} virtual void cleanup() = 0; protected: - virtual ~Cleanup() {} + virtual ~Cleanup() = default; }; // used as header for memory allocated outside the stash struct DeleteMemory : public Cleanup { explicit DeleteMemory(Cleanup *next_in) noexcept : Cleanup(next_in) {} - virtual void cleanup() override { free((void*)this); } + void cleanup() override { free((void*)this); } }; // used as prefix for objects to be destructed template<typename T> struct DestructObject : public Cleanup { explicit DestructObject(Cleanup *next_in) noexcept : Cleanup(next_in) {} - virtual void cleanup() override { reinterpret_cast<T*>(this + 1)->~T(); } + void cleanup() override { reinterpret_cast<T*>(this + 1)->~T(); } }; // used as prefix for arrays to be destructed @@ -34,7 +34,7 @@ template<typename T> struct DestructArray : public Cleanup { size_t size; explicit DestructArray(Cleanup *next_in, size_t size_in) noexcept : Cleanup(next_in), size(size_in) {} - virtual void cleanup() override { + void cleanup() override { T *array = reinterpret_cast<T*>(this + 1); for (size_t i = size; i-- > 0;) { array[i].~T(); @@ -46,7 +46,7 @@ struct Chunk { Chunk *next; size_t used; Chunk(const Chunk &) = delete; - Chunk(Chunk *next_in) : next(next_in), used(sizeof(Chunk)) {} + explicit Chunk(Chunk *next_in) : next(next_in), used(sizeof(Chunk)) {} void clear() { used = sizeof(Chunk); } char *alloc(size_t size, size_t chunk_size) { size_t aligned_size = ((size + (sizeof(char *) - 1)) @@ -124,14 +124,14 @@ public: }; typedef std::unique_ptr<Stash> UP; - explicit Stash(size_t chunk_size); - Stash() : Stash(4096) {} - Stash(Stash &&rhs); + explicit Stash(size_t chunk_size) noexcept ; + Stash() noexcept : Stash(4096) {} + Stash(Stash &&rhs) noexcept; Stash(const Stash &) = delete; Stash & operator = (const Stash &) = delete; ~Stash(); - Stash &operator=(Stash &&rhs); + Stash &operator=(Stash &&rhs) noexcept; void clear(); |