diff options
256 files changed, 4018 insertions, 1877 deletions
diff --git a/config-model/pom.xml b/config-model/pom.xml index 25b733985f5..33f6657561c 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -105,6 +105,10 @@ <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> </exclusion> + <exclusion> + <groupId>org.apache.velocity</groupId> + <artifactId>velocity</artifactId> + </exclusion> </exclusions> </dependency> <dependency> diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java index e46db1d1b5f..90f061d933d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition; import com.yahoo.searchdefinition.document.BooleanIndexDefinition; +import com.yahoo.searchdefinition.document.HnswIndexParams; import com.yahoo.searchdefinition.document.RankType; import com.yahoo.searchdefinition.document.Stemming; @@ -9,8 +10,11 @@ import java.io.Serializable; import java.util.Collections; import java.util.Iterator; import java.util.LinkedHashSet; +import java.util.Objects; +import java.util.Optional; import java.util.Set; + /** * An index definition in a search definition. * Two indices are equal if they have the same name and the same settings, except @@ -57,6 +61,8 @@ public class Index implements Cloneable, Serializable { /** The boolean index definition, if set */ private BooleanIndexDefinition boolIndex; + private Optional<HnswIndexParams> hnswIndexParams; + /** Whether the posting lists of this index field should have interleaved features (num occs, field length) in document id stream. */ private boolean interleavedFeatures = false; @@ -115,20 +121,25 @@ public class Index implements Cloneable, Serializable { } @Override - public int hashCode() { - return name.hashCode() + ( prefix ? 17 : 0 ); + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Index index = (Index) o; + return prefix == index.prefix && + normalized == index.normalized && + interleavedFeatures == index.interleavedFeatures && + Objects.equals(name, index.name) && + rankType == index.rankType && + Objects.equals(aliases, index.aliases) && + stemming == index.stemming && + type == index.type && + Objects.equals(boolIndex, index.boolIndex) && + Objects.equals(hnswIndexParams, index.hnswIndexParams); } @Override - public boolean equals(Object object) { - if ( ! (object instanceof Index)) return false; - - Index other=(Index)object; - return - this.name.equals(other.name) && - this.prefix==other.prefix && - this.stemming==other.stemming && - this.normalized==other.normalized; + public int hashCode() { + return Objects.hash(name, rankType, prefix, aliases, stemming, normalized, type, boolIndex, hnswIndexParams, interleavedFeatures); } public String toString() { @@ -176,6 +187,14 @@ public class Index implements Cloneable, Serializable { boolIndex = def; } + public Optional<HnswIndexParams> getHnswIndexParams() { + return hnswIndexParams; + } + + public void setHnswIndexParams(HnswIndexParams params) { + hnswIndexParams = Optional.of(params); + } + public void setInterleavedFeatures(boolean value) { interleavedFeatures = value; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index 76dff404568..8de8b239c93 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -240,6 +240,14 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce aaB.tensortype(attribute.tensorType().get().toString()); } aaB.imported(imported); + if (attribute.hnswIndexParams().isPresent()) { + var ib = new AttributesConfig.Attribute.Index.Builder(); + var params = attribute.hnswIndexParams().get(); + ib.hnsw.enabled(true); + ib.hnsw.maxlinkspernode(params.maxLinksPerNode()); + ib.hnsw.neighborstoexploreatinsert(params.neighborsToExploreAtInsert()); + aaB.index(ib); + } return aaB; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java index d8773063053..39a67a69575 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java @@ -5,6 +5,7 @@ import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.Field; import com.yahoo.document.StructuredDataType; +import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.BooleanIndexDefinition; @@ -20,7 +21,7 @@ import java.util.List; import java.util.Map; /** - * Deriver of indexschema config containing information of all index fields with name and data type. + * Deriver of indexschema config containing information of all text index fields with name and data type. * * @author geirst */ @@ -44,9 +45,14 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { super.derive(search); } + private boolean isTensorField(ImmutableSDField field) { + return field.getDataType() instanceof TensorDataType; + } + private void deriveIndexFields(ImmutableSDField field, Search search) { - if (!field.doesIndexing() && - !field.isIndexStructureField()) + // Note: Indexes for tensor fields are NOT part of the index schema for text fields. + if ((!field.doesIndexing() && !field.isIndexStructureField()) || + isTensorField(field)) { return; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java index fbcaf2a3a80..9ed5e4ca2de 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java @@ -66,6 +66,8 @@ public final class Attribute implements Cloneable, Serializable { /** This is set if the type of this is REFERENCE */ private final Optional<StructuredDataType> referenceDocumentType; + private Optional<HnswIndexParams> hnswIndexParams; + private boolean isPosition = false; private final Sorting sorting = new Sorting(); @@ -150,6 +152,7 @@ public final class Attribute implements Cloneable, Serializable { setCollectionType(collectionType); this.tensorType = tensorType; this.referenceDocumentType = referenceDocumentType; + this.hnswIndexParams = Optional.empty(); } public Attribute convertToArray() { @@ -194,6 +197,7 @@ public final class Attribute implements Cloneable, Serializable { public double densePostingListThreshold() { return densePostingListThreshold; } public Optional<TensorType> tensorType() { return tensorType; } public Optional<StructuredDataType> referenceDocumentType() { return referenceDocumentType; } + public Optional<HnswIndexParams> hnswIndexParams() { return hnswIndexParams; } public Sorting getSorting() { return sorting; } @@ -217,6 +221,7 @@ public final class Attribute implements Cloneable, Serializable { public void setUpperBound(long upperBound) { this.upperBound = upperBound; } public void setDensePostingListThreshold(double threshold) { this.densePostingListThreshold = threshold; } public void setTensorType(TensorType tensorType) { this.tensorType = Optional.of(tensorType); } + public void setHnswIndexParams(HnswIndexParams params) { this.hnswIndexParams = Optional.of(params); } public String getName() { return name; } public Type getType() { return type; } @@ -335,7 +340,7 @@ public final class Attribute implements Cloneable, Serializable { public int hashCode() { return Objects.hash( name, type, collectionType, sorting, isPrefetch(), fastAccess, removeIfZero, createIfNonExistent, - isPosition, huge, enableBitVectors, enableOnlyBitVector, tensorType, referenceDocumentType); + isPosition, huge, enableBitVectors, enableOnlyBitVector, tensorType, referenceDocumentType, hnswIndexParams); } @Override @@ -362,6 +367,7 @@ public final class Attribute implements Cloneable, Serializable { if ( ! this.sorting.equals(other.sorting)) return false; if (!this.tensorType.equals(other.tensorType)) return false; if (!this.referenceDocumentType.equals(other.referenceDocumentType)) return false; + if (!this.hnswIndexParams.equals(other.hnswIndexParams)) return false; return true; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java new file mode 100644 index 00000000000..70d0df8be7f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java @@ -0,0 +1,60 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; + +import java.util.OptionalInt; + +/** + * Configuration parameters for a hnsw index used together with a 1-dimensional indexed tensor for approximate nearest neighbor search. + * + * @author geirst + */ +public class HnswIndexParams { + + public static final int DEFAULT_MAX_LINKS_PER_NODE = 16; + public static final int DEFAULT_NEIGHBORS_TO_EXPLORE_AT_INSERT = 200; + + private final OptionalInt maxLinksPerNode; + private final OptionalInt neighborsToExploreAtInsert; + + public static class Builder { + private OptionalInt maxLinksPerNode = OptionalInt.empty(); + private OptionalInt neighborsToExploreAtInsert = OptionalInt.empty(); + + public void setMaxLinksPerNode(int value) { + maxLinksPerNode = OptionalInt.of(value); + } + public void setNeighborsToExploreAtInsert(int value) { + neighborsToExploreAtInsert = OptionalInt.of(value); + } + public HnswIndexParams build() { + return new HnswIndexParams(maxLinksPerNode, neighborsToExploreAtInsert); + } + } + + public HnswIndexParams() { + this.maxLinksPerNode = OptionalInt.empty(); + this.neighborsToExploreAtInsert = OptionalInt.empty(); + } + + public HnswIndexParams(OptionalInt maxLinksPerNode, OptionalInt neighborsToExploreAtInsert) { + this.maxLinksPerNode = maxLinksPerNode; + this.neighborsToExploreAtInsert = neighborsToExploreAtInsert; + } + + /** + * Creates a new instance where values from the given parameter instance are used where they are present, + * otherwise we use values from this. + */ + public HnswIndexParams overrideFrom(HnswIndexParams rhs) { + return new HnswIndexParams(rhs.maxLinksPerNode.isPresent() ? rhs.maxLinksPerNode : maxLinksPerNode, + rhs.neighborsToExploreAtInsert.isPresent() ? rhs.neighborsToExploreAtInsert : neighborsToExploreAtInsert); + } + + public int maxLinksPerNode() { + return maxLinksPerNode.orElse(DEFAULT_MAX_LINKS_PER_NODE); + } + + public int neighborsToExploreAtInsert() { + return neighborsToExploreAtInsert.orElse(DEFAULT_NEIGHBORS_TO_EXPLORE_AT_INSERT); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java index 39f543c7db3..7f9da28b9ca 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java @@ -4,6 +4,7 @@ package com.yahoo.searchdefinition.fieldoperation; import com.yahoo.searchdefinition.Index; import com.yahoo.searchdefinition.Index.Type; import com.yahoo.searchdefinition.document.BooleanIndexDefinition; +import com.yahoo.searchdefinition.document.HnswIndexParams; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.document.Stemming; @@ -31,6 +32,8 @@ public class IndexOperation implements FieldOperation { private OptionalDouble densePostingListThreshold = OptionalDouble.empty(); private Optional<Boolean> enableBm25 = Optional.empty(); + private Optional<HnswIndexParams.Builder> hnswIndexParams = Optional.empty(); + public String getIndexName() { return indexName; } @@ -91,6 +94,9 @@ public class IndexOperation implements FieldOperation { if (enableBm25.isPresent()) { index.setInterleavedFeatures(enableBm25.get()); } + if (hnswIndexParams.isPresent()) { + index.setHnswIndexParams(hnswIndexParams.get().build()); + } } public Type getType() { @@ -116,8 +122,13 @@ public class IndexOperation implements FieldOperation { public void setDensePostingListThreshold(double densePostingListThreshold) { this.densePostingListThreshold = OptionalDouble.of(densePostingListThreshold); } + public void setEnableBm25(boolean value) { enableBm25 = Optional.of(value); } + public void setHnswIndexParams(HnswIndexParams.Builder params) { + this.hnswIndexParams = Optional.of(params); + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java index 8e54d7c00d6..2790f2ddf6e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java @@ -6,7 +6,8 @@ import com.yahoo.document.CollectionDataType; import com.yahoo.document.TensorDataType; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; -import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.HnswIndexParams; +import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -23,34 +24,70 @@ public class TensorFieldProcessor extends Processor { @Override public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - - for (SDField field : search.allConcreteFields()) { + for (var field : search.allConcreteFields()) { if ( field.getDataType() instanceof TensorDataType ) { - validateIndexingScripsForTensorField(field); - validateAttributeSettingForTensorField(field); + if (validate) { + validateIndexingScripsForTensorField(field); + validateAttributeSettingForTensorField(field); + } + processIndexSettingsForTensorField(field, validate); } else if (field.getDataType() instanceof CollectionDataType){ - validateDataTypeForCollectionField(field); + if (validate) { + validateDataTypeForCollectionField(field); + } } } } private void validateIndexingScripsForTensorField(SDField field) { - if (field.doesIndexing()) { - fail(search, field, "A field of type 'tensor' cannot be specified as an 'index' field."); + if (field.doesIndexing() && !isTensorTypeThatSupportsHnswIndex(field)) { + fail(search, field, "A tensor of type '" + tensorTypeToString(field) + "' does not support having an 'index'. " + + "Currently, only tensors with 1 indexed dimension supports that."); + } + } + + private boolean isTensorTypeThatSupportsHnswIndex(ImmutableSDField field) { + var type = ((TensorDataType)field.getDataType()).getTensorType(); + // Tensors with 1 indexed dimension supports a hnsw index (used for approximate nearest neighbor search). + if ((type.dimensions().size() == 1) && + type.dimensions().get(0).isIndexed()) { + return true; } + return false; + } + + private String tensorTypeToString(ImmutableSDField field) { + return ((TensorDataType)field.getDataType()).getTensorType().toString(); } private void validateAttributeSettingForTensorField(SDField field) { if (field.doesAttributing()) { - Attribute attribute = field.getAttributes().get(field.getName()); + var attribute = field.getAttributes().get(field.getName()); if (attribute != null && attribute.isFastSearch()) { fail(search, field, "An attribute of type 'tensor' cannot be 'fast-search'."); } } } + private void processIndexSettingsForTensorField(SDField field, boolean validate) { + if (!field.doesIndexing()) { + return; + } + if (isTensorTypeThatSupportsHnswIndex(field)) { + if (validate && !field.doesAttributing()) { + fail(search, field, "A tensor that has an index must also be an attribute."); + } + var index = field.getIndex(field.getName()); + // TODO: Calculate default params based on tensor dimension size + var params = new HnswIndexParams(); + if (index != null && index.getHnswIndexParams().isPresent()) { + params = params.overrideFrom(index.getHnswIndexParams().get()); + } + field.getAttribute().setHnswIndexParams(params); + } + } + private void validateDataTypeForCollectionField(SDField field) { if (((CollectionDataType)field.getDataType()).getNestedType() instanceof TensorDataType) fail(search, field, "A field with collection type of tensor is not supported. Use simple type 'tensor' instead."); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java index 5714d41ef67..38037c8a522 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.admin; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.handler.ThreadpoolConfig; +import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Handler; @@ -27,6 +28,12 @@ public class LogserverContainerCluster extends ContainerCluster<LogserverContain builder.maxthreads(10); } + @Override + public void getConfig(QrStartConfig.Builder builder) { + super.getConfig(builder); + builder.jvm.heapsize(384); + } + protected boolean messageBusEnabled() { return false; } private void addLogHandler() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 071666b5bc7..f81757ac568 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -20,6 +20,7 @@ import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; import ai.vespa.metricsproxy.rpc.RpcServer; import ai.vespa.metricsproxy.service.ConfigSentinelClient; import ai.vespa.metricsproxy.service.SystemPollerProvider; +import ai.vespa.metricsproxy.telegraf.TelegrafConfig; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; @@ -67,6 +68,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC ApplicationDimensionsConfig.Producer, ConsumersConfig.Producer, MonitoringConfig.Producer, + TelegrafConfig.Producer, ThreadpoolConfig.Producer, MetricsNodesConfig.Producer { @@ -161,6 +163,25 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC } @Override + public void getConfig(TelegrafConfig.Builder builder) { + var userConsumers = getUserMetricsConsumers(); + for (var consumer : userConsumers.values()) { + for (var cloudWatch : consumer.cloudWatches()) { + var cloudWatchBuilder = new TelegrafConfig.CloudWatch.Builder(); + cloudWatchBuilder + .region(cloudWatch.region()) + .namespace(cloudWatch.namespace()) + .consumer(cloudWatch.consumer()); + cloudWatch.hostedAuth().ifPresent(hostedAuth -> cloudWatchBuilder + .accessKeyName(hostedAuth.accessKeyName) + .secretKeyName(hostedAuth.secretKeyName)); + cloudWatch.profile().ifPresent(cloudWatchBuilder::profile); + builder.cloudWatch(cloudWatchBuilder); + } + } + } + + @Override public void getConfig(ThreadpoolConfig.Builder builder) { builder.maxthreads(10); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java new file mode 100644 index 00000000000..fd290409ea5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java @@ -0,0 +1,49 @@ +package com.yahoo.vespa.model.admin.monitoring; + +import java.util.Optional; + +/** + * Helper object for CloudWatch configuration. + * + * @author gjoranv + */ +public class CloudWatch { + private final String region; + private final String namespace; + private final MetricsConsumer consumer; + + private HostedAuth hostedAuth; + private String profile; + + public CloudWatch(String region, String namespace, MetricsConsumer consumer) { + this.region = region; + this.namespace = namespace; + this.consumer = consumer; + } + + public String region() { return region; } + public String namespace() { return namespace; } + public String consumer() { return consumer.getId(); } + + public Optional<HostedAuth> hostedAuth() {return Optional.ofNullable(hostedAuth); } + public Optional<String> profile() { return Optional.ofNullable(profile); } + + public void setHostedAuth(HostedAuth hostedAuth) { + this.hostedAuth = hostedAuth; + } + + public void setProfile(String profile) { + this.profile = profile; + } + + public static class HostedAuth { + public final String accessKeyName; + public final String secretKeyName; + + public HostedAuth(String accessKeyName, String secretKeyName) { + this.accessKeyName = accessKeyName; + this.secretKeyName = secretKeyName; + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java index 529ed6ecf67..9c752f3aa0d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java @@ -2,9 +2,13 @@ package com.yahoo.vespa.model.admin.monitoring; import javax.annotation.concurrent.Immutable; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Objects; +import static java.util.Collections.unmodifiableList; + /** * Represents an arbitrary metric consumer * @@ -16,6 +20,8 @@ public class MetricsConsumer { private final String id; private final MetricSet metricSet; + private final List<CloudWatch> cloudWatches = new ArrayList<>(); + /** * @param id The consumer * @param metricSet The metrics for this consumer @@ -38,4 +44,12 @@ public class MetricsConsumer { return metricSet.getMetrics(); } + public void addCloudWatch(CloudWatch cloudWatch) { + cloudWatches.add(cloudWatch); + } + + public List<CloudWatch> cloudWatches() { + return unmodifiableList(cloudWatches); + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 617e83bcc8e..e308be00faf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -615,18 +615,24 @@ public class VespaMetricSet { metrics.add(new Metric("vds.distributor.puts.sum.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("vds.distributor.puts.sum.ok.rate")); metrics.add(new Metric("vds.distributor.puts.sum.failures.total.rate")); + metrics.add(new Metric("vds.distributor.puts.sum.failures.notfound.rate")); + metrics.add(new Metric("vds.distributor.puts.sum.failures.test_and_set_failed.rate")); metrics.add(new Metric("vds.distributor.removes.sum.latency.max")); metrics.add(new Metric("vds.distributor.removes.sum.latency.sum")); metrics.add(new Metric("vds.distributor.removes.sum.latency.count")); metrics.add(new Metric("vds.distributor.removes.sum.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("vds.distributor.removes.sum.ok.rate")); metrics.add(new Metric("vds.distributor.removes.sum.failures.total.rate")); + metrics.add(new Metric("vds.distributor.removes.sum.failures.notfound.rate")); + metrics.add(new Metric("vds.distributor.removes.sum.failures.test_and_set_failed.rate")); metrics.add(new Metric("vds.distributor.updates.sum.latency.max")); metrics.add(new Metric("vds.distributor.updates.sum.latency.sum")); metrics.add(new Metric("vds.distributor.updates.sum.latency.count")); metrics.add(new Metric("vds.distributor.updates.sum.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("vds.distributor.updates.sum.ok.rate")); metrics.add(new Metric("vds.distributor.updates.sum.failures.total.rate")); + metrics.add(new Metric("vds.distributor.updates.sum.failures.notfound.rate")); + metrics.add(new Metric("vds.distributor.updates.sum.failures.test_and_set_failed.rate")); metrics.add(new Metric("vds.distributor.updates.sum.diverging_timestamp_updates.rate")); metrics.add(new Metric("vds.distributor.removelocations.sum.ok.rate")); metrics.add(new Metric("vds.distributor.removelocations.sum.failures.total.rate")); @@ -636,6 +642,7 @@ public class VespaMetricSet { metrics.add(new Metric("vds.distributor.gets.sum.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("vds.distributor.gets.sum.ok.rate")); metrics.add(new Metric("vds.distributor.gets.sum.failures.total.rate")); + metrics.add(new Metric("vds.distributor.gets.sum.failures.notfound.rate")); metrics.add(new Metric("vds.distributor.visitor.sum.latency.max")); metrics.add(new Metric("vds.distributor.visitor.sum.latency.sum")); metrics.add(new Metric("vds.distributor.visitor.sum.latency.count")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java new file mode 100644 index 00000000000..4b9d5542aa9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java @@ -0,0 +1,36 @@ +package com.yahoo.vespa.model.admin.monitoring.builder.xml; + +import com.yahoo.vespa.model.admin.monitoring.CloudWatch; +import com.yahoo.vespa.model.admin.monitoring.CloudWatch.HostedAuth; +import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; +import org.w3c.dom.Element; + +import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalChildValue; + +/** + * @author gjoranv + */ +public class CloudWatchBuilder { + + private static final String REGION_ATTRIBUTE = "region"; + private static final String NAMESPACE_ATTRIBUTE = "namespace"; + private static final String ACCESS_KEY_ELEMENT = "access-key-name"; + private static final String SECRET_KEY_ELEMENT = "secret-key-name"; + private static final String PROFILE_ELEMENT = "profile"; + + public static CloudWatch buildCloudWatch(Element cloudwatchElement, MetricsConsumer consumer) { + CloudWatch cloudWatch = new CloudWatch(cloudwatchElement.getAttribute(REGION_ATTRIBUTE), + cloudwatchElement.getAttribute(NAMESPACE_ATTRIBUTE), + consumer); + + getOptionalChildValue(cloudwatchElement, PROFILE_ELEMENT).ifPresent(cloudWatch::setProfile); + + getOptionalChildValue(cloudwatchElement, ACCESS_KEY_ELEMENT) + .ifPresent(accessKey -> cloudWatch.setHostedAuth( + new HostedAuth(accessKey, + getOptionalChildValue(cloudwatchElement, SECRET_KEY_ELEMENT) + .orElseThrow(() -> new IllegalArgumentException("Access key given without a secret key."))))); + return cloudWatch; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java index f029dad01a9..b686288868f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java @@ -42,7 +42,11 @@ public class MetricsBuilder { throwIfIllegalConsumerId(metrics, consumerId); MetricSet metricSet = buildMetricSet(consumerId, consumerElement); - metrics.addConsumer(new MetricsConsumer(consumerId, metricSet)); + var consumer = new MetricsConsumer(consumerId, metricSet); + for (Element cloudwatchElement : XML.getChildren(consumerElement, "cloudwatch")) { + consumer.addCloudWatch(CloudWatchBuilder.buildCloudWatch(cloudwatchElement, consumer)); + } + metrics.addConsumer(consumer); } return metrics; } @@ -58,7 +62,7 @@ public class MetricsBuilder { private MetricSet buildMetricSet(String consumerId, Element consumerElement) { List<Metric> metrics = XML.getChildren(consumerElement, "metric").stream() - .map(metricElement -> metricFromElement(metricElement)) + .map(MetricsBuilder::metricFromElement) .collect(Collectors.toCollection(LinkedList::new)); List<MetricSet> metricSets = XML.getChildren(consumerElement, "metric-set").stream() diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java index f00ad0f0dbb..4b8bbd4ff08 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java @@ -12,7 +12,7 @@ public class EndpointCertificateSecretsValidator extends Validator { @Override public void validate(VespaModel model, DeployState deployState) { if (deployState.endpointCertificateSecrets().isPresent() && deployState.endpointCertificateSecrets().get() == EndpointCertificateSecrets.MISSING) { - throw new CertificateNotReadyException("TLS enabled, but could not retrieve certificate yet"); + throw new CertificateNotReadyException("TLS enabled, but could not yet retrieve certificate for application " + deployState.getProperties().applicationId().serializedForm()); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 58679c63565..3632cb08da5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -215,6 +215,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat super.getConfig(builder); builder.jvm.verbosegc(true) .availableProcessors(0) + .compressedClassSpaceSize(0) //TODO Reduce, next step is 512m .minHeapsize(1536) .heapsize(1536); if (getMemoryPercentage().isPresent()) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 6fa446bf365..3b132ab2342 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -484,6 +484,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> builder.jvm .verbosegc(false) .availableProcessors(2) + .compressedClassSpaceSize(32) .minHeapsize(32) .heapsize(512) .heapSizeAsPercentageOfPhysicalMemory(0) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java index c671749cff0..a466dabe984 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java @@ -4,7 +4,12 @@ package com.yahoo.vespa.model.container.component; import com.yahoo.component.ComponentId; import com.yahoo.config.model.producer.AbstractConfigProducer; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + /** * A group of config producers that have a component id. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java index b37caf22216..b492941fd13 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java @@ -139,13 +139,9 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> int legalPortInHostedVespa = Container.BASEPORT; if (isHosted && port != legalPortInHostedVespa && ! spec.booleanAttribute("required", false)) { - // TODO: After January 2020: - // - Set required='true' for the http server on port 4443 in the tester services.xml in InternalStepRunner - // - Enable 2 currently ignored tests in this module - // - throw IllegalArgumentException here instead of warning - logger.log(Level.WARNING, "Illegal port " + port + " in http server '" + - spec.stringAttribute("id") + "'" + - ": Port must be set to " + legalPortInHostedVespa); + throw new IllegalArgumentException("Illegal port " + port + " in http server '" + + spec.stringAttribute("id") + "'" + + ": Port must be set to " + legalPortInHostedVespa); } return port; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index e19d81e7fb2..fa5fa4bd227 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -8,6 +8,7 @@ import com.yahoo.search.pagetemplates.PageTemplatesConfig; import com.yahoo.search.query.profile.config.QueryProfilesConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; +import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.ContainerSubsystem; import com.yahoo.vespa.model.container.search.searchchain.LocalProvider; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; @@ -58,7 +59,8 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> private void initializeDispatchers(Collection<AbstractSearchCluster> searchClusters) { for (AbstractSearchCluster searchCluster : searchClusters) { if ( ! ( searchCluster instanceof IndexedSearchCluster)) continue; - owningCluster.addComponent(new DispatcherComponent((IndexedSearchCluster)searchCluster)); + var dispatcher = new DispatcherComponent((IndexedSearchCluster)searchCluster); + owningCluster.addComponent(dispatcher); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java index 704188e80e8..284aa3b46c0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java @@ -1,6 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.search; +import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.model.container.component.Component; @@ -13,7 +14,7 @@ import com.yahoo.vespa.model.search.IndexedSearchCluster; * * @author bratseth */ -public class DispatcherComponent extends Component<DispatcherComponent, ComponentModel> +public class DispatcherComponent extends Component<AbstractConfigProducer<?>, ComponentModel> implements DispatchConfig.Producer { private final IndexedSearchCluster indexedSearchCluster; @@ -21,14 +22,17 @@ public class DispatcherComponent extends Component<DispatcherComponent, Componen public DispatcherComponent(IndexedSearchCluster indexedSearchCluster) { super(toComponentModel(indexedSearchCluster)); this.indexedSearchCluster = indexedSearchCluster; + String clusterName = indexedSearchCluster.getClusterName(); + var rpcResoucePool = new RpcResourcePoolComponent(clusterName); + inject(rpcResoucePool); + addComponent(rpcResoucePool); } private static ComponentModel toComponentModel(IndexedSearchCluster indexedSearchCluster) { String dispatcherComponentId = "dispatcher." + indexedSearchCluster.getClusterName(); // used by ClusterSearcher return new ComponentModel(dispatcherComponentId, - "com.yahoo.search.dispatch.Dispatcher", - BundleMapper.searchAndDocprocBundle, - null); + com.yahoo.search.dispatch.Dispatcher.class.getName(), + BundleMapper.searchAndDocprocBundle); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java new file mode 100644 index 00000000000..2689c2ce71b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java @@ -0,0 +1,18 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search; + +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.xml.BundleMapper; + +public class RpcResourcePoolComponent extends Component<RpcResourcePoolComponent, ComponentModel> { + + public RpcResourcePoolComponent(String clusterName) { + super(toComponentModel(clusterName)); + } + + private static ComponentModel toComponentModel(String clusterName) { + String componentId = "rpcresourcepool." + clusterName; + return new ComponentModel(componentId, com.yahoo.search.dispatch.rpc.RpcResourcePool.class.getName(), BundleMapper.searchAndDocprocBundle); + } +} diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index eb7d7763cd1..0cfd13f2a9c 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -332,6 +332,9 @@ TOKEN : | < UPPERBOUND: "upper-bound" > | < DENSEPOSTINGLISTTHRESHOLD: "dense-posting-list-threshold" > | < ENABLE_BM25: "enable-bm25" > +| < HNSW: "hnsw" > +| < MAXLINKSPERNODE: "max-links-per-node" > +| < NEIGHBORSTOEXPLOREATINSERT: "neighbors-to-explore-at-insert" > | < SUMMARYFEATURES_SL: "summary-features" (" ")* ":" (~["}","\n"])* ("\n")? > | < SUMMARYFEATURES_ML: "summary-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" > | < RANKFEATURES_SL: "rank-features" (" ")* ":" (~["}","\n"])* ("\n")? > @@ -1811,10 +1814,33 @@ Object indexBody(IndexOperation index) : | <UPPERBOUND> <COLON> num = consumeLong() { index.setUpperBound(num); } | <DENSEPOSTINGLISTTHRESHOLD> <COLON> threshold = consumeFloat() { index.setDensePostingListThreshold(threshold); } | <ENABLE_BM25> { index.setEnableBm25(true); } + | hnswIndex(index) { } ) { return null; } } +void hnswIndex(IndexOperation index) : +{ + HnswIndexParams.Builder params = new HnswIndexParams.Builder(); +} +{ + ( LOOKAHEAD(<HNSW> lbrace()) + <HNSW> ( (lbrace() (hnswIndexBody(params) (<NL>)*)* <RBRACE>) ) | + <HNSW> ) + { + index.setHnswIndexParams(params); + } +} + +void hnswIndexBody(HnswIndexParams.Builder params) : +{ + int num; +} +{ + ( <MAXLINKSPERNODE> <COLON> num = integer() { params.setMaxLinksPerNode(num); } + | <NEIGHBORSTOEXPLOREATINSERT> <COLON> num = integer() { params.setNeighborsToExploreAtInsert(num); } ) +} + /** * Consumes a constant block of a search element. * diff --git a/config-model/src/main/perl/vespa-deploy b/config-model/src/main/perl/vespa-deploy index 59a84f5b0c0..a128e4a8d4c 100755 --- a/config-model/src/main/perl/vespa-deploy +++ b/config-model/src/main/perl/vespa-deploy @@ -154,7 +154,7 @@ my $command = shift; $command ||= "help"; # The '--insecure' parameter is sadly required as it is not possible to disable or alter hostname verification with curl -my $curl_command = $VESPA_HOME . '/libexec/vespa/vespa-curl-wrapper --insecure -A vespa-deploy --silent --show-error --connect-timeout 30 --max-time 1200'; +my $curl_command = $VESPA_HOME . '/libexec/vespa/vespa-curl-wrapper -A vespa-deploy --silent --show-error --connect-timeout 30 --max-time 1200'; my $CURL_PUT = $curl_command . ' --write-out \%{http_code} --request PUT'; my $CURL_GET = $curl_command . ' --request GET'; diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc index 7a3e2916f94..055f57dd7c0 100644 --- a/config-model/src/main/resources/schema/admin.rnc +++ b/config-model/src/main/resources/schema/admin.rnc @@ -82,10 +82,25 @@ Metrics = element metrics { element metric { attribute id { xsd:Name } & attribute display-name { xsd:Name }? - }* + }* & + Cloudwatch? }+ } +Cloudwatch = element cloudwatch { + attribute region { xsd:Name } & + attribute namespace { xsd:Name } & + ( + ( + element access-key-name { xsd:Name } & + element secret-key-name { xsd:Name } + ) + | + element profile { xsd:Name } + )? + +} + ClusterControllers = element cluster-controllers { attribute standalone-zookeeper { xsd:string }? & element cluster-controller { diff --git a/config-model/src/test/derived/advanced/attributes.cfg b/config-model/src/test/derived/advanced/attributes.cfg index 97f480745cf..cf8644ebe83 100644 --- a/config-model/src/test/derived/advanced/attributes.cfg +++ b/config-model/src/test/derived/advanced/attributes.cfg @@ -19,3 +19,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg index 56fd15f9f5d..29d5dd92043 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "elem_array.weight" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -40,3 +43,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/attributeprefetch/attributes.cfg b/config-model/src/test/derived/attributeprefetch/attributes.cfg index 022bdbd31a4..773f796ed59 100644 --- a/config-model/src/test/derived/attributeprefetch/attributes.cfg +++ b/config-model/src/test/derived/attributeprefetch/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multibyte" attribute[].datatype INT8 attribute[].collectiontype ARRAY @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wsbyte" attribute[].datatype INT8 attribute[].collectiontype WEIGHTEDSET @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "singleint" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multiint" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -103,6 +115,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wsint" attribute[].datatype INT32 attribute[].collectiontype WEIGHTEDSET @@ -124,6 +139,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "singlelong" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -145,6 +163,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multilong" attribute[].datatype INT64 attribute[].collectiontype ARRAY @@ -166,6 +187,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wslong" attribute[].datatype INT64 attribute[].collectiontype WEIGHTEDSET @@ -187,6 +211,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "singlefloat" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -208,6 +235,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multifloat" attribute[].datatype FLOAT attribute[].collectiontype ARRAY @@ -229,6 +259,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wsfloat" attribute[].datatype FLOAT attribute[].collectiontype WEIGHTEDSET @@ -250,6 +283,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "singledouble" attribute[].datatype DOUBLE attribute[].collectiontype SINGLE @@ -271,6 +307,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multidouble" attribute[].datatype DOUBLE attribute[].collectiontype ARRAY @@ -292,6 +331,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wsdouble" attribute[].datatype DOUBLE attribute[].collectiontype WEIGHTEDSET @@ -313,6 +355,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "singlestring" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -334,6 +379,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multistring" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -355,6 +403,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wsstring" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -376,3 +427,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/attributes/attributes.cfg b/config-model/src/test/derived/attributes/attributes.cfg index 7a21001a9ed..e3faf7662f4 100644 --- a/config-model/src/test/derived/attributes/attributes.cfg +++ b/config-model/src/test/derived/attributes/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a2" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a3" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a5" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a6" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -103,6 +115,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b1" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -124,6 +139,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b2" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -145,6 +163,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b3" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -166,6 +187,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b4" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -187,6 +211,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b5" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -208,6 +235,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b6" attribute[].datatype INT64 attribute[].collectiontype ARRAY @@ -229,6 +259,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b7" attribute[].datatype DOUBLE attribute[].collectiontype WEIGHTEDSET @@ -250,6 +283,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a9" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -271,6 +307,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a10" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -292,6 +331,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a11" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -313,6 +355,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a12" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -334,6 +379,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a7_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -355,6 +403,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a8_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -376,3 +427,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/complex/attributes.cfg b/config-model/src/test/derived/complex/attributes.cfg index dda746da5b4..b4971487bd1 100644 --- a/config-model/src/test/derived/complex/attributes.cfg +++ b/config-model/src/test/derived/complex/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "fleeting" attribute[].datatype FLOAT attribute[].collectiontype ARRAY @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "fleeting2" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "foundat" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "collapseby" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -103,6 +115,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "ts" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -124,6 +139,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "combineda" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -145,6 +163,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "year_arr" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -166,6 +187,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "year_sub" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -187,3 +211,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/hnsw_index/attributes.cfg b/config-model/src/test/derived/hnsw_index/attributes.cfg new file mode 100644 index 00000000000..27c9f1e0d13 --- /dev/null +++ b/config-model/src/test/derived/hnsw_index/attributes.cfg @@ -0,0 +1,24 @@ +attribute[].name "t1" +attribute[].datatype TENSOR +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "tensor(x[128])" +attribute[].imported false +attribute[].index.hnsw.enabled true +attribute[].index.hnsw.maxlinkspernode 32 +attribute[].index.hnsw.neighborstoexploreatinsert 300 diff --git a/config-model/src/test/derived/hnsw_index/ilscripts.cfg b/config-model/src/test/derived/hnsw_index/ilscripts.cfg new file mode 100644 index 00000000000..e9fc265ca67 --- /dev/null +++ b/config-model/src/test/derived/hnsw_index/ilscripts.cfg @@ -0,0 +1,5 @@ +maxtermoccurrences 100 +fieldmatchmaxlength 1000000 +ilscript[].doctype "test" +ilscript[].docfield[] "t1" +ilscript[].content[] "clear_state | guard { input t1 | attribute t1 | index t1; }" diff --git a/config-model/src/test/derived/hnsw_index/test.sd b/config-model/src/test/derived/hnsw_index/test.sd new file mode 100644 index 00000000000..03ede04208b --- /dev/null +++ b/config-model/src/test/derived/hnsw_index/test.sd @@ -0,0 +1,13 @@ +search test { + document test { + field t1 type tensor(x[128]) { + indexing: attribute | index + index { + hnsw { + max-links-per-node: 32 + neighbors-to-explore-at-insert: 300 + } + } + } + } +} diff --git a/config-model/src/test/derived/imported_position_field/attributes.cfg b/config-model/src/test/derived/imported_position_field/attributes.cfg index db2280a7846..5427e856df0 100644 --- a/config-model/src/test/derived/imported_position_field/attributes.cfg +++ b/config-model/src/test/derived/imported_position_field/attributes.cfg @@ -1,42 +1,48 @@ -attribute[0].name "parent_ref" -attribute[0].datatype REFERENCE -attribute[0].collectiontype SINGLE -attribute[0].removeifzero false -attribute[0].createifnonexistent false -attribute[0].fastsearch false -attribute[0].huge false -attribute[0].ismutable false -attribute[0].sortascending true -attribute[0].sortfunction UCA -attribute[0].sortstrength PRIMARY -attribute[0].sortlocale "" -attribute[0].enablebitvectors false -attribute[0].enableonlybitvector false -attribute[0].fastaccess false -attribute[0].arity 8 -attribute[0].lowerbound -9223372036854775808 -attribute[0].upperbound 9223372036854775807 -attribute[0].densepostinglistthreshold 0.4 -attribute[0].tensortype "" -attribute[0].imported false -attribute[1].name "my_pos_zcurve" -attribute[1].datatype INT64 -attribute[1].collectiontype SINGLE -attribute[1].removeifzero false -attribute[1].createifnonexistent false -attribute[1].fastsearch true -attribute[1].huge false -attribute[1].ismutable false -attribute[1].sortascending true -attribute[1].sortfunction UCA -attribute[1].sortstrength PRIMARY -attribute[1].sortlocale "" -attribute[1].enablebitvectors false -attribute[1].enableonlybitvector false -attribute[1].fastaccess false -attribute[1].arity 8 -attribute[1].lowerbound -9223372036854775808 -attribute[1].upperbound 9223372036854775807 -attribute[1].densepostinglistthreshold 0.4 -attribute[1].tensortype "" -attribute[1].imported true
\ No newline at end of file +attribute[].name "parent_ref" +attribute[].datatype REFERENCE +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_pos_zcurve" +attribute[].datatype INT64 +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch true +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/imported_struct_fields/attributes.cfg b/config-model/src/test/derived/imported_struct_fields/attributes.cfg index ce6ff5e54ae..e1969e991dd 100644 --- a/config-model/src/test/derived/imported_struct_fields/attributes.cfg +++ b/config-model/src/test/derived/imported_struct_fields/attributes.cfg @@ -1,168 +1,192 @@ -attribute[0].name "parent_ref" -attribute[0].datatype REFERENCE -attribute[0].collectiontype SINGLE -attribute[0].removeifzero false -attribute[0].createifnonexistent false -attribute[0].fastsearch false -attribute[0].huge false -attribute[0].ismutable false -attribute[0].sortascending true -attribute[0].sortfunction UCA -attribute[0].sortstrength PRIMARY -attribute[0].sortlocale "" -attribute[0].enablebitvectors false -attribute[0].enableonlybitvector false -attribute[0].fastaccess false -attribute[0].arity 8 -attribute[0].lowerbound -9223372036854775808 -attribute[0].upperbound 9223372036854775807 -attribute[0].densepostinglistthreshold 0.4 -attribute[0].tensortype "" -attribute[0].imported false -attribute[1].name "my_elem_array.name" -attribute[1].datatype STRING -attribute[1].collectiontype SINGLE -attribute[1].removeifzero false -attribute[1].createifnonexistent false -attribute[1].fastsearch true -attribute[1].huge false -attribute[1].ismutable false -attribute[1].sortascending true -attribute[1].sortfunction UCA -attribute[1].sortstrength PRIMARY -attribute[1].sortlocale "" -attribute[1].enablebitvectors false -attribute[1].enableonlybitvector false -attribute[1].fastaccess false -attribute[1].arity 8 -attribute[1].lowerbound -9223372036854775808 -attribute[1].upperbound 9223372036854775807 -attribute[1].densepostinglistthreshold 0.4 -attribute[1].tensortype "" -attribute[1].imported true -attribute[2].name "my_elem_array.weight" -attribute[2].datatype INT32 -attribute[2].collectiontype SINGLE -attribute[2].removeifzero false -attribute[2].createifnonexistent false -attribute[2].fastsearch false -attribute[2].huge false -attribute[2].ismutable false -attribute[2].sortascending true -attribute[2].sortfunction UCA -attribute[2].sortstrength PRIMARY -attribute[2].sortlocale "" -attribute[2].enablebitvectors false -attribute[2].enableonlybitvector false -attribute[2].fastaccess false -attribute[2].arity 8 -attribute[2].lowerbound -9223372036854775808 -attribute[2].upperbound 9223372036854775807 -attribute[2].densepostinglistthreshold 0.4 -attribute[2].tensortype "" -attribute[2].imported true -attribute[3].name "my_elem_map.key" -attribute[3].datatype STRING -attribute[3].collectiontype SINGLE -attribute[3].removeifzero false -attribute[3].createifnonexistent false -attribute[3].fastsearch true -attribute[3].huge false -attribute[3].ismutable false -attribute[3].sortascending true -attribute[3].sortfunction UCA -attribute[3].sortstrength PRIMARY -attribute[3].sortlocale "" -attribute[3].enablebitvectors false -attribute[3].enableonlybitvector false -attribute[3].fastaccess false -attribute[3].arity 8 -attribute[3].lowerbound -9223372036854775808 -attribute[3].upperbound 9223372036854775807 -attribute[3].densepostinglistthreshold 0.4 -attribute[3].tensortype "" -attribute[3].imported true -attribute[4].name "my_elem_map.value.name" -attribute[4].datatype STRING -attribute[4].collectiontype SINGLE -attribute[4].removeifzero false -attribute[4].createifnonexistent false -attribute[4].fastsearch true -attribute[4].huge false -attribute[4].ismutable false -attribute[4].sortascending true -attribute[4].sortfunction UCA -attribute[4].sortstrength PRIMARY -attribute[4].sortlocale "" -attribute[4].enablebitvectors false -attribute[4].enableonlybitvector false -attribute[4].fastaccess false -attribute[4].arity 8 -attribute[4].lowerbound -9223372036854775808 -attribute[4].upperbound 9223372036854775807 -attribute[4].densepostinglistthreshold 0.4 -attribute[4].tensortype "" -attribute[4].imported true -attribute[5].name "my_elem_map.value.weight" -attribute[5].datatype INT32 -attribute[5].collectiontype SINGLE -attribute[5].removeifzero false -attribute[5].createifnonexistent false -attribute[5].fastsearch false -attribute[5].huge false -attribute[5].ismutable false -attribute[5].sortascending true -attribute[5].sortfunction UCA -attribute[5].sortstrength PRIMARY -attribute[5].sortlocale "" -attribute[5].enablebitvectors false -attribute[5].enableonlybitvector false -attribute[5].fastaccess false -attribute[5].arity 8 -attribute[5].lowerbound -9223372036854775808 -attribute[5].upperbound 9223372036854775807 -attribute[5].densepostinglistthreshold 0.4 -attribute[5].tensortype "" -attribute[5].imported true -attribute[6].name "my_str_int_map.key" -attribute[6].datatype STRING -attribute[6].collectiontype SINGLE -attribute[6].removeifzero false -attribute[6].createifnonexistent false -attribute[6].fastsearch true -attribute[6].huge false -attribute[6].ismutable false -attribute[6].sortascending true -attribute[6].sortfunction UCA -attribute[6].sortstrength PRIMARY -attribute[6].sortlocale "" -attribute[6].enablebitvectors false -attribute[6].enableonlybitvector false -attribute[6].fastaccess false -attribute[6].arity 8 -attribute[6].lowerbound -9223372036854775808 -attribute[6].upperbound 9223372036854775807 -attribute[6].densepostinglistthreshold 0.4 -attribute[6].tensortype "" -attribute[6].imported true -attribute[7].name "my_str_int_map.value" -attribute[7].datatype INT32 -attribute[7].collectiontype SINGLE -attribute[7].removeifzero false -attribute[7].createifnonexistent false -attribute[7].fastsearch false -attribute[7].huge false -attribute[7].ismutable false -attribute[7].sortascending true -attribute[7].sortfunction UCA -attribute[7].sortstrength PRIMARY -attribute[7].sortlocale "" -attribute[7].enablebitvectors false -attribute[7].enableonlybitvector false -attribute[7].fastaccess false -attribute[7].arity 8 -attribute[7].lowerbound -9223372036854775808 -attribute[7].upperbound 9223372036854775807 -attribute[7].densepostinglistthreshold 0.4 -attribute[7].tensortype "" -attribute[7].imported true
\ No newline at end of file +attribute[].name "parent_ref" +attribute[].datatype REFERENCE +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_elem_array.name" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch true +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_elem_array.weight" +attribute[].datatype INT32 +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_elem_map.key" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch true +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_elem_map.value.name" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch true +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_elem_map.value.weight" +attribute[].datatype INT32 +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_str_int_map.key" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch true +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_str_int_map.value" +attribute[].datatype INT32 +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/importedfields/attributes.cfg b/config-model/src/test/derived/importedfields/attributes.cfg index 15b1771a2e8..168c9bd4659 100644 --- a/config-model/src/test/derived/importedfields/attributes.cfg +++ b/config-model/src/test/derived/importedfields/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b_ref_with_summary" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "my_int_field" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "my_string_field" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -103,6 +115,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "my_int_array_field" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -124,6 +139,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "my_int_wset_field" attribute[].datatype INT32 attribute[].collectiontype WEIGHTEDSET @@ -145,6 +163,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "my_ancient_int_field" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -166,3 +187,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/inheritance/attributes.cfg b/config-model/src/test/derived/inheritance/attributes.cfg index 4a081edbf54..05a980a8347 100644 --- a/config-model/src/test/derived/inheritance/attributes.cfg +++ b/config-model/src/test/derived/inheritance/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "overridden" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "onlymother" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -61,3 +67,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/inheritfromparent/attributes.cfg b/config-model/src/test/derived/inheritfromparent/attributes.cfg index 13f59d4925f..9f01b6c45ce 100644 --- a/config-model/src/test/derived/inheritfromparent/attributes.cfg +++ b/config-model/src/test/derived/inheritfromparent/attributes.cfg @@ -19,3 +19,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/map_attribute/attributes.cfg b/config-model/src/test/derived/map_attribute/attributes.cfg index 8901acf63d1..28cb551cced 100644 --- a/config-model/src/test/derived/map_attribute/attributes.cfg +++ b/config-model/src/test/derived/map_attribute/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "str_map.value" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "int_map.key" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -61,3 +67,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg index 665edcdf45d..caae49a0252 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "str_elem_map.value.name" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "str_elem_map.value.weight" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "int_elem_map.key" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "int_elem_map.value.name" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -103,3 +115,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/music/attributes.cfg b/config-model/src/test/derived/music/attributes.cfg index 7f9592dafc8..a045a532965 100644 --- a/config-model/src/test/derived/music/attributes.cfg +++ b/config-model/src/test/derived/music/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "pto" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "mid" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "weight" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "bgnpfrom" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -103,6 +115,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "newestedition" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -124,6 +139,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "year" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -145,6 +163,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "did" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -166,6 +187,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "cbid" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -187,6 +211,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "hiphopvalue_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -208,6 +235,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "metalvalue_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -229,3 +259,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/newrank/attributes.cfg b/config-model/src/test/derived/newrank/attributes.cfg index b33c2fbdf9b..f728b1b1d33 100644 --- a/config-model/src/test/derived/newrank/attributes.cfg +++ b/config-model/src/test/derived/newrank/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "pto" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "mid" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "weight" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "bgnpfrom" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -103,6 +115,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "newestedition" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -124,6 +139,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "year" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -145,6 +163,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "did" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -166,6 +187,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "scorekey" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -187,6 +211,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "cbid" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -208,3 +235,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/predicate_attribute/attributes.cfg b/config-model/src/test/derived/predicate_attribute/attributes.cfg index 47e07e2a524..458ee86031d 100644 --- a/config-model/src/test/derived/predicate_attribute/attributes.cfg +++ b/config-model/src/test/derived/predicate_attribute/attributes.cfg @@ -19,3 +19,6 @@ attribute[].upperbound 200 attribute[].densepostinglistthreshold 0.2 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/prefixexactattribute/attributes.cfg b/config-model/src/test/derived/prefixexactattribute/attributes.cfg index d7922a0de69..33fcecb1008 100644 --- a/config-model/src/test/derived/prefixexactattribute/attributes.cfg +++ b/config-model/src/test/derived/prefixexactattribute/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "attributefield2" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -40,3 +43,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/reference_fields/attributes.cfg b/config-model/src/test/derived/reference_fields/attributes.cfg index 12dbf896edc..58125f73f9c 100644 --- a/config-model/src/test/derived/reference_fields/attributes.cfg +++ b/config-model/src/test/derived/reference_fields/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "other_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "yet_another_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE @@ -61,3 +67,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/sorting/attributes.cfg b/config-model/src/test/derived/sorting/attributes.cfg index e88dfde03bb..3404d6a0384 100644 --- a/config-model/src/test/derived/sorting/attributes.cfg +++ b/config-model/src/test/derived/sorting/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "syntaxcheck2" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "infieldonly" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -61,3 +67,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/tensor/attributes.cfg b/config-model/src/test/derived/tensor/attributes.cfg index 4634e120a3a..a8531f73c1e 100644 --- a/config-model/src/test/derived/tensor/attributes.cfg +++ b/config-model/src/test/derived/tensor/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "tensor<float>(x[2],y[1])" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "f3" attribute[].datatype TENSOR attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "tensor(x{})" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "f4" attribute[].datatype TENSOR attribute[].collectiontype SINGLE @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "tensor(x[10],y[10])" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "f5" attribute[].datatype TENSOR attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "tensor<float>(x[10])" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "f6" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -103,3 +115,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/types/attributes.cfg b/config-model/src/test/derived/types/attributes.cfg index e6ffc37e871..290dd5b9a8b 100644 --- a/config-model/src/test/derived/types/attributes.cfg +++ b/config-model/src/test/derived/types/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "along" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "abool" attribute[].datatype BOOL attribute[].collectiontype SINGLE @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "ashortfloat" attribute[].datatype FLOAT16 attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "arrayfield" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -103,6 +115,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "setfield" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -124,6 +139,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "setfield2" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -145,6 +163,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "setfield3" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -166,6 +187,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "setfield4" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -187,6 +211,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "tagfield" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -208,6 +235,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "juletre" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -229,6 +259,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "album1" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -250,6 +283,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "other" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -271,3 +307,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 7b4b650295c..c010b23e207 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -1315,7 +1315,6 @@ public class ModelProvisioningTest { } @Test - @Ignore // TODO: Enable when turning the port check on public void testThatStandaloneSyntaxOnHostedVespaRequiresDefaultPort() { try { String services = diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index 3fc05d789f6..e785792839d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -155,4 +155,9 @@ public class ExportingTestCase extends AbstractExportingTestCase { assertCorrectConfigFiles("tensor2"); } + @Test + public void testHnswIndex() throws IOException, ParseException { + assertCorrectDeriving("hnsw_index"); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java index b6569357495..b9702c6c4f7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java @@ -1,11 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; -import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.config.model.test.TestUtil; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; + +import static com.yahoo.searchdefinition.SearchBuilder.createFromString; +import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** @@ -16,7 +21,7 @@ public class TensorFieldTestCase { @Test public void requireThatTensorFieldCannotBeOfCollectionType() throws ParseException { try { - SearchBuilder.createFromString(getSd("field f1 type array<tensor(x{})> {}")); + createFromString(getSd("field f1 type array<tensor(x{})> {}")); fail("Expected exception"); } catch (IllegalArgumentException e) { @@ -28,11 +33,12 @@ public class TensorFieldTestCase { @Test public void requireThatTensorFieldCannotBeIndexField() throws ParseException { try { - SearchBuilder.createFromString(getSd("field f1 type tensor(x{}) { indexing: index }")); + createFromString(getSd("field f1 type tensor(x{}) { indexing: index }")); fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("For search 'test', field 'f1': A field of type 'tensor' cannot be specified as an 'index' field.", + assertEquals("For search 'test', field 'f1': A tensor of type 'tensor(x{})' does not support having an 'index'. " + + "Currently, only tensors with 1 indexed dimension supports that.", e.getMessage()); } } @@ -40,7 +46,7 @@ public class TensorFieldTestCase { @Test public void requireThatTensorAttributeCannotBeFastSearch() throws ParseException { try { - SearchBuilder.createFromString(getSd("field f1 type tensor(x{}) { indexing: attribute \n attribute: fast-search }")); + createFromString(getSd("field f1 type tensor(x{}) { indexing: attribute \n attribute: fast-search }")); fail("Expected exception"); } catch (IllegalArgumentException e) { @@ -51,7 +57,7 @@ public class TensorFieldTestCase { @Test public void requireThatIllegalTensorTypeSpecThrowsException() throws ParseException { try { - SearchBuilder.createFromString(getSd("field f1 type tensor(invalid) { indexing: attribute }")); + createFromString(getSd("field f1 type tensor(invalid) { indexing: attribute }")); fail("Expected exception"); } catch (IllegalArgumentException e) { @@ -59,8 +65,67 @@ public class TensorFieldTestCase { } } + @Test + public void hnsw_index_is_default_turned_off() throws ParseException { + var attr = createFromString(getSd("field t1 type tensor(x[64]) { indexing: attribute }")) + .getSearch().getAttribute("t1"); + assertFalse(attr.hnswIndexParams().isPresent()); + } + + @Test + public void hnsw_index_gets_default_parameters_if_not_specified() throws ParseException { + assertHnswIndexParams("", 16, 200); + assertHnswIndexParams("index: hnsw", 16, 200); + } + + @Test + public void hnsw_index_parameters_can_be_specified() throws ParseException { + assertHnswIndexParams("index { hnsw { max-links-per-node: 32 } }", 32, 200); + assertHnswIndexParams("index { hnsw { neighbors-to-explore-at-insert: 300 } }", 16, 300); + assertHnswIndexParams(joinLines("index {", + " hnsw {", + " max-links-per-node: 32", + " neighbors-to-explore-at-insert: 300", + " }", + "}"), + 32, 300); + } + + @Test + public void tensor_with_hnsw_index_must_be_an_attribute() throws ParseException { + try { + createFromString(getSd("field t1 type tensor(x[64]) { indexing: index }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 't1': A tensor that has an index must also be an attribute.", e.getMessage()); + } + } + private static String getSd(String field) { - return "search test {\n document test {\n" + field + "}\n}\n"; + return joinLines("search test {", + " document test {", + " " + field, + " }", + "}"); + } + + private void assertHnswIndexParams(String indexSpec, int maxLinksPerNode, int neighborsToExploreAtInsert) throws ParseException { + var sd = getSdWithIndexSpec(indexSpec); + System.out.println(sd); + var search = createFromString(sd).getSearch(); + var attr = search.getAttribute("t1"); + var params = attr.hnswIndexParams(); + assertTrue(params.isPresent()); + assertEquals(maxLinksPerNode, params.get().maxLinksPerNode()); + assertEquals(neighborsToExploreAtInsert, params.get().neighborsToExploreAtInsert()); + } + + private String getSdWithIndexSpec(String indexSpec) { + return getSd(joinLines("field t1 type tensor(x[64]) {", + " indexing: attribute | index", + " " + indexSpec, + "}")); } private void assertStartsWith(String prefix, String string) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java new file mode 100644 index 00000000000..b441f1e1993 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java @@ -0,0 +1,244 @@ +package com.yahoo.vespa.model.admin.metricsproxy; + +import ai.vespa.metricsproxy.core.ConsumersConfig; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.admin.monitoring.Metric; +import com.yahoo.vespa.model.admin.monitoring.MetricSet; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.checkMetric; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromModel; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromXml; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getCustomConsumer; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.servicesWithAdminOnly; +import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicConsumer.DEFAULT_PUBLIC_CONSUMER_ID; +import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicMetrics.defaultPublicMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.VESPA_CONSUMER_ID; +import static java.util.Collections.singleton; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Tests for {@link MetricsProxyContainerCluster} related to metrics consumers. + * + * @author gjoranv + */ +public class MetricsConsumersTest { + + private static int numPublicDefaultMetrics = defaultPublicMetricSet.getMetrics().size(); + private static int numDefaultVespaMetrics = defaultVespaMetricSet.getMetrics().size(); + private static int numVespaMetrics = vespaMetricSet.getMetrics().size(); + private static int numSystemMetrics = systemMetricSet.getMetrics().size(); + private static int numNetworkMetrics = networkMetricSet.getMetrics().size(); + private static int numMetricsForVespaConsumer = numVespaMetrics + numSystemMetrics + numNetworkMetrics; + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Test + public void default_public_consumer_is_set_up_for_self_hosted() { + ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted); + assertEquals(2, config.consumer().size()); + assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID); + + int numMetricsForPublicDefaultConsumer = defaultPublicMetricSet.getMetrics().size() + numSystemMetrics; + assertEquals(numMetricsForPublicDefaultConsumer, config.consumer(1).metric().size()); + } + + @Test + public void vespa_consumer_and_default_public_consumer_is_set_up_for_hosted() { + ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), hosted); + assertEquals(2, config.consumer().size()); + assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID); + assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID); + } + + @Test + public void vespa_consumer_is_always_present_and_has_all_vespa_metrics_and_all_system_metrics() { + ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted); + assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID); + assertEquals(numMetricsForVespaConsumer, config.consumer(0).metric().size()); + } + + @Test + public void vespa_consumer_can_be_amended_via_admin_object() { + VespaModel model = getModel(servicesWithAdminOnly(), self_hosted); + var additionalMetric = new Metric("additional-metric"); + model.getAdmin().setAdditionalDefaultMetrics(new MetricSet("amender-metrics", singleton(additionalMetric))); + + ConsumersConfig config = consumersConfigFromModel(model); + assertEquals(numMetricsForVespaConsumer + 1, config.consumer(0).metric().size()); + + ConsumersConfig.Consumer vespaConsumer = config.consumer(0); + assertTrue("Did not contain additional metric", checkMetric(vespaConsumer, additionalMetric)); + } + + @Test + public void vespa_is_a_reserved_consumer_id() { + assertReservedConsumerId("Vespa"); + } + + @Test + public void default_is_a_reserved_consumer_id() { + assertReservedConsumerId("default"); + } + + private void assertReservedConsumerId(String consumerId) { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='" + consumerId + "'/>", + " </metrics>", + " </admin>", + "</services>" + ); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("'" + consumerId + "' is not allowed as metrics consumer id"); + consumersConfigFromXml(services, self_hosted); + } + + @Test + public void vespa_consumer_id_is_allowed_for_hosted_infrastructure_applications() { + String services = String.join("\n", + "<services application-type='hosted-infrastructure'>", + " <admin version='4.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='Vespa'>", + " <metric id='custom.metric1'/>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + VespaModel hostedModel = getModel(services, hosted); + ConsumersConfig config = consumersConfigFromModel(hostedModel); + assertEquals(2, config.consumer().size()); + + // All default metrics are retained + ConsumersConfig.Consumer vespaConsumer = config.consumer(0); + assertEquals(numMetricsForVespaConsumer + 1, vespaConsumer.metric().size()); + + Metric customMetric1 = new Metric("custom.metric1"); + assertTrue("Did not contain metric: " + customMetric1, checkMetric(vespaConsumer, customMetric1)); + } + + @Test + public void consumer_id_is_case_insensitive() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='A'/>", + " <consumer id='a'/>", + " </metrics>", + " </admin>", + "</services>" + ); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("'a' is used as id for two metrics consumers"); + consumersConfigFromXml(services, self_hosted); + } + + @Test + public void non_existent_metric_set_causes_exception() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='consumer-with-non-existent-default-set'>", + " <metric-set id='non-existent'/>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("No such metric-set: non-existent"); + consumersConfigFromXml(services, self_hosted); + } + + @Test + public void consumer_with_no_metric_set_has_its_own_metrics_plus_system_metrics_plus_default_vespa_metrics() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='consumer-with-metrics-only'>", + " <metric id='custom.metric1'/>", + " <metric id='custom.metric2'/>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + ConsumersConfig.Consumer consumer = getCustomConsumer(services); + + assertEquals(numSystemMetrics + numDefaultVespaMetrics + 2, consumer.metric().size()); + + Metric customMetric1 = new Metric("custom.metric1"); + Metric customMetric2 = new Metric("custom.metric2"); + assertTrue("Did not contain metric: " + customMetric1, checkMetric(consumer, customMetric1)); + assertTrue("Did not contain metric: " + customMetric2, checkMetric(consumer, customMetric2)); + } + + @Test + public void consumer_with_default_public_metric_set_has_all_public_metrics_plus_all_system_metrics_plus_its_own() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='consumer-with-public-default-set'>", + " <metric-set id='public'/>", + " <metric id='custom.metric'/>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + ConsumersConfig.Consumer consumer = getCustomConsumer(services); + + assertEquals(numPublicDefaultMetrics + numSystemMetrics + 1, consumer.metric().size()); + + Metric customMetric = new Metric("custom.metric"); + assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric)); + } + + @Test + public void consumer_with_vespa_metric_set_has_all_vespa_metrics_plus_all_system_metrics_plus_its_own() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='consumer-with-vespa-set'>", + " <metric-set id='vespa'/>", + " <metric id='my.extra.metric'/>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + ConsumersConfig.Consumer consumer = getCustomConsumer(services); + assertEquals(numVespaMetrics + numSystemMetrics + 1, consumer.metric().size()); + + Metric customMetric = new Metric("my.extra.metric"); + assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric)); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index 9265e4437f1..bed77bd5c77 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -5,10 +5,9 @@ package com.yahoo.vespa.model.admin.metricsproxy; -import ai.vespa.metricsproxy.core.ConsumersConfig; -import ai.vespa.metricsproxy.http.metrics.MetricsV1Handler; import ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler; import ai.vespa.metricsproxy.http.application.MetricsNodesConfig; +import ai.vespa.metricsproxy.http.metrics.MetricsV1Handler; import ai.vespa.metricsproxy.http.prometheus.PrometheusHandler; import ai.vespa.metricsproxy.http.yamas.YamasHandler; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; @@ -21,13 +20,9 @@ import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames; -import com.yahoo.vespa.model.admin.monitoring.Metric; -import com.yahoo.vespa.model.admin.monitoring.MetricSet; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import java.util.Collection; @@ -39,26 +34,17 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.M import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.MY_TENANT; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.checkMetric; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromModel; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromXml; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getApplicationDimensionsConfig; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getCustomConsumer; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getMetricsNodesConfig; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getQrStartConfig; -import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicConsumer.DEFAULT_PUBLIC_CONSUMER_ID; -import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicMetrics.defaultPublicMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.VESPA_CONSUMER_ID; -import static java.util.Collections.singleton; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.servicesWithAdminOnly; import static java.util.stream.Collectors.toList; import static org.hamcrest.CoreMatchers.endsWith; import static org.hamcrest.CoreMatchers.hasItem; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -67,16 +53,6 @@ import static org.junit.Assert.assertTrue; */ public class MetricsProxyContainerClusterTest { - private static int numPublicDefaultMetrics = defaultPublicMetricSet.getMetrics().size(); - private static int numDefaultVespaMetrics = defaultVespaMetricSet.getMetrics().size(); - private static int numVespaMetrics = vespaMetricSet.getMetrics().size(); - private static int numSystemMetrics = systemMetricSet.getMetrics().size(); - private static int numNetworkMetrics = networkMetricSet.getMetrics().size(); - private static int numMetricsForVespaConsumer = numVespaMetrics + numSystemMetrics + numNetworkMetrics; - - @Rule - public ExpectedException thrown = ExpectedException.none(); - @Test public void metrics_proxy_bundle_is_included_in_bundles_config() { VespaModel model = getModel(servicesWithAdminOnly(), self_hosted); @@ -105,10 +81,11 @@ public class MetricsProxyContainerClusterTest { assertEquals(512, qrStartConfig.jvm().heapsize()); assertEquals(0, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory()); assertEquals(2, qrStartConfig.jvm().availableProcessors()); - assertEquals(false, qrStartConfig.jvm().verbosegc()); + assertFalse(qrStartConfig.jvm().verbosegc()); assertEquals("-XX:+UseG1GC -XX:MaxTenuringThreshold=15", qrStartConfig.jvm().gcopts()); assertEquals(512, qrStartConfig.jvm().stacksize()); assertEquals(0, qrStartConfig.jvm().directMemorySizeCache()); + assertEquals(32, qrStartConfig.jvm().compressedClassSpaceSize()); assertEquals(75, qrStartConfig.jvm().baseMaxDirectMemorySize()); } @@ -131,203 +108,6 @@ public class MetricsProxyContainerClusterTest { } @Test - public void default_public_consumer_is_set_up_for_self_hosted() { - ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted); - assertEquals(2, config.consumer().size()); - assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID); - - int numMetricsForPublicDefaultConsumer = defaultPublicMetricSet.getMetrics().size() + numSystemMetrics; - assertEquals(numMetricsForPublicDefaultConsumer, config.consumer(1).metric().size()); - } - - @Test - public void vespa_consumer_and_default_public_consumer_is_set_up_for_hosted() { - ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), hosted); - assertEquals(2, config.consumer().size()); - assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID); - assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID); - } - - @Test - public void vespa_consumer_is_always_present_and_has_all_vespa_metrics_and_all_system_metrics() { - ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted); - assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID); - assertEquals(numMetricsForVespaConsumer, config.consumer(0).metric().size()); - } - - @Test - public void vespa_consumer_can_be_amended_via_admin_object() { - VespaModel model = getModel(servicesWithAdminOnly(), self_hosted); - var additionalMetric = new Metric("additional-metric"); - model.getAdmin().setAdditionalDefaultMetrics(new MetricSet("amender-metrics", singleton(additionalMetric))); - - ConsumersConfig config = consumersConfigFromModel(model); - assertEquals(numMetricsForVespaConsumer + 1, config.consumer(0).metric().size()); - - ConsumersConfig.Consumer vespaConsumer = config.consumer(0); - assertTrue("Did not contain additional metric", checkMetric(vespaConsumer, additionalMetric)); - } - - @Test - public void vespa_is_a_reserved_consumer_id() { - assertReservedConsumerId("Vespa"); - } - - @Test - public void default_is_a_reserved_consumer_id() { - assertReservedConsumerId("default"); - } - - private void assertReservedConsumerId(String consumerId) { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='" + consumerId + "'/>", - " </metrics>", - " </admin>", - "</services>" - ); - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("'" + consumerId + "' is not allowed as metrics consumer id"); - consumersConfigFromXml(services, self_hosted); - } - - @Test - public void vespa_consumer_id_is_allowed_for_hosted_infrastructure_applications() { - String services = String.join("\n", - "<services application-type='hosted-infrastructure'>", - " <admin version='4.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='Vespa'>", - " <metric id='custom.metric1'/>", - " </consumer>", - " </metrics>", - " </admin>", - "</services>" - ); - VespaModel hostedModel = getModel(services, hosted); - ConsumersConfig config = consumersConfigFromModel(hostedModel); - assertEquals(2, config.consumer().size()); - - // All default metrics are retained - ConsumersConfig.Consumer vespaConsumer = config.consumer(0); - assertEquals(numMetricsForVespaConsumer + 1, vespaConsumer.metric().size()); - - Metric customMetric1 = new Metric("custom.metric1"); - assertTrue("Did not contain metric: " + customMetric1, checkMetric(vespaConsumer, customMetric1)); - } - - @Test - public void consumer_id_is_case_insensitive() { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='A'/>", - " <consumer id='a'/>", - " </metrics>", - " </admin>", - "</services>" - ); - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("'a' is used as id for two metrics consumers"); - consumersConfigFromXml(services, self_hosted); - } - - @Test - public void non_existent_metric_set_causes_exception() { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='consumer-with-non-existent-default-set'>", - " <metric-set id='non-existent'/>", - " </consumer>", - " </metrics>", - " </admin>", - "</services>" - ); - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("No such metric-set: non-existent"); - consumersConfigFromXml(services, self_hosted); - } - - @Test - public void consumer_with_no_metric_set_has_its_own_metrics_plus_system_metrics_plus_default_vespa_metrics() { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='consumer-with-metrics-only'>", - " <metric id='custom.metric1'/>", - " <metric id='custom.metric2'/>", - " </consumer>", - " </metrics>", - " </admin>", - "</services>" - ); - ConsumersConfig.Consumer consumer = getCustomConsumer(services); - - assertEquals(numSystemMetrics + numDefaultVespaMetrics + 2, consumer.metric().size()); - - Metric customMetric1 = new Metric("custom.metric1"); - Metric customMetric2 = new Metric("custom.metric2"); - assertTrue("Did not contain metric: " + customMetric1, checkMetric(consumer, customMetric1)); - assertTrue("Did not contain metric: " + customMetric2, checkMetric(consumer, customMetric2)); - } - - @Test - public void consumer_with_default_public_metric_set_has_all_public_metrics_plus_all_system_metrics_plus_its_own() { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='consumer-with-public-default-set'>", - " <metric-set id='public'/>", - " <metric id='custom.metric'/>", - " </consumer>", - " </metrics>", - " </admin>", - "</services>" - ); - ConsumersConfig.Consumer consumer = getCustomConsumer(services); - - assertEquals(numPublicDefaultMetrics + numSystemMetrics + 1, consumer.metric().size()); - - Metric customMetric = new Metric("custom.metric"); - assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric)); - } - - @Test - public void consumer_with_vespa_metric_set_has_all_vespa_metrics_plus_all_system_metrics_plus_its_own() { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='consumer-with-vespa-set'>", - " <metric-set id='vespa'/>", - " <metric id='my.extra.metric'/>", - " </consumer>", - " </metrics>", - " </admin>", - "</services>" - ); - ConsumersConfig.Consumer consumer = getCustomConsumer(services); - assertEquals(numVespaMetrics + numSystemMetrics + 1, consumer.metric().size()); - - Metric customMetric = new Metric("my.extra.metric"); - assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric)); - } - - @Test public void hosted_application_propagates_application_dimensions() { VespaModel hostedModel = getModel(servicesWithAdminOnly(), hosted); ApplicationDimensionsConfig config = getApplicationDimensionsConfig(hostedModel); @@ -357,16 +137,6 @@ public class MetricsProxyContainerClusterTest { assertEquals(MetricsV1Handler.VALUES_PATH, node.metricsPath()); } - private static String servicesWithAdminOnly() { - return String.join("\n", - "<services>", - " <admin version='4.0'>", - " <adminserver hostalias='node1'/>", - " </admin>", - "</services>" - ); - } - private static String servicesWithTwoNodes() { return String.join("\n", "<services>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java index 7cbc9db5eb2..8ecb13d7ae5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java @@ -55,6 +55,16 @@ class MetricsProxyModelTester { : CONTAINER_CONFIG_ID; } + static String servicesWithAdminOnly() { + return String.join("\n", + "<services>", + " <admin version='4.0'>", + " <adminserver hostalias='node1'/>", + " </admin>", + "</services>" + ); + } + static boolean checkMetric(ConsumersConfig.Consumer consumer, Metric metric) { for (ConsumersConfig.Consumer.Metric m : consumer.metric()) { if (metric.name.equals(m.name()) && metric.outputName.equals(m.outputname())) @@ -77,32 +87,32 @@ class MetricsProxyModelTester { } static ConsumersConfig consumersConfigFromModel(VespaModel model) { - return new ConsumersConfig((ConsumersConfig.Builder) model.getConfig(new ConsumersConfig.Builder(), CLUSTER_CONFIG_ID)); + return model.getConfig(ConsumersConfig.class, CLUSTER_CONFIG_ID); } static MetricsNodesConfig getMetricsNodesConfig(VespaModel model) { - return new MetricsNodesConfig((MetricsNodesConfig.Builder) model.getConfig(new MetricsNodesConfig.Builder(), CLUSTER_CONFIG_ID)); + return model.getConfig(MetricsNodesConfig.class, CLUSTER_CONFIG_ID); } static ApplicationDimensionsConfig getApplicationDimensionsConfig(VespaModel model) { - return new ApplicationDimensionsConfig((ApplicationDimensionsConfig.Builder) model.getConfig(new ApplicationDimensionsConfig.Builder(), CLUSTER_CONFIG_ID)); + return model.getConfig(ApplicationDimensionsConfig.class, CLUSTER_CONFIG_ID); } static QrStartConfig getQrStartConfig(VespaModel model) { - return new QrStartConfig((QrStartConfig.Builder) model.getConfig(new QrStartConfig.Builder(), CLUSTER_CONFIG_ID)); + return model.getConfig(QrStartConfig.class, CLUSTER_CONFIG_ID); } static NodeDimensionsConfig getNodeDimensionsConfig(VespaModel model, String configId) { - return new NodeDimensionsConfig((NodeDimensionsConfig.Builder) model.getConfig(new NodeDimensionsConfig.Builder(), configId)); + return model.getConfig(NodeDimensionsConfig.class, configId); } static VespaServicesConfig getVespaServicesConfig(String servicesXml) { VespaModel model = getModel(servicesXml, self_hosted); - return new VespaServicesConfig((VespaServicesConfig.Builder) model.getConfig(new VespaServicesConfig.Builder(), CONTAINER_CONFIG_ID)); + return model.getConfig(VespaServicesConfig.class, CONTAINER_CONFIG_ID); } static RpcConnectorConfig getRpcConnectorConfig(VespaModel model) { - return new RpcConnectorConfig((RpcConnectorConfig.Builder) model.getConfig(new RpcConnectorConfig.Builder(), CONTAINER_CONFIG_ID)); + return model.getConfig(RpcConnectorConfig.class, CONTAINER_CONFIG_ID); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java new file mode 100644 index 00000000000..144c45a7dd2 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java @@ -0,0 +1,87 @@ +package com.yahoo.vespa.model.admin.metricsproxy; + +import ai.vespa.metricsproxy.telegraf.TelegrafConfig; +import com.yahoo.vespa.model.VespaModel; +import org.junit.Test; + +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CLUSTER_CONFIG_ID; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; +import static org.junit.Assert.assertEquals; + +/** + * @author gjoranv + */ +public class TelegrafTest { + + @Test + public void telegraf_config_is_generated_for_cloudwatch_in_services() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='cloudwatch-consumer'>", + " <metric id='my-metric'/>", + " <cloudwatch region='us-east-1' namespace='my-namespace' >", + " <access-key-name>my-access-key</access-key-name>", + " <secret-key-name>my-secret-key</secret-key-name>", + " </cloudwatch>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + VespaModel hostedModel = getModel(services, hosted); + TelegrafConfig config = hostedModel.getConfig(TelegrafConfig.class, CLUSTER_CONFIG_ID); + var cloudWatch0 = config.cloudWatch(0); + assertEquals("cloudwatch-consumer", cloudWatch0.consumer()); + assertEquals("us-east-1", cloudWatch0.region()); + assertEquals("my-namespace", cloudWatch0.namespace()); + assertEquals("my-access-key", cloudWatch0.accessKeyName()); + assertEquals("my-secret-key", cloudWatch0.secretKeyName()); + assertEquals("", cloudWatch0.profile()); + } + + @Test + public void multiple_cloudwatches_are_allowed_for_the_same_consumer() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='cloudwatch-consumer'>", + " <metric id='my-metric'/>", + " <cloudwatch region='us-east-1' namespace='namespace-1' >", + " <access-key-name>access-key-1</access-key-name>", + " <secret-key-name>secret-key-1</secret-key-name>", + " </cloudwatch>", + " <cloudwatch region='us-east-1' namespace='namespace-2' >", + " <profile>profile-2</profile>", + " </cloudwatch>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + VespaModel hostedModel = getModel(services, hosted); + TelegrafConfig config = hostedModel.getConfig(TelegrafConfig.class, CLUSTER_CONFIG_ID); + + var cloudWatch0 = config.cloudWatch(0); + assertEquals("cloudwatch-consumer", cloudWatch0.consumer()); + assertEquals("us-east-1", cloudWatch0.region()); + assertEquals("namespace-1", cloudWatch0.namespace()); + assertEquals("access-key-1", cloudWatch0.accessKeyName()); + assertEquals("secret-key-1", cloudWatch0.secretKeyName()); + assertEquals("", cloudWatch0.profile()); + + var cloudWatch1 = config.cloudWatch(1); + assertEquals("cloudwatch-consumer", cloudWatch1.consumer()); + assertEquals("us-east-1", cloudWatch1.region()); + assertEquals("namespace-2", cloudWatch1.namespace()); + assertEquals("", cloudWatch1.accessKeyName()); + assertEquals("", cloudWatch1.secretKeyName()); + assertEquals("profile-2", cloudWatch1.profile()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java index 21df39ebde8..318a0630c4d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java @@ -47,7 +47,7 @@ public class EndpointCertificateSecretsValidatorTest { VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); exceptionRule.expect(CertificateNotReadyException.class); - exceptionRule.expectMessage("TLS enabled, but could not retrieve certificate yet"); + exceptionRule.expectMessage("TLS enabled, but could not yet retrieve certificate for application default:default:default"); new EndpointCertificateSecretsValidator().validate(model, deployState); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index d1c5e344c24..ce565989c18 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -97,6 +97,7 @@ public class ContainerClusterTest { cluster.getConfig(qsB); QrStartConfig qsC= new QrStartConfig(qsB); assertEquals(expectedMemoryPercentage, qsC.jvm().heapSizeAsPercentageOfPhysicalMemory()); + assertEquals(0, qsC.jvm().compressedClassSpaceSize()); } @Test @@ -156,6 +157,7 @@ public class ContainerClusterTest { QrStartConfig qrStartConfig = new QrStartConfig(qrBuilder); assertEquals(32, qrStartConfig.jvm().minHeapsize()); assertEquals(512, qrStartConfig.jvm().heapsize()); + assertEquals(32, qrStartConfig.jvm().compressedClassSpaceSize()); assertEquals(0, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory()); ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 75b41ff3667..53c99d1d3dc 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -151,7 +151,6 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test - @Ignore // TODO: Enable when turning the port check on public void fail_if_http_port_is_not_default_in_hosted_vespa() throws Exception { try { String servicesXml = diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java index 9ecd33f4273..eda90b03147 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java @@ -28,7 +28,11 @@ import java.util.Map; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.core.IsNull.notNullValue; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertThat; + /** * @author einarmr @@ -204,24 +208,25 @@ public class DocprocBuilderTest extends DomBuilderTest { @Test public void testBundlesConfig() { - assertThat(bundlesConfig.bundle().size(), is(0)); + assertTrue(bundlesConfig.bundle().isEmpty()); } @Test public void testSchemaMappingConfig() { - assertThat(schemamappingConfig.fieldmapping().size(), is(0)); + assertTrue(schemamappingConfig.fieldmapping().isEmpty()); } @Test public void testQrStartConfig() { QrStartConfig.Jvm jvm = qrStartConfig.jvm(); - assertThat(jvm.server(), is(true)); - assertThat(jvm.verbosegc(), is(true)); - assertThat(jvm.gcopts(), is("-XX:+UseG1GC -XX:MaxTenuringThreshold=15")); - assertThat(jvm.minHeapsize(), is(1536)); - assertThat(jvm.heapsize(), is(1536)); - assertThat(jvm.stacksize(), is(512)); - assertThat(qrStartConfig.ulimitv(), is("")); + assertTrue(jvm.server()); + assertTrue(jvm.verbosegc()); + assertEquals("-XX:+UseG1GC -XX:MaxTenuringThreshold=15", jvm.gcopts()); + assertEquals(1536, jvm.minHeapsize()); + assertEquals(1536, jvm.heapsize()); + assertEquals(512, jvm.stacksize()); + assertTrue(qrStartConfig.ulimitv().isEmpty()); + assertEquals(0, jvm.compressedClassSpaceSize()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java index 1c4e005cb67..70e307e1748 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java @@ -18,12 +18,15 @@ import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.search.AbstractSearchCluster; -import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.Test; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + /** * Unit tests for SearchCluster. Please use this instead of SearchModelTestCase if possible and @@ -160,23 +163,30 @@ public class SearchClusterTest { AbstractSearchCluster searchCluster2 = model.getSearchClusters().get(xbulkIndex); assertEquals("xbulk", searchCluster2.getClusterName()); - Component<?,?> normalDispatcher = (Component<?, ?>)containerCluster1.getComponentsMap().get(new ComponentId("dispatcher.normal")); - assertNotNull(normalDispatcher); - assertEquals("dispatcher.normal", normalDispatcher.getComponentId().stringValue()); - assertEquals("com.yahoo.search.dispatch.Dispatcher", normalDispatcher.getClassId().stringValue()); - assertEquals("j1/component/dispatcher.normal", normalDispatcher.getConfigId()); - DispatchConfig.Builder normalDispatchConfigBuilder = new DispatchConfig.Builder(); - model.getConfig(normalDispatchConfigBuilder, "j1/component/dispatcher.normal"); - assertEquals("node2host", normalDispatchConfigBuilder.build().node(0).host()); - - Component<?,?> xbulkDispatcher = (Component<?, ?>)containerCluster1.getComponentsMap().get(new ComponentId("dispatcher.xbulk")); - assertNotNull(xbulkDispatcher); - assertEquals("dispatcher.xbulk", xbulkDispatcher.getComponentId().stringValue()); - assertEquals("com.yahoo.search.dispatch.Dispatcher", xbulkDispatcher.getClassId().stringValue()); - assertEquals("j1/component/dispatcher.xbulk", xbulkDispatcher.getConfigId()); - DispatchConfig.Builder xbulkDispatchConfigBuilder = new DispatchConfig.Builder(); - model.getConfig(xbulkDispatchConfigBuilder, "j1/component/dispatcher.xbulk"); - assertEquals("node0host", xbulkDispatchConfigBuilder.build().node(0).host()); + verifyDispatch(model, containerCluster1, "normal", "node2host"); + verifyDispatch(model, containerCluster1, "xbulk", "node0host"); + } + + private void verifyDispatch(VespaModel model, ContainerCluster containerCluster, String cluster, String host) { + Component<?,?> dispatcher = (Component<?, ?>)containerCluster.getComponentsMap().get(new ComponentId("dispatcher." + cluster)); + assertNotNull(dispatcher); + assertEquals("dispatcher." + cluster, dispatcher.getComponentId().stringValue()); + assertEquals("com.yahoo.search.dispatch.Dispatcher", dispatcher.getClassId().stringValue()); + assertEquals("j1/component/dispatcher." + cluster, dispatcher.getConfigId()); + DispatchConfig.Builder dispatchConfigBuilder = new DispatchConfig.Builder(); + model.getConfig(dispatchConfigBuilder, dispatcher.getConfigId()); + assertEquals(host, dispatchConfigBuilder.build().node(0).host()); + + assertTrue(dispatcher.getInjectedComponentIds().contains("rpcresourcepool." + cluster)); + + Component<?,?> rpcResourcePool = (Component<?, ?>)dispatcher.getChildren().get("rpcresourcepool." + cluster); + assertNotNull(rpcResourcePool); + assertEquals("rpcresourcepool." + cluster, rpcResourcePool.getComponentId().stringValue()); + assertEquals("com.yahoo.search.dispatch.rpc.RpcResourcePool", rpcResourcePool.getClassId().stringValue()); + assertEquals("j1/component/dispatcher." + cluster + "/rpcresourcepool." + cluster, rpcResourcePool.getConfigId()); + dispatchConfigBuilder = new DispatchConfig.Builder(); + model.getConfig(dispatchConfigBuilder, rpcResourcePool.getConfigId()); + assertEquals(host, dispatchConfigBuilder.build().node(0).host()); } } diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 1bf42650123..b06c93d6406 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -15,14 +15,25 @@ <slobrok hostalias="rtc-1" /> </slobroks> <metrics> - <consumer id="my-consumer"> + <consumer id="cloudwatch-hosted"> <metric-set id="my-set" /> <metric id="my-metric"/> <metric id="my-metric2" display-name="my-metric3"/> <metric display-name="my-metric4" id="my-metric4.avg"/> + <cloudwatch region="us-east1" namespace="my-namespace"> + <access-key-name>my-access-key</access-key-name> + <secret-key-name>my-secret-key</secret-key-name> + </cloudwatch> </consumer> - <consumer id="my-consumer2"> - <metric-set id="my-set2" /> + <consumer id="cloudwatch-self-hosted-with-default-auth"> + <metric-set id="public" /> + <cloudwatch region="us-east1" namespace="my-namespace" /> + </consumer> + <consumer id="cloudwatch-self-hosted-with-profile"> + <metric id="my-custom-metric" /> + <cloudwatch region="us-east1" namespace="another-namespace"> + <profile>profile-in-credentials-file</profile> + </cloudwatch> </consumer> </metrics> <logforwarding> diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java index 892ac639198..2603b157f91 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java @@ -14,4 +14,6 @@ public enum RoutingMethod { /** Routing happens through a dedicated layer 4 load balancer */ exclusive, + /** Routing happens through a shared layer 4 load balancer */ + sharedLayer4 } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java index d77206aee81..0966de940f1 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java @@ -70,12 +70,11 @@ public class ProxyServer implements Runnable { defaultTimingValues = tv; } - ProxyServer(Spec spec, ConfigSourceSet source, TimingValues timingValues, - MemoryCache memoryCache, ConfigSourceClient configClient) { + ProxyServer(Spec spec, ConfigSourceSet source, MemoryCache memoryCache, ConfigSourceClient configClient) { this.delayedResponses = new DelayedResponses(); this.configSource = source; log.log(LogLevel.DEBUG, "Using config source '" + source); - this.timingValues = timingValues; + this.timingValues = defaultTimingValues; this.memoryCache = memoryCache; this.rpcServer = createRpcServer(spec); this.configClient = createClient(rpcServer, delayedResponses, source, timingValues, memoryCache, configClient); @@ -181,8 +180,7 @@ public class ProxyServer implements Runnable { Event.started("configproxy"); ConfigSourceSet configSources = new ConfigSourceSet(properties.configSources); - ProxyServer proxyServer = new ProxyServer(new Spec(null, port), configSources, - defaultTimingValues(), new MemoryCache(), null); + ProxyServer proxyServer = new ProxyServer(new Spec(null, port), configSources, new MemoryCache(), null); // catch termination and interrupt signal proxyServer.setupSignalHandler(); Thread proxyserverThread = new Thread(proxyServer); diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java index ee843088086..47afbe83bb6 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java @@ -11,13 +11,14 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; import com.yahoo.log.LogLevel; -import com.yahoo.vespa.config.*; +import com.yahoo.vespa.config.ConfigCacheKey; +import com.yahoo.vespa.config.RawConfig; +import com.yahoo.vespa.config.TimingValues; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.concurrent.DelayQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -42,7 +43,7 @@ class RpcConfigSourceClient implements ConfigSourceClient { private final TimingValues timingValues; private final ExecutorService exec; - private final Map<ConfigSourceSet, JRTConfigRequester> requesterPool; + private final JRTConfigRequester requester; RpcConfigSourceClient(RpcServer rpcServer, @@ -57,20 +58,7 @@ class RpcConfigSourceClient implements ConfigSourceClient { this.timingValues = timingValues; checkConfigSources(); exec = Executors.newCachedThreadPool(new DaemonThreadFactory("subscriber-")); - requesterPool = createRequesterPool(configSourceSet, timingValues); - } - - /** - * Creates a requester (connection) pool of one entry, to be used each time this {@link RpcConfigSourceClient} is used - * @param ccs a {@link ConfigSourceSet} - * @param timingValues a {@link TimingValues} - * @return requester map - */ - private Map<ConfigSourceSet, JRTConfigRequester> createRequesterPool(ConfigSourceSet ccs, TimingValues timingValues) { - Map<ConfigSourceSet, JRTConfigRequester> ret = new HashMap<>(); - if (ccs.getSources().isEmpty()) return ret; // unit test, just skip creating any requester - ret.put(ccs, new JRTConfigRequester(new JRTConnectionPool(ccs), timingValues)); - return ret; + requester = JRTConfigRequester.create(configSourceSet, timingValues); } /** @@ -153,7 +141,7 @@ class RpcConfigSourceClient implements ConfigSourceClient { } else { log.log(LogLevel.DEBUG, () -> "Could not find good config in cache, creating subscriber for: " + configCacheKey); UpstreamConfigSubscriber subscriber = new UpstreamConfigSubscriber(input, this, configSourceSet, - timingValues, requesterPool, memoryCache); + timingValues, requester, memoryCache); try { subscriber.subscribe(); activeSubscribers.put(configCacheKey, subscriber); @@ -183,25 +171,18 @@ class RpcConfigSourceClient implements ConfigSourceClient { activeSubscribers.clear(); } exec.shutdown(); - for (JRTConfigRequester requester : requesterPool.values()) { - requester.close(); - } + requester.close(); } @Override public String getActiveSourceConnection() { - if (requesterPool.get(configSourceSet) != null) { - return requesterPool.get(configSourceSet).getConnectionPool().getCurrent().getAddress(); - } else { - return ""; - } + return requester.getConnectionPool().getCurrent().getAddress(); } @Override public List<String> getSourceConnections() { ArrayList<String> ret = new ArrayList<>(); - final JRTConfigRequester jrtConfigRequester = requesterPool.get(configSourceSet); - if (jrtConfigRequester != null) { + if (configSourceSet != null) { ret.addAll(configSourceSet.getSources()); } return ret; diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java index f8df16cb3d2..d8a8c5ce941 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java @@ -1,16 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.proxy; -import com.yahoo.config.subscription.ConfigSource; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.config.subscription.impl.GenericConfigHandle; import com.yahoo.config.subscription.impl.GenericConfigSubscriber; import com.yahoo.config.subscription.impl.JRTConfigRequester; import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.yolean.Exceptions; import com.yahoo.vespa.config.RawConfig; import com.yahoo.vespa.config.TimingValues; +import com.yahoo.yolean.Exceptions; import java.util.Map; import java.util.logging.Logger; @@ -24,26 +23,26 @@ public class UpstreamConfigSubscriber implements Subscriber { private final RawConfig config; private final ConfigSourceClient configSourceClient; - private final ConfigSource configSourceSet; + private final ConfigSourceSet configSourceSet; private final TimingValues timingValues; - private final Map<ConfigSourceSet, JRTConfigRequester> requesterPool; + private final JRTConfigRequester requester; private final MemoryCache memoryCache; private GenericConfigSubscriber subscriber; private GenericConfigHandle handle; - UpstreamConfigSubscriber(RawConfig config, ConfigSourceClient configSourceClient, ConfigSource configSourceSet, - TimingValues timingValues, Map<ConfigSourceSet, JRTConfigRequester> requesterPool, + UpstreamConfigSubscriber(RawConfig config, ConfigSourceClient configSourceClient, ConfigSourceSet configSourceSet, + TimingValues timingValues, JRTConfigRequester requester, MemoryCache memoryCache) { this.config = config; this.configSourceClient = configSourceClient; this.configSourceSet = configSourceSet; this.timingValues = timingValues; - this.requesterPool = requesterPool; + this.requester = requester; this.memoryCache = memoryCache; } void subscribe() { - subscriber = new GenericConfigSubscriber(requesterPool); + subscriber = new GenericConfigSubscriber(Map.of(configSourceSet, requester)); ConfigKey<?> key = config.getKey(); handle = subscriber.subscribe(new ConfigKey<>(key.getName(), key.getConfigId(), key.getNamespace()), config.getDefContent(), configSourceSet, timingValues); diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java index dc1c995fbb5..29bd38ea891 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java @@ -264,7 +264,7 @@ public class ConfigProxyRpcServerTest { } private static ProxyServer createTestServer(ConfigSourceSet source) { - return new ProxyServer(null, source, ProxyServer.defaultTimingValues(), new MemoryCache(), null); + return new ProxyServer(null, source, new MemoryCache(), null); } private static class TestServer implements AutoCloseable { diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java index 712567774f1..bc35a8670a3 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java @@ -14,7 +14,11 @@ import org.junit.rules.TemporaryFolder; import java.util.Optional; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author hmusum @@ -222,7 +226,7 @@ public class ProxyServerTest { private static ProxyServer createTestServer(ConfigSourceSet source, ConfigSourceClient configSourceClient, MemoryCache memoryCache) { - return new ProxyServer(null, source, ProxyServer.defaultTimingValues(), memoryCache, configSourceClient); + return new ProxyServer(null, source, memoryCache, configSourceClient); } static RawConfig createConfigWithNextConfigGeneration(RawConfig config, int errorCode) { diff --git a/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java index 80c55c7b558..474c9f7a4db 100644 --- a/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java +++ b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java @@ -8,7 +8,8 @@ import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.ConfigPayloadBuilder; -import java.util.*; +import java.util.ArrayList; +import java.util.List; /** * Deserializes config payload (cfg format) to a ConfigPayload. @@ -33,15 +34,10 @@ public class CfgConfigPayloadBuilder { int lineNum = 1; ConfigPayloadBuilder payloadBuilder = new ConfigPayloadBuilder(); for (String line : lines) { - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "line " + lineNum + ": '" + line + "'"); - } parseLine(line, lineNum, payloadBuilder); lineNum++; } - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "payload=" + payloadBuilder.toString()); - } + log.log(LogLevel.SPAM, () -> "payload=" + payloadBuilder.toString()); return payloadBuilder; } @@ -52,16 +48,13 @@ public class CfgConfigPayloadBuilder { String field = fieldAndValue.getFirst(); String value = fieldAndValue.getSecond(); if (field==null || value==null) { - log.log(LogLevel.DEBUG, "Got field without value in line " + lineNum + ": " + line + ", skipping"); + log.log(LogLevel.DEBUG, () -> "Got field without value in line " + lineNum + ": " + line + ", skipping"); return; } field=field.trim(); value=value.trim(); validateField(field, trimmedLine, lineNum); validateValue(value, trimmedLine, lineNum); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "field=" + field + ",value=" + value); - } List<String> fields = parseFieldList(field); ConfigPayloadBuilder currentBuilder = payloadBuilder; for (int fieldNum = 0; fieldNum < fields.size(); fieldNum++) { diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java index c799186435c..7472439d6a4 100755 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java @@ -3,7 +3,11 @@ package com.yahoo.config.subscription; import com.yahoo.log.LogLevel; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; import java.util.logging.Logger; diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java index 5c0b932dcce..3891d710fa3 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java @@ -319,13 +319,14 @@ public class ConfigSubscriber implements AutoCloseable { @Override public void close() { synchronized (monitor) { + if (state == State.CLOSED) return; state = State.CLOSED; } for (ConfigHandle<? extends ConfigInstance> h : subscriptionHandles) { h.subscription().close(); } closeRequesters(); - log.log(LogLevel.DEBUG, "Config subscriber has been closed."); + log.log(LogLevel.DEBUG, () -> "Config subscriber has been closed."); } /** diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java index 0887ed9aad5..8ce3449fba5 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java @@ -33,9 +33,7 @@ public class GenericJRTConfigSubscription extends JRTConfigSubscription<RawConfi @Override protected void setNewConfig(JRTClientConfigRequest jrtReq) { setConfig(jrtReq.getNewGeneration(), jrtReq.responseIsInternalRedeploy(), RawConfig.createFromResponseParameters(jrtReq) ); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "in setNewConfig, config=" + this.getConfigState().getConfig()); - } + log.log(LogLevel.DEBUG, () -> "in setNewConfig, config=" + this.getConfigState().getConfig()); } // This method is overridden because config needs to have its generation diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java index 989d8c6c8de..49c5dcd343c 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java @@ -20,7 +20,6 @@ import com.yahoo.yolean.Exceptions; import java.time.Duration; import java.time.Instant; import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; @@ -37,15 +36,17 @@ public class JRTConfigRequester implements RequestWaiter { private static final Logger log = Logger.getLogger(JRTConfigRequester.class.getName()); public static final ConfigSourceSet defaultSourceSet = ConfigSourceSet.createDefault(); + private static final JRTManagedConnectionPools managedPool = new JRTManagedConnectionPools(); private static final int TRACELEVEL = 6; private final TimingValues timingValues; private int fatalFailures = 0; // independent of transientFailures private int transientFailures = 0; // independent of fatalFailures - private final ScheduledThreadPoolExecutor scheduler = new ScheduledThreadPoolExecutor(1, new JRTSourceThreadFactory()); + private final ScheduledThreadPoolExecutor scheduler; private Instant suspendWarningLogged = Instant.MIN; private Instant noApplicationWarningLogged = Instant.MIN; private static final Duration delayBetweenWarnings = Duration.ofSeconds(60); private final ConnectionPool connectionPool; + private final ConfigSourceSet configSourceSet; static final float randomFraction = 0.2f; /* Time to be added to server timeout to create client timeout. This is the time allowed for the server to respond after serverTimeout has elapsed. */ private static final Double additionalTimeForClientTimeout = 10.0; @@ -56,11 +57,23 @@ public class JRTConfigRequester implements RequestWaiter { * @param connectionPool the connectionPool this requester should use * @param timingValues timeouts and delays used when sending JRT config requests */ - public JRTConfigRequester(ConnectionPool connectionPool, TimingValues timingValues) { + JRTConfigRequester(ConfigSourceSet configSourceSet, ScheduledThreadPoolExecutor scheduler, + ConnectionPool connectionPool, TimingValues timingValues) { + this.configSourceSet = configSourceSet; + this.scheduler = scheduler; this.connectionPool = connectionPool; this.timingValues = timingValues; } + /** Only for testing */ + public JRTConfigRequester(ConnectionPool connectionPool, TimingValues timingValues) { + this(null, new ScheduledThreadPoolExecutor(1), connectionPool, timingValues); + } + + public static JRTConfigRequester create(ConfigSourceSet sourceSet, TimingValues timingValues) { + return managedPool.acquire(sourceSet, timingValues); + } + /** * Requests the config for the {@link com.yahoo.config.ConfigInstance} on the given {@link ConfigSubscription} * @@ -77,11 +90,9 @@ public class JRTConfigRequester implements RequestWaiter { if ( ! req.validateParameters()) throw new ConfigurationRuntimeException("Error in parameters for config request: " + req); double jrtClientTimeout = getClientTimeout(req); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Requesting config for " + sub + " on connection " + connection - + " with client timeout " + jrtClientTimeout + - (log.isLoggable(LogLevel.SPAM) ? (",defcontent=" + req.getDefContent().asString()) : "")); - } + log.log(LogLevel.DEBUG, () -> "Requesting config for " + sub + " on connection " + connection + + " with client timeout " + jrtClientTimeout + + (log.isLoggable(LogLevel.SPAM) ? (",defcontent=" + req.getDefContent().asString()) : "")); connection.invokeAsync(req.getRequest(), jrtClientTimeout, this); } @@ -111,7 +122,7 @@ public class JRTConfigRequester implements RequestWaiter { if (sub.getState() == ConfigSubscription.State.CLOSED) return; // Avoid error messages etc. after closing Trace trace = jrtReq.getResponseTrace(); trace.trace(TRACELEVEL, "JRTConfigRequester.doHandle()"); - log.log(LogLevel.SPAM, trace::toString); + log.log(LogLevel.SPAM, () -> trace.toString()); if (validResponse) { handleOKRequest(jrtReq, sub, connection); } else { @@ -148,7 +159,7 @@ public class JRTConfigRequester implements RequestWaiter { // The subscription object has an "old" config, which is all we have to offer back now log.log(LogLevel.INFO, "Failure of config subscription, clients will keep existing config until resolved: " + sub); } - final ErrorType errorType = ErrorType.getErrorType(jrtReq.errorCode()); + ErrorType errorType = ErrorType.getErrorType(jrtReq.errorCode()); connectionPool.setError(connection, jrtReq.errorCode()); long delay = calculateFailedRequestDelay(errorType, transientFailures, fatalFailures, timingValues, configured); if (errorType == ErrorType.TRANSIENT) { @@ -249,7 +260,7 @@ public class JRTConfigRequester implements RequestWaiter { private void scheduleNextRequest(JRTClientConfigRequest jrtReq, JRTConfigSubscription<?> sub, long delay, long timeout) { long delayBeforeSendingRequest = (delay < 0) ? 0 : delay; JRTClientConfigRequest jrtReqNew = jrtReq.nextRequest(timeout); - log.log(LogLevel.DEBUG, timingValues::toString); + log.log(LogLevel.SPAM, () -> timingValues.toString()); log.log(LogLevel.DEBUG, () -> "Scheduling new request " + delayBeforeSendingRequest + " millis from now for " + jrtReqNew.getConfigKey()); scheduler.schedule(new GetConfigTask(jrtReqNew, sub), delayBeforeSendingRequest, TimeUnit.MILLISECONDS); } @@ -275,18 +286,8 @@ public class JRTConfigRequester implements RequestWaiter { // Fake that we have logged to avoid printing warnings after this suspendWarningLogged = Instant.now(); noApplicationWarningLogged = Instant.now(); - - connectionPool.close(); - scheduler.shutdown(); - } - - private static class JRTSourceThreadFactory implements ThreadFactory { - @Override - public Thread newThread(Runnable runnable) { - Thread t = new Thread(runnable, String.format("jrt-config-requester-%d", System.currentTimeMillis())); - // We want a daemon thread to avoid hanging threads in case something goes wrong in the config system - t.setDaemon(true); - return t; + if (configSourceSet != null) { + managedPool.release(configSourceSet); } } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java index 39e6c69f539..a94a135f9d8 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java @@ -28,7 +28,7 @@ import com.yahoo.vespa.config.protocol.Payload; public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { private JRTConfigRequester requester; - private TimingValues timingValues; + private final TimingValues timingValues; // Last time we got an OK JRT callback private Instant lastOK = Instant.MIN; @@ -156,7 +156,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc private JRTConfigRequester getRequester() { JRTConfigRequester requester = subscriber.requesters().get(sources); if (requester == null) { - requester = new JRTConfigRequester(new JRTConnectionPool(sources), timingValues); + requester = JRTConfigRequester.create(sources, timingValues); subscriber.requesters().put(sources, requester); } return requester; diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTManagedConnectionPools.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTManagedConnectionPools.java new file mode 100644 index 00000000000..32d2d962e4d --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTManagedConnectionPools.java @@ -0,0 +1,66 @@ +package com.yahoo.config.subscription.impl; + +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.vespa.config.JRTConnectionPool; +import com.yahoo.vespa.config.TimingValues; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; + +public class JRTManagedConnectionPools { + private static class JRTSourceThreadFactory implements ThreadFactory { + @Override + public Thread newThread(Runnable runnable) { + Thread t = new Thread(runnable, String.format("jrt-config-requester-%d", System.currentTimeMillis())); + // We want a daemon thread to avoid hanging threads in case something goes wrong in the config system + t.setDaemon(true); + return t; + } + } + private static class CountedPool { + long count; + final JRTConnectionPool pool; + final ScheduledThreadPoolExecutor scheduler = new ScheduledThreadPoolExecutor(1, new JRTSourceThreadFactory()); + CountedPool(JRTConnectionPool requester) { + this.pool = requester; + count = 0; + } + } + + private final Map<ConfigSourceSet, CountedPool> pools = new HashMap<>(); + + public JRTConfigRequester acquire(ConfigSourceSet sourceSet, TimingValues timingValues) { + CountedPool countedPool; + synchronized (pools) { + countedPool = pools.get(sourceSet); + if (countedPool == null) { + countedPool = new CountedPool(new JRTConnectionPool(sourceSet)); + pools.put(sourceSet, countedPool); + } + countedPool.count++; + } + return new JRTConfigRequester(sourceSet, countedPool.scheduler, countedPool.pool, timingValues); + } + + public synchronized void release(ConfigSourceSet sourceSet) { + CountedPool countedPool; + synchronized (pools) { + countedPool = pools.get(sourceSet); + if (countedPool != null) + countedPool.count--; + if (countedPool == null || countedPool.count > 0) return; + pools.remove(sourceSet); + } + + countedPool.pool.close(); + countedPool.scheduler.shutdown(); + try { + countedPool.scheduler.awaitTermination(30, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw new RuntimeException("Failed shutting down scheduler:", e); + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java index 97e85f1a39b..134352736b6 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java @@ -479,15 +479,11 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { } private void debug(String message) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, message); - } + log.log(LogLevel.DEBUG, () -> message); } private void trace(String message) { - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, message); - } + log.log(LogLevel.SPAM, () -> message); } private void printStack() { diff --git a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java index efeaacf225b..326c1287468 100644 --- a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java +++ b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java @@ -74,18 +74,14 @@ public class JRTConnectionPool implements ConnectionPool { public synchronized JRTConnection setNewCurrentConnection() { List<JRTConnection> sources = getSources(); currentConnection = sources.get(ThreadLocalRandom.current().nextInt(0, sources.size())); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Choosing new connection: " + currentConnection); - } + log.log(LogLevel.DEBUG, () -> "Choosing new connection: " + currentConnection); return currentConnection; } List<JRTConnection> getSources() { - List<JRTConnection> ret = new ArrayList<>(); + List<JRTConnection> ret; synchronized (connections) { - for (JRTConnection source : connections.values()) { - ret.add(source); - } + ret = new ArrayList<>(connections.values()); } return ret; } @@ -116,19 +112,6 @@ public class JRTConnectionPool implements ConnectionPool { return this; } - public String getAllSourceAddresses() { - StringBuilder sb = new StringBuilder(); - synchronized (connections) { - for (JRTConnection conn : connections.values()) { - sb.append(conn.getAddress()); - sb.append(","); - } - } - // Remove trailing "," - sb.deleteCharAt(sb.length() - 1); - return sb.toString(); - } - public String toString() { StringBuilder sb = new StringBuilder(); synchronized (connections) { diff --git a/config/src/main/java/com/yahoo/vespa/config/TimingValues.java b/config/src/main/java/com/yahoo/vespa/config/TimingValues.java index 780cf657009..5d5967e56c4 100644 --- a/config/src/main/java/com/yahoo/vespa/config/TimingValues.java +++ b/config/src/main/java/com/yahoo/vespa/config/TimingValues.java @@ -12,12 +12,11 @@ public class TimingValues { public static final long defaultNextConfigTimeout = 1000; // See getters below for an explanation of how these values are used and interpreted // All time values in milliseconds. - private long successTimeout = 600000; - private long errorTimeout = 20000; - private long initialTimeout = 15000; + private final long successTimeout; + private final long errorTimeout; + private final long initialTimeout; private long subscribeTimeout = 55000; private long configuredErrorTimeout = -1; // Don't ever timeout (and do not use error response) when we are already configured - private long nextConfigTimeout = defaultNextConfigTimeout; private long fixedDelay = 5000; private long unconfiguredDelay = 1000; @@ -26,6 +25,9 @@ public class TimingValues { private final Random rand; public TimingValues() { + successTimeout = 600000; + errorTimeout = 20000; + initialTimeout = 15000; this.rand = new Random(System.currentTimeMillis()); } @@ -100,20 +102,6 @@ public class TimingValues { } /** - * Returns initial timeout to use as server timeout when a config is requested for the first time. - * - * @return timeout in milliseconds. - */ - public long getInitialTimeout() { - return initialTimeout; - } - - public TimingValues setInitialTimeout(long t) { - initialTimeout = t; - return this; - } - - /** * Returns timeout to use as server timeout when subscribing for the first time. * * @return timeout in milliseconds. @@ -127,38 +115,12 @@ public class TimingValues { return this; } - /** - * Returns the time to retry getting config from the remote sources, until the next error response will - * be set as config. Counted from the last ok request was received. A negative value means that - * we will always retry getting config and never set an error response as config. - * - * @return timeout in milliseconds. - */ - public long getConfiguredErrorTimeout() { - return configuredErrorTimeout; - } - public TimingValues setConfiguredErrorTimeout(long t) { configuredErrorTimeout = t; return this; } /** - * Returns timeout used when calling {@link com.yahoo.config.subscription.ConfigSubscriber#nextConfig()} or - * {@link com.yahoo.config.subscription.ConfigSubscriber#nextGeneration()} - * - * @return timeout in milliseconds. - */ - public long getNextConfigTimeout() { - return nextConfigTimeout; - } - - public TimingValues setNextConfigTimeout(long t) { - nextConfigTimeout = t; - return this; - } - - /** * Returns time to wait until next attempt to get config after a failed request when the client has not * gotten a successful response to a config subscription (i.e, the client has not been configured). * A negative value means that there will never be a next attempt. If a negative value is set, the @@ -201,12 +163,6 @@ public class TimingValues { return maxDelayMultiplier; } - - public TimingValues setSuccessTimeout(long successTimeout) { - this.successTimeout = successTimeout; - return this; - } - /** * Returns fixed delay that is used when retrying getting config no matter if it was a success or an error * and independent of number of retries. @@ -228,10 +184,6 @@ public class TimingValues { return Math.round(val - (val * fraction) + (rand.nextFloat() * 2L * val * fraction)); } - Random getRandom() { - return rand; - } - @Override public String toString() { return "TimingValues [successTimeout=" + successTimeout diff --git a/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java index b7d9f9e14ca..863e95c5625 100644 --- a/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java +++ b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java @@ -47,7 +47,7 @@ public class UrlDownloader { Request request = new Request("frt.rpc.ping"); target.invokeSync(request, 5.0); if (! request.isError()) { - log.log(LogLevel.DEBUG, "Successfully connected to '" + spec + "', this = " + System.identityHashCode(this)); + log.log(LogLevel.DEBUG, () -> "Successfully connected to '" + spec + "', this = " + System.identityHashCode(this)); return; } else { target.close(); @@ -78,7 +78,7 @@ public class UrlDownloader { request.parameters().add(new StringValue(urlReference.value())); double rpcTimeout = Math.min(timeLeft, 60 * 60.0); - log.log(LogLevel.DEBUG, "InvokeSync waitFor " + urlReference + " with " + rpcTimeout + " seconds timeout"); + log.log(LogLevel.DEBUG, () -> "InvokeSync waitFor " + urlReference + " with " + rpcTimeout + " seconds timeout"); target.invokeSync(request, rpcTimeout); if (request.checkReturnTypes("s")) { diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java index 83befda7a59..3609ba04424 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java @@ -7,7 +7,6 @@ import com.yahoo.jrt.DataValue; import com.yahoo.jrt.Request; import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Value; -import com.yahoo.log.LogLevel; import com.yahoo.text.Utf8Array; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ErrorCode; @@ -92,9 +91,7 @@ public class JRTServerConfigRequestV3 implements JRTServerConfigRequest { } compressionInfo.serialize(jsonGenerator); jsonGenerator.writeEndObject(); - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, getConfigKey() + ": response dataXXXXX" + payload.withCompression(CompressionType.UNCOMPRESSED) + "XXXXX"); - } + jsonGenerator.writeEndObject(); jsonGenerator.close(); } catch (IOException e) { diff --git a/config/src/test/java/com/yahoo/config/subscription/BasicTest.java b/config/src/test/java/com/yahoo/config/subscription/BasicTest.java index 3b8b7db6487..5b145d40b7f 100644 --- a/config/src/test/java/com/yahoo/config/subscription/BasicTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/BasicTest.java @@ -1,13 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription; -import static org.junit.Assert.*; -import static org.hamcrest.CoreMatchers.is; import com.yahoo.foo.AppConfig; import org.junit.Test; +import static org.junit.Assert.assertEquals; + public class BasicTest { @@ -17,7 +17,8 @@ public class BasicTest { ConfigHandle<AppConfig> h = s.subscribe(AppConfig.class, "raw:times 0"); s.nextConfig(0); AppConfig c = h.getConfig(); - assertThat(c.times(), is(0)); + assertEquals(0, c.times()); + s.close(); } @Test @@ -26,6 +27,7 @@ public class BasicTest { ConfigHandle<AppConfig> h = s.subscribe(AppConfig.class, "raw:times 2"); s.nextGeneration(0); AppConfig c = h.getConfig(); - assertThat(c.times(), is(2)); + assertEquals(2, c.times()); + s.close(); } } diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java index 21cdfbe7d30..db30e7b7389 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java @@ -128,6 +128,7 @@ public class ConfigSetSubscriptionTest { assertEquals(hA0.getConfig().times(), 8800); assertEquals(hA1.getConfig().times(), 890); assertEquals(hS.getConfig().stringVal(), "new StringVal"); + subscriber.close(); } @Test diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java index 933a9fd130a..c8d4c081fc9 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java @@ -60,6 +60,7 @@ public class ConfigSubscriptionTest { assertEquals(c1, c1); assertNotEquals(c1, c2); + sub.close(); } @Test @@ -70,6 +71,7 @@ public class ConfigSubscriptionTest { sub.nextConfig(); assertTrue(handle.getConfig().boolval()); //assertTrue(sub.getSource() instanceof RawSource); + sub.close(); } // Test that subscription is closed and subscriptionHandles is empty if we get an exception diff --git a/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java b/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java index e9dc9cf7b98..9c83f2f3c9a 100644 --- a/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java @@ -49,14 +49,17 @@ public class GenericConfigSubscriberTest { public void testGenericRequesterPooling() { ConfigSourceSet source1 = new ConfigSourceSet("tcp/foo:78"); ConfigSourceSet source2 = new ConfigSourceSet("tcp/bar:79"); - JRTConfigRequester req1 = new JRTConfigRequester(new JRTConnectionPool(source1), JRTConfigRequesterTest.getTestTimingValues()); - JRTConfigRequester req2 = new JRTConfigRequester(new JRTConnectionPool(source2), JRTConfigRequesterTest.getTestTimingValues()); + JRTConfigRequester req1 = JRTConfigRequester.create(source1, JRTConfigRequesterTest.getTestTimingValues()); + JRTConfigRequester req2 = JRTConfigRequester.create(source2, JRTConfigRequesterTest.getTestTimingValues()); Map<ConfigSourceSet, JRTConfigRequester> requesters = new LinkedHashMap<>(); requesters.put(source1, req1); requesters.put(source2, req2); GenericConfigSubscriber sub = new GenericConfigSubscriber(requesters); assertEquals(sub.requesters().get(source1).getConnectionPool().getCurrent().getAddress(), "tcp/foo:78"); assertEquals(sub.requesters().get(source2).getConnectionPool().getCurrent().getAddress(), "tcp/bar:79"); + for (JRTConfigRequester requester : requesters.values()) { + requester.close(); + } } @Test(expected=UnsupportedOperationException.class) diff --git a/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java b/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java index 757dd99f43b..4211345dff7 100644 --- a/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java @@ -1,10 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription.impl; +import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.foo.SimpletypesConfig; import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.jrt.Request; import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConnectionPool; import com.yahoo.vespa.config.ErrorCode; import com.yahoo.vespa.config.ErrorType; import com.yahoo.vespa.config.TimingValues; @@ -17,6 +19,8 @@ import java.util.Random; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -349,4 +353,23 @@ public class JRTConfigRequesterTest { } } + @Test + public void testManagedPool() { + ConfigSourceSet sourceSet = ConfigSourceSet.createDefault(); + TimingValues timingValues = new TimingValues(); + JRTConfigRequester requester1 = JRTConfigRequester.create(sourceSet, timingValues); + JRTConfigRequester requester2 = JRTConfigRequester.create(sourceSet, timingValues); + assertNotSame(requester1, requester2); + assertSame(requester1.getConnectionPool(), requester2.getConnectionPool()); + ConnectionPool firstPool = requester1.getConnectionPool(); + requester1.close(); + requester2.close(); + requester1 = JRTConfigRequester.create(sourceSet, timingValues); + assertNotSame(firstPool, requester1.getConnectionPool()); + requester2 = JRTConfigRequester.create(new ConfigSourceSet("test-managed-pool-2"), timingValues); + assertNotSame(requester1.getConnectionPool(), requester2.getConnectionPool()); + requester1.close(); + requester2.close(); + } + } diff --git a/configdefinitions/src/vespa/attributes.def b/configdefinitions/src/vespa/attributes.def index f9db9eb2f0d..eb7dad88995 100644 --- a/configdefinitions/src/vespa/attributes.def +++ b/configdefinitions/src/vespa/attributes.def @@ -30,3 +30,8 @@ attribute[].densepostinglistthreshold double default=0.40 attribute[].tensortype string default="" # Whether this is an imported attribute (from parent document db) or not. attribute[].imported bool default=false + +# Configuration parameters for a hnsw index used together with a 1-dimensional indexed tensor for approximate nearest neighbor search. +attribute[].index.hnsw.enabled bool default=false +attribute[].index.hnsw.maxlinkspernode int default=16 +attribute[].index.hnsw.neighborstoexploreatinsert int default=200 diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java index 67549672408..2d5cce2d4f1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java @@ -34,7 +34,7 @@ public class TesterClient { public HttpResponse getLog(String testerHostname, int port, Long after) { URI testerUri; try { - testerUri = createBuilder(testerHostname, port, "/tester/v1/log2") + testerUri = createBuilder(testerHostname, port, "/tester/v1/log") .addParameter("after", String.valueOf(after)) .build(); } catch (URISyntaxException e) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java index 0894e38ce09..3f67d8e2cac 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java @@ -14,13 +14,15 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.function.BiConsumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; /** @@ -29,11 +31,13 @@ import static org.junit.Assert.*; public class ClusterMetricsRetrieverTest { @Rule - public final WireMockRule wireMock = new WireMockRule(options().port(8080), true); + public final WireMockRule wireMock = new WireMockRule(options().dynamicPort(), true); @Test public void testMetricAggregation() throws IOException { - List<URI> hosts = List.of(URI.create("http://localhost:8080/1"), URI.create("http://localhost:8080/2"), URI.create("http://localhost:8080/3")); + List<URI> hosts = Stream.of(1, 2, 3) + .map(item -> URI.create("http://localhost:" + wireMock.port() + "/" + item)) + .collect(Collectors.toList()); stubFor(get(urlEqualTo("/1")) .willReturn(aResponse() diff --git a/container-accesslogging/pom.xml b/container-accesslogging/pom.xml index 53c17257992..59dabba9efe 100644 --- a/container-accesslogging/pom.xml +++ b/container-accesslogging/pom.xml @@ -65,6 +65,11 @@ <artifactId>jackson-databind</artifactId> <scope>provided</scope> </dependency> + <dependency> + <groupId>org.assertj</groupId> + <artifactId>assertj-core</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> @@ -73,6 +78,10 @@ <artifactId>bundle-plugin</artifactId> <extensions>true</extensions> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> </plugins> <outputDirectory>${buildOutputDirectory}</outputDirectory> </build> diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java index 82c89276319..a3d34ae6a2c 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java @@ -11,7 +11,6 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; -import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.concurrent.ArrayBlockingQueue; @@ -41,7 +40,7 @@ public class LogFileHandler extends StreamHandler { private String filePattern = "./log.%T"; // default to current directory, ms time stamp private long nextRotationTime = 0; private FileOutputStream currentOutputStream = null; - private String fileName; + private volatile String fileName; private String symlinkName = null; private ArrayBlockingQueue<LogRecord> logQueue = new ArrayBlockingQueue<>(100000); private LogRecord rotateCmd = new LogRecord(Level.SEVERE, "rotateNow"); diff --git a/container-accesslogging/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java b/container-accesslogging/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java index de4823e66c6..d7361eec488 100644 --- a/container-accesslogging/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java +++ b/container-accesslogging/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java @@ -2,7 +2,9 @@ package com.yahoo.container.logging; import com.yahoo.io.IOUtils; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.FileInputStream; @@ -17,24 +19,23 @@ import java.util.logging.LogRecord; import java.util.logging.SimpleFormatter; import java.util.zip.GZIPInputStream; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.assertj.core.api.Assertions.assertThat; /** - * @author <a href="mailto:travisb@yahoo-inc.com">Bob Travis</a> + * @author Bob Travis + * @author bjorncs */ -// TODO: Make these tests wait until the right things happen rather than waiting for a predetermined time -// These tests take too long, and are not cleaning up properly. See how this should be done in YApacheLogTestCase public class LogFileHandlerTestCase { - /** - * The scenario - */ + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Test - public void testIt() { + public void testIt() throws IOException { + File root = temporaryFolder.newFolder("logfilehandlertest"); + LogFileHandler h = new LogFileHandler(); - h.setFilePattern("./logfilehandlertest.%Y%m%d%H%M%S"); + h.setFilePattern(root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S"); h.setFormatter(new Formatter() { public String format(LogRecord r) { DateFormat df = new SimpleDateFormat("yyyy.MM.dd:HH:mm:ss.SSS"); @@ -46,42 +47,27 @@ public class LogFileHandlerTestCase { long millisPerDay = 60*60*24*1000; long tomorrowDays = (now / millisPerDay) +1; long tomorrowMillis = tomorrowDays * millisPerDay; - assertEquals (tomorrowMillis, h.getNextRotationTime(now)); + assertThat(tomorrowMillis).isEqualTo(h.getNextRotationTime(now)); long[] rTimes = {1000, 2000, 10000}; h.setRotationTimes(rTimes); - assertEquals (tomorrowMillis+1000, h.getNextRotationTime(tomorrowMillis)); - assertEquals (tomorrowMillis+10000, h.getNextRotationTime(tomorrowMillis+3000)); - boolean okToWrite = false; // don't want regular unit tests to create tiles.... - if (okToWrite) { - LogRecord lr = new LogRecord(Level.INFO, "test"); - h.publish(lr); - h.publish(new LogRecord(Level.INFO, "another test")); - h.rotateNow(); - h.publish(lr); - h.flush(); - } - } - - private boolean delete(String fileOrDir) { - File file = new File(fileOrDir); - return file.delete(); - } - - private void deleteOnExit(String fileOrDir) { - new File(fileOrDir).deleteOnExit(); - } - - private static void deleteRecursive(String directory) { - IOUtils.recursiveDeleteDir(new File(directory)); + assertThat(tomorrowMillis+1000).isEqualTo(h.getNextRotationTime(tomorrowMillis)); + assertThat(tomorrowMillis+10000).isEqualTo(h.getNextRotationTime(tomorrowMillis+3000)); + LogRecord lr = new LogRecord(Level.INFO, "test"); + h.publish(lr); + h.publish(new LogRecord(Level.INFO, "another test")); + h.rotateNow(); + h.publish(lr); + h.flush(); + h.shutdown(); } @Test - public void testSimpleLogging() { - String logFilePattern = "./testLogFileG1.txt"; + public void testSimpleLogging() throws IOException { + File logFile = temporaryFolder.newFile("testLogFileG1.txt"); //create logfilehandler LogFileHandler h = new LogFileHandler(); - h.setFilePattern(logFilePattern); + h.setFilePattern(logFile.getAbsolutePath()); h.setFormatter(new SimpleFormatter()); h.setRotationTimes("0 5 ..."); @@ -89,17 +75,16 @@ public class LogFileHandlerTestCase { LogRecord lr = new LogRecord(Level.INFO, "testDeleteFileFirst1"); h.publish(lr); h.flush(); - - new File(logFilePattern).deleteOnExit(); + h.shutdown(); } @Test - public void testDeleteFileDuringLogging() { - String logFilePattern = "./testLogFileG2.txt"; + public void testDeleteFileDuringLogging() throws IOException { + File logFile = temporaryFolder.newFile("testLogFileG2.txt"); //create logfilehandler LogFileHandler h = new LogFileHandler(); - h.setFilePattern(logFilePattern); + h.setFilePattern(logFile.getAbsolutePath()); h.setFormatter(new SimpleFormatter()); h.setRotationTimes("0 5 ..."); @@ -109,20 +94,20 @@ public class LogFileHandlerTestCase { h.flush(); //delete log file - delete(logFilePattern); + logFile.delete(); //write log again lr = new LogRecord(Level.INFO, "testDeleteFileDuringLogging2"); h.publish(lr); h.flush(); - - new File(logFilePattern).deleteOnExit(); + h.shutdown(); } @Test - public void testSymlink() { + public void testSymlink() throws IOException, InterruptedException { + File root = temporaryFolder.newFolder("testlogforsymlinkchecking"); LogFileHandler h = new LogFileHandler(); - h.setFilePattern("./testlogforsymlinkchecking/logfilehandlertest.%Y%m%d%H%M%S%s"); + h.setFilePattern(root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S%s"); h.setFormatter(new Formatter() { public String format(LogRecord r) { DateFormat df = new SimpleDateFormat("yyyy.MM.dd:HH:mm:ss.SSS"); @@ -135,50 +120,43 @@ public class LogFileHandlerTestCase { h.publish(lr); String f1 = h.getFileName(); String f2 = null; - try { - while (f1 == null) { - Thread.sleep(1); - f1 = h.getFileName(); - } - h.rotateNow(); + while (f1 == null) { + Thread.sleep(1); + f1 = h.getFileName(); + } + h.rotateNow(); + Thread.sleep(1); + f2 = h.getFileName(); + while (f1.equals(f2)) { Thread.sleep(1); f2 = h.getFileName(); - while (f1.equals(f2)) { - Thread.sleep(1); - f2 = h.getFileName(); - } - lr = new LogRecord(Level.INFO, "string which is way longer than the word test"); - h.publish(lr); - Thread.sleep(1000); - File f = new File(f1); - long first = f.length(); - f = new File(f2); - long second = f.length(); - final long secondLength = 72; - for (int n = 0; n < 20 && second != secondLength; ++n) { - Thread.sleep(1000); - second = f.length(); - } - f = new File("./testlogforsymlinkchecking", "symlink"); - long link = f.length(); - assertEquals(secondLength, link); - assertEquals(31, first); - assertEquals(secondLength, second); - } catch (InterruptedException e) { - // just let the test pass } - deleteOnExit("./testlogforsymlinkchecking"); - deleteOnExit("./testlogforsymlinkchecking/symlink"); - deleteOnExit(f1); - if (f2 != null) - deleteOnExit(f2); + lr = new LogRecord(Level.INFO, "string which is way longer than the word test"); + h.publish(lr); + h.waitDrained(); + File f = new File(f1); + long first = f.length(); + f = new File(f2); + long second = f.length(); + final long secondLength = 72; + for (int n = 0; n < 20 && second != secondLength; ++n) { + Thread.sleep(1); + second = f.length(); + } + f = new File(root, "symlink"); + long link = f.length(); + assertThat(secondLength).isEqualTo(link); + assertThat(31).isEqualTo(first); + assertThat(secondLength).isEqualTo(second); + h.shutdown(); } @Test public void testcompression() throws InterruptedException, IOException { - IOUtils.recursiveDeleteDir(new File("./testcompression")); + File root = temporaryFolder.newFolder("testcompression"); + LogFileHandler h = new LogFileHandler(true); - h.setFilePattern("./testcompression/logfilehandlertest.%Y%m%d%H%M%S%s"); + h.setFilePattern(root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S%s"); h.setFormatter(new Formatter() { public String format(LogRecord r) { DateFormat df = new SimpleDateFormat("yyyy.MM.dd:HH:mm:ss.SSS"); @@ -186,28 +164,28 @@ public class LogFileHandlerTestCase { return ("["+timeStamp+"]" + " " + formatMessage(r) + "\n"); } } ); - for (int i=0; i < 10000; i++) { + int logEntries = 10000; + for (int i = 0; i < logEntries; i++) { LogRecord lr = new LogRecord(Level.INFO, "test"); h.publish(lr); } h.waitDrained(); String f1 = h.getFileName(); - assertTrue(f1.startsWith("./testcompression/logfilehandlertest.")); + assertThat(f1).startsWith(root.getAbsolutePath() + "/logfilehandlertest."); File uncompressed = new File(f1); File compressed = new File(f1 + ".gz"); - assertTrue(uncompressed.exists()); - assertFalse(compressed.exists()); + assertThat(uncompressed).exists(); + assertThat(compressed).doesNotExist(); String content = IOUtils.readFile(uncompressed); - assertEquals(310000, content.length()); + assertThat(content).hasLineCount(logEntries); h.rotateNow(); while (uncompressed.exists()) { - Thread.sleep(10); + Thread.sleep(1); } - assertTrue(compressed.exists()); + assertThat(compressed).exists(); String unzipped = IOUtils.readAll(new InputStreamReader(new GZIPInputStream(new FileInputStream(compressed)))); - assertEquals(content, unzipped); - - IOUtils.recursiveDeleteDir(new File("./testcompression")); + assertThat(content).isEqualTo(unzipped); + h.shutdown(); } } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java index bfcecd61fa4..6c8cee8433c 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java @@ -17,6 +17,7 @@ import com.yahoo.jdisc.handler.ResponseDispatch; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.log.LogLevel; +import java.net.URI; import java.time.Duration; import java.util.HashMap; import java.util.Map; @@ -91,6 +92,11 @@ public abstract class ThreadedRequestHandler extends AbstractRequestHandler { if (endpoint != null) { dimensions.put("endpoint", endpoint); } + URI uri = request.getUri(); + dimensions.put("scheme", uri.getScheme()); + dimensions.put("port", Integer.toString(uri.getPort())); + String handlerClassName = getClass().getName(); + dimensions.put("handler-name", handlerClassName); return this.metric.createContext(dimensions); } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java index fbd7b62b718..3a153ec3d8a 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java @@ -19,6 +19,8 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; +import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.INFO; import static java.util.logging.Level.SEVERE; import static java.util.logging.Level.WARNING; @@ -50,11 +52,11 @@ public class Deconstructor implements ComponentDeconstructor { } } else if (component instanceof Provider) { // TODO Providers should most likely be deconstructed similarly to AbstractComponent - log.log(LogLevel.DEBUG, "Starting deconstruction of provider " + component); + log.log(DEBUG, () -> "Starting deconstruction of provider " + component); ((Provider<?>) component).deconstruct(); - log.log(LogLevel.DEBUG, "Finished deconstruction of provider " + component); + log.log(DEBUG, () -> "Finished deconstruction of provider " + component); } else if (component instanceof SharedResource) { - log.log(LogLevel.DEBUG, "Releasing container reference to resource " + component); + log.log(DEBUG, () -> "Releasing container reference to resource " + component); // No need to delay release, as jdisc does ref-counting ((SharedResource) component).release(); } @@ -87,10 +89,10 @@ public class Deconstructor implements ComponentDeconstructor { @Override public void run() { for (var component : components) { - log.info("Starting deconstruction of component " + component); + log.log(DEBUG, () -> "Starting deconstruction of component " + component); try { component.deconstruct(); - log.log(LogLevel.INFO, "Finished deconstructing of component " + component); + log.log(DEBUG, () -> "Finished deconstructing of component " + component); } catch (Exception | NoClassDefFoundError e) { // May get class not found due to it being already unloaded log.log(WARNING, "Exception thrown when deconstructing component " + component, e); } catch (Error e) { @@ -110,7 +112,7 @@ public class Deconstructor implements ComponentDeconstructor { // It should now be safe to uninstall the old bundles. for (var bundle : bundles) { try { - log.log(LogLevel.INFO, "Uninstalling bundle " + bundle); + log.log(INFO, "Uninstalling bundle " + bundle); bundle.uninstall(); } catch (BundleException e) { log.log(SEVERE, "Could not uninstall bundle " + bundle); diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh index 382843b5688..af429d56a75 100755 --- a/container-disc/src/main/sh/vespa-start-container-daemon.sh +++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh @@ -64,6 +64,7 @@ configure_memory() { consider_fallback jvm_heapsize 1536 consider_fallback jvm_stacksize 512 consider_fallback jvm_baseMaxDirectMemorySize 75 + consider_fallback jvm_compressedClassSpaceSize 32 consider_fallback jvm_directMemorySizeCache 0 # Update jvm_heapsize only if percentage is explicitly set (default is 0). @@ -80,16 +81,20 @@ configure_memory() { fi # Safety measure against bad min vs max heapsize. - if ((jvm_minHeapsize > jvm_heapsize)); then + if ((jvm_minHeapsize > jvm_heapsize)); then jvm_minHeapsize=${jvm_heapsize} echo "Misconfigured heap size, jvm_minHeapsize(${jvm_minHeapsize} is larger than jvm_heapsize(${jvm_heapsize}). It has been capped." - fi + fi maxDirectMemorySize=$(( jvm_baseMaxDirectMemorySize + jvm_heapsize / 8 + jvm_directMemorySizeCache )) memory_options="-Xms${jvm_minHeapsize}m -Xmx${jvm_heapsize}m" memory_options="${memory_options} -XX:ThreadStackSize=${jvm_stacksize}" - memory_options="${memory_options} -XX:MaxDirectMemorySize=${maxDirectMemorySize}m" + memory_options="${memory_options} -XX:MaxDirectMemorySize=${maxDirectMemorySize}m" + + if ((jvm_compressedClassSpaceSize != 0)); then + memory_options="${memory_options} -XX:CompressedClassSpaceSize=${jvm_compressedClassSpaceSize}m" + fi if [ "${VESPA_USE_HUGEPAGES}" ]; then memory_options="${memory_options} -XX:+UseLargePages" diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 24ecea9c3ee..423de07f8c4 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -18,7 +18,6 @@ import com.yahoo.search.dispatch.rpc.RpcPingFactory; import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.dispatch.searchcluster.Group; import com.yahoo.search.dispatch.searchcluster.Node; -import com.yahoo.search.dispatch.searchcluster.PingFactory; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.FieldType; @@ -85,21 +84,13 @@ public class Dispatcher extends AbstractComponent { public static QueryProfileType getArgumentType() { return argumentType; } @Inject - public Dispatcher(ComponentId clusterId, + public Dispatcher(RpcResourcePool resourcePool, + ComponentId clusterId, DispatchConfig dispatchConfig, ClusterInfoConfig clusterInfoConfig, VipStatus vipStatus, Metric metric) { - this(new RpcResourcePool(dispatchConfig), clusterId, dispatchConfig, clusterInfoConfig, vipStatus, metric); - } - - private Dispatcher(RpcResourcePool resourcePool, - ComponentId clusterId, - DispatchConfig dispatchConfig, - ClusterInfoConfig clusterInfoConfig, - VipStatus vipStatus, - Metric metric) { - this(resourcePool, new SearchCluster(clusterId.stringValue(), dispatchConfig, clusterInfoConfig.nodeCount(), + this(resourcePool, new SearchCluster(clusterId.stringValue(), dispatchConfig,clusterInfoConfig.nodeCount(), vipStatus, new RpcPingFactory(resourcePool)), dispatchConfig, metric); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java index a45ec59c3ee..74bc9e8bfbb 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java @@ -1,25 +1,19 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch.rpc; -import com.yahoo.prelude.Pong; import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; -import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.dispatch.FillInvoker; import com.yahoo.search.dispatch.InvokerFactory; import com.yahoo.search.dispatch.SearchInvoker; import com.yahoo.search.dispatch.searchcluster.Node; -import com.yahoo.search.dispatch.searchcluster.PingFactory; -import com.yahoo.search.dispatch.searchcluster.Pinger; -import com.yahoo.search.dispatch.searchcluster.PongHandler; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import java.util.Optional; -import java.util.concurrent.Callable; /** * @author ollivir @@ -62,7 +56,4 @@ public class RpcInvokerFactory extends InvokerFactory { return new RpcFillInvoker(rpcResourcePool, documentDb); } - public void release() { - rpcResourcePool.release(); - } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java index ca2a0c9bfb0..065489ef9a0 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java @@ -2,6 +2,9 @@ package com.yahoo.search.dispatch.rpc; import com.google.common.collect.ImmutableMap; +import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.ComponentId; import com.yahoo.compress.CompressionType; import com.yahoo.compress.Compressor; import com.yahoo.compress.Compressor.Compression; @@ -23,7 +26,7 @@ import java.util.Random; * * @author ollivir */ -public class RpcResourcePool { +public class RpcResourcePool extends AbstractComponent { /** The compression method which will be used with rpc dispatch. "lz4" (default) and "none" is supported. */ public final static CompoundName dispatchCompression = new CompoundName("dispatch.compression"); @@ -33,13 +36,15 @@ public class RpcResourcePool { /** Connections to the search nodes this talks to, indexed by node id ("partid") */ private final ImmutableMap<Integer, NodeConnectionPool> nodeConnectionPools; - public RpcResourcePool(Map<Integer, NodeConnection> nodeConnections) { + RpcResourcePool(Map<Integer, NodeConnection> nodeConnections) { var builder = new ImmutableMap.Builder<Integer, NodeConnectionPool>(); nodeConnections.forEach((key, connection) -> builder.put(key, new NodeConnectionPool(Collections.singletonList(connection)))); this.nodeConnectionPools = builder.build(); } + @Inject public RpcResourcePool(DispatchConfig dispatchConfig) { + super(); var client = new RpcClient(dispatchConfig.numJrtTransportThreads()); // Create rpc node connection pools indexed by the node distribution key @@ -73,7 +78,9 @@ public class RpcResourcePool { } } - public void release() { + @Override + public void deconstruct() { + super.deconstruct(); nodeConnectionPools.values().forEach(NodeConnectionPool::release); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java index 07d8439ff46..76240e55c98 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java @@ -5,7 +5,6 @@ import com.yahoo.compress.CompressionType; import com.yahoo.compress.Compressor; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; -import com.yahoo.search.Result; import com.yahoo.search.dispatch.InvokerResult; import com.yahoo.search.dispatch.SearchInvoker; import com.yahoo.search.dispatch.rpc.Client.ProtobufResponse; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 1da13f6c082..7619cb34b77 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -83,6 +83,13 @@ public class SearchCluster implements NodeManager<Node> { nodesByHost, groups); } + + /* Testing only */ + public SearchCluster(String clusterId, DispatchConfig dispatchConfig, + VipStatus vipStatus, PingFactory pingFactory) { + this(clusterId, dispatchConfig, 1, vipStatus, pingFactory); + } + public void addMonitoring(ClusterMonitor clusterMonitor) { for (var group : orderedGroups) { for (var node : group.nodes()) diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java index d0b3189a79c..421544b5b49 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java @@ -78,7 +78,6 @@ public class FederationSearcher extends ForkingSearcher { /** The name of the query property containing the source name added to the query to each source by this */ public final static CompoundName SOURCENAME = new CompoundName("sourceName"); public final static CompoundName PROVIDERNAME = new CompoundName("providerName"); - /** Logging field name constants */ public static final String LOG_COUNT_PREFIX = "count_"; diff --git a/container-search/src/main/resources/configdefinitions/qr-start.def b/container-search/src/main/resources/configdefinitions/qr-start.def index 031877ada81..95e9d4575dd 100644 --- a/container-search/src/main/resources/configdefinitions/qr-start.def +++ b/container-search/src/main/resources/configdefinitions/qr-start.def @@ -20,6 +20,9 @@ jvm.minHeapsize int default=1536 restart ## Stack size (in kilobytes) jvm.stacksize int default=512 restart +## CompressedOOps size in megabytes +jvm.compressedClassSpaceSize int default=32 restart + ## Base value of maximum direct memory size (in megabytes) jvm.baseMaxDirectMemorySize int default=75 restart diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index 5b6a4b68930..8b7a57c38e7 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -22,6 +22,7 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.config.ClusterConfig; import com.yahoo.search.dispatch.Dispatcher; +import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; import com.yahoo.vespa.config.search.DispatchConfig; @@ -514,8 +515,10 @@ public class ClusterSearcherTestCase { DocumentdbInfoConfig.Builder documentDbConfig = new DocumentdbInfoConfig.Builder(); documentDbConfig.documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name("type1")); - Dispatcher dispatcher = new Dispatcher(new ComponentId("test-id"), - new DispatchConfig.Builder().build(), + DispatchConfig dispatchConfig = new DispatchConfig.Builder().build(); + Dispatcher dispatcher = new Dispatcher(new RpcResourcePool(dispatchConfig), + ComponentId.createAnonymousComponentId("test-id"), + dispatchConfig, createClusterInfoConfig(), vipStatus, new MockMetric()); diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index 0f49f6029ad..63475c9c189 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -14,7 +14,6 @@ import com.yahoo.prelude.fastsearch.SummaryParameters; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; -import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.grouping.GroupingRequest; @@ -151,7 +150,7 @@ public class FastSearcherTestCase { VipStatus vipStatus = new VipStatus(b.build()); List<Node> nodes_1 = ImmutableList.of(new Node(0, "host0", 0)); RpcResourcePool rpcPool_1 = new RpcResourcePool(MockDispatcher.toDispatchConfig(nodes_1)); - MockDispatcher dispatch_1 = MockDispatcher.create(nodes_1, rpcPool_1, 1, vipStatus); + MockDispatcher dispatch_1 = MockDispatcher.create(nodes_1, rpcPool_1, vipStatus); dispatch_1.clusterMonitor.shutdown(); vipStatus.addToRotation(clusterName); assertTrue(vipStatus.isInRotation()); diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java index 707b051c03b..2a2c8410b2c 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java @@ -20,13 +20,12 @@ class MockDispatcher extends Dispatcher { public static MockDispatcher create(List<Node> nodes) { var rpcResourcePool = new RpcResourcePool(toDispatchConfig(nodes)); - return create(nodes, rpcResourcePool, 1, new VipStatus()); + return create(nodes, rpcResourcePool, new VipStatus()); } - public static MockDispatcher create(List<Node> nodes, RpcResourcePool rpcResourcePool, - int containerClusterSize, VipStatus vipStatus) { + public static MockDispatcher create(List<Node> nodes, RpcResourcePool rpcResourcePool, VipStatus vipStatus) { var dispatchConfig = toDispatchConfig(nodes); - var searchCluster = new SearchCluster("a", dispatchConfig, containerClusterSize, vipStatus, new RpcPingFactory(rpcResourcePool)); + var searchCluster = new SearchCluster("a", dispatchConfig, vipStatus, new RpcPingFactory(rpcResourcePool)); return new MockDispatcher(new ClusterMonitor<>(searchCluster, true), searchCluster, dispatchConfig, rpcResourcePool); } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java index 6eedb8239a9..36b476e2936 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java @@ -29,7 +29,7 @@ public class LoadBalancerTest { @Test public void requireThatLoadBalancerServesSingleNodeSetups() { Node n1 = new Node(0, "test-node1", 0); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1), 1, null, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1), null, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroup(null); @@ -43,7 +43,7 @@ public class LoadBalancerTest { public void requireThatLoadBalancerServesMultiGroupSetups() { Node n1 = new Node(0, "test-node1", 0); Node n2 = new Node(1, "test-node2", 1); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), null, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroup(null); @@ -59,7 +59,7 @@ public class LoadBalancerTest { Node n2 = new Node(1, "test-node2", 0); Node n3 = new Node(0, "test-node3", 1); Node n4 = new Node(1, "test-node4", 1); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2, n3, n4), 2, null, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2, n3, n4), null, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroup(null); @@ -70,7 +70,7 @@ public class LoadBalancerTest { public void requireThatLoadBalancerReturnsDifferentGroups() { Node n1 = new Node(0, "test-node1", 0); Node n2 = new Node(1, "test-node2", 1); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null,null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), null,null); LoadBalancer lb = new LoadBalancer(cluster, true); // get first group diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index 3b4d58cdfc2..32c6738fc3b 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -30,7 +30,7 @@ public class MockSearchCluster extends SearchCluster { } public MockSearchCluster(String clusterId, DispatchConfig dispatchConfig, int groups, int nodesPerGroup) { - super(clusterId, dispatchConfig, 1, null, null); + super(clusterId, dispatchConfig, null, null); ImmutableList.Builder<Group> orderedGroupBuilder = ImmutableList.builder(); ImmutableMap.Builder<Integer, Group> groupBuilder = ImmutableMap.builder(); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java index 766f9ea6c2d..cf90a1c6d81 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java @@ -14,7 +14,6 @@ import org.junit.Test; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceInformation.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceInformation.java index 5c279547e17..d33f9a45c82 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceInformation.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceInformation.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.api.application.v4.model; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.vespa.hosted.controller.api.identifiers.GitBranch; import com.yahoo.vespa.hosted.controller.api.identifiers.GitCommit; import com.yahoo.vespa.hosted.controller.api.identifiers.GitRepository; @@ -35,23 +36,31 @@ public class InstanceInformation { public String cluster; public boolean tls; public URI url; + public String scope; + public RoutingMethod routingMethod; @JsonCreator public Endpoint(@JsonProperty("cluster") String cluster , @JsonProperty("tls") boolean tls, - @JsonProperty("url") URI url) { + @JsonProperty("url") URI url, + @JsonProperty("scope") String scope, + @JsonProperty("routingMethod") RoutingMethod routingMethod) { this.cluster = cluster; this.tls = tls; this.url = url; + this.scope = scope; + this.routingMethod = routingMethod; } @Override public String toString() { - return "Endpoint {" + - "cluster=" + cluster+ - ", tls='" + tls + '\'' + - ", url='" + url+ '\'' + - '}'; + return "Endpoint{" + + "cluster='" + cluster + '\'' + + ", tls=" + tls + + ", url=" + url + + ", scope='" + scope + '\'' + + ", routingMethod=" + routingMethod + + '}'; } } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java index 0aa0df8ae2b..171c5caa756 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java @@ -18,25 +18,23 @@ public class EndpointCertificateMetadata { private final int version; private final Optional<String> request_id; private final Optional<List<String>> requestedDnsSans; + private final Optional<String> issuer; public EndpointCertificateMetadata(String keyName, String certName, int version) { - this.keyName = keyName; - this.certName = certName; - this.version = version; - this.request_id = Optional.empty(); - this.requestedDnsSans = Optional.empty(); + this(keyName, certName, version, Optional.empty(), Optional.empty(), Optional.empty()); + } + + public EndpointCertificateMetadata(String keyName, String certName, int version, String request_id, List<String> requestedDnsSans) { + this(keyName, certName, version, Optional.of(request_id), Optional.of(requestedDnsSans), Optional.empty()); } - public EndpointCertificateMetadata(String keyName, String certName, int version, Optional<String> request_id, Optional<List<String>> requestedDnsSans) { + public EndpointCertificateMetadata(String keyName, String certName, int version, Optional<String> request_id, Optional<List<String>> requestedDnsSans, Optional<String> issuer) { this.keyName = keyName; this.certName = certName; this.version = version; this.request_id = request_id; this.requestedDnsSans = requestedDnsSans; - } - - public EndpointCertificateMetadata(String keyName, String certName, int version, String request_id, List<String> requestedDnsSans) { - this(keyName, certName, version, Optional.of(request_id), Optional.of(requestedDnsSans)); + this.issuer = issuer; } public String keyName() { @@ -59,6 +57,10 @@ public class EndpointCertificateMetadata { return requestedDnsSans; } + public Optional<String> issuer() { + return issuer; + } + @Override public String toString() { return "EndpointCertificateMetadata{" + @@ -67,6 +69,7 @@ public class EndpointCertificateMetadata { ", version=" + version + ", request_id=" + request_id + ", requestedDnsSans=" + requestedDnsSans + + ", issuer=" + issuer + '}'; } @@ -79,11 +82,12 @@ public class EndpointCertificateMetadata { keyName.equals(that.keyName) && certName.equals(that.certName) && request_id.equals(that.request_id) && - requestedDnsSans.equals(that.requestedDnsSans); + requestedDnsSans.equals(that.requestedDnsSans) && + issuer.equals(that.issuer); } @Override public int hashCode() { - return Objects.hash(keyName, certName, version, request_id, requestedDnsSans); + return Objects.hash(keyName, certName, version, request_id, requestedDnsSans, issuer); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java index 8e81400f3c8..c38ea158507 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java @@ -7,6 +7,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; /** @@ -21,7 +22,7 @@ public class EndpointCertificateMock implements EndpointCertificateProvider { } @Override - public EndpointCertificateMetadata requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames) { + public EndpointCertificateMetadata requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames, Optional<EndpointCertificateMetadata> currentMetadata) { this.dnsNames.put(applicationId, dnsNames); String endpointCertificatePrefix = String.format("vespa.tls.%s.%s@%s", applicationId.tenant(), applicationId.application(), diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java index 97d2bdb3343..9c5c25c1c71 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.certificates; import com.yahoo.config.provision.ApplicationId; import java.util.List; +import java.util.Optional; /** * Generates an endpoint certificate for an application instance. @@ -12,7 +13,7 @@ import java.util.List; */ public interface EndpointCertificateProvider { - EndpointCertificateMetadata requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames); + EndpointCertificateMetadata requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames, Optional<EndpointCertificateMetadata> currentMetadata); List<EndpointCertificateMetadata> listCertificates(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 67a6faac606..4c91238453a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -179,8 +179,7 @@ enum PathGroup { "/application/v4/tenant/"), /** Paths which contain (not very strictly) classified information about, e.g., customers. */ - classifiedInfo("/cost/v1/{*}", - "/", + classifiedInfo("/", "/d/{*}"), /** Same as classifiedInfo, but with optional /api prefix */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 74304f2e49d..cf272d94dcd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -237,7 +237,8 @@ public class RoutingController { } } - private boolean supportsRoutingMethod(RoutingMethod routingMethod, ZoneId zone) { + /** Returns whether given routingMethod is supported by zone */ + public boolean supportsRoutingMethod(RoutingMethod routingMethod, ZoneId zone) { return controller.zoneRegistry().zones().routingMethod(routingMethod).ids().contains(zone); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java index 3994e05cbac..083984bd13c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java @@ -79,7 +79,7 @@ public class ApplicationPackage { Optional<Inspector> buildMetaObject = files.get("build-meta.json").map(SlimeUtils::jsonToSlime).map(Slime::get); if (requireFiles && buildMetaObject.isEmpty()) - throw new IllegalArgumentException("Missing required file 'deployment.xml'"); + throw new IllegalArgumentException("Missing required file 'build-meta.json'"); this.compileVersion = buildMetaObject.flatMap(object -> parse(object, "compileVersion", field -> Version.fromString(field.asString()))); this.buildTime = buildMetaObject.flatMap(object -> parse(object, "buildTime", field -> Instant.ofEpochMilli(field.asLong()))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 060ffd63fb3..4d09765f78b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -28,7 +28,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; @@ -38,6 +37,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentFailureMails; +import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; @@ -115,6 +115,7 @@ public class InternalStepRunner implements StepRunner { new NodeResources(2, 8, 50, 0.3, NodeResources.DiskSpeed.any); static final Duration endpointTimeout = Duration.ofMinutes(15); + static final Duration endpointCertificateTimeout = Duration.ofMinutes(15); static final Duration testerTimeout = Duration.ofMinutes(30); static final Duration installationTimeout = Duration.ofMinutes(60); static final Duration certificateTimeout = Duration.ofMinutes(300); @@ -273,9 +274,14 @@ public class InternalStepRunner implements StepRunner { Optional<RunStatus> result = startTime.isBefore(controller.clock().instant().minus(Duration.ofHours(1))) ? Optional.of(deploymentFailed) : Optional.empty(); switch (e.getErrorCode()) { + case CERTIFICATE_NOT_READY: + if (startTime.plus(endpointCertificateTimeout).isBefore(controller.clock().instant())) { + logger.log("Deployment failed to find provisioned endpoint certificate after " + endpointCertificateTimeout); + return Optional.of(RunStatus.endpointCertificateTimeout); + } + return result; case ACTIVATION_CONFLICT: case APPLICATION_LOCK_FAILURE: - case CERTIFICATE_NOT_READY: logger.log("Deployment failed with possibly transient error " + e.getErrorCode() + ", will retry: " + e.getMessage()); return result; @@ -665,22 +671,36 @@ public class InternalStepRunner implements StepRunner { return; try { - if (run.status() == outOfCapacity && run.id().type().isProduction()) - controller.serviceRegistry().mailer().send(mails.outOfCapacity(run.id(), recipients)); - if (run.status() == deploymentFailed) - controller.serviceRegistry().mailer().send(mails.deploymentFailure(run.id(), recipients)); - if (run.status() == installationFailed) - controller.serviceRegistry().mailer().send(mails.installationFailure(run.id(), recipients)); - if (run.status() == testFailure) - controller.serviceRegistry().mailer().send(mails.testFailure(run.id(), recipients)); - if (run.status() == error) - controller.serviceRegistry().mailer().send(mails.systemError(run.id(), recipients)); + mailOf(run, recipients).ifPresent(controller.serviceRegistry().mailer()::send); } catch (RuntimeException e) { logger.log(INFO, "Exception trying to send mail for " + run.id(), e); } } + private Optional<Mail> mailOf(Run run, List<String> recipients) { + switch (run.status()) { + case running: + case aborted: + case success: + return Optional.empty(); + case outOfCapacity: + return run.id().type().isProduction() ? Optional.of(mails.outOfCapacity(run.id(), recipients)) : Optional.empty(); + case deploymentFailed: + return Optional.of(mails.deploymentFailure(run.id(), recipients)); + case installationFailed: + return Optional.of(mails.installationFailure(run.id(), recipients)); + case testFailure: + return Optional.of(mails.testFailure(run.id(), recipients)); + case error: + case endpointCertificateTimeout: + return Optional.of(mails.systemError(run.id(), recipients)); + default: + logger.log(WARNING, "Don't know what mail to send for run status '" + run.status() + "'"); + return Optional.of(mails.systemError(run.id(), recipients)); + } + } + /** Returns the deployment of the real application in the zone of the given job, if it exists. */ private Optional<Deployment> deployment(ApplicationId id, JobType type) { return Optional.ofNullable(application(id).deployments().get(type.zone(controller.system()))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index d8d9431dc22..be189004f6d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -73,8 +73,8 @@ import static java.util.stream.Collectors.toUnmodifiableMap; */ public class JobController { - private static final int historyLength = 64; - private static final Duration maxHistoryAge = Duration.ofDays(60); + public static final int historyLength = 64; + public static final Duration maxHistoryAge = Duration.ofDays(60); private final Controller controller; private final CuratorDb curator; @@ -356,7 +356,9 @@ public class JobController { old = oldEntries.next()) { // Make sure we keep the last success and the first failing - if (successes == 1 && old.getValue().status() == RunStatus.success) { + if ( successes == 1 + && old.getValue().status() == RunStatus.success + && ! old.getValue().start().isBefore(controller.clock().instant().minus(maxHistoryAge))) { oldEntries.next(); continue; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java index 5df914bad80..80924c3c0aa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java @@ -16,6 +16,7 @@ public class JobMetrics { public static final String start = "deployment.start"; public static final String outOfCapacity = "deployment.outOfCapacity"; + public static final String endpointCertificateTimeout = "deployment.endpointCertificateTimeout"; public static final String deploymentFailure = "deployment.deploymentFailure"; public static final String convergenceFailure = "deployment.convergenceFailure"; public static final String testFailure = "deployment.testFailure"; @@ -50,6 +51,7 @@ public class JobMetrics { static String valueOf(RunStatus status) { switch (status) { case outOfCapacity: return outOfCapacity; + case endpointCertificateTimeout: return endpointCertificateTimeout; case deploymentFailed: return deploymentFailure; case installationFailed: return convergenceFailure; case testFailure: return testFailure; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java index 4d0b7ef3b90..fba3f7ae6e9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java @@ -17,6 +17,9 @@ public enum RunStatus { /** Deployment of the real application was rejected. */ deploymentFailed, + /** Deployment timed out waiting for endpoint certificate */ + endpointCertificateTimeout, + /** Installation of the real application timed out. */ installationFailed, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java index 1bb449b0a16..23a3ffb42b6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java @@ -58,6 +58,7 @@ public class EndpointCertificateManager { private final Clock clock; private final BooleanFlag useRefreshedEndpointCertificate; private final StringFlag endpointCertificateBackfill; + private final BooleanFlag endpointCertInSharedRouting; public EndpointCertificateManager(ZoneRegistry zoneRegistry, CuratorDb curator, @@ -71,6 +72,7 @@ public class EndpointCertificateManager { this.clock = clock; this.useRefreshedEndpointCertificate = Flags.USE_REFRESHED_ENDPOINT_CERTIFICATE.bindTo(flagSource); this.endpointCertificateBackfill = Flags.ENDPOINT_CERTIFICATE_BACKFILL.bindTo(flagSource); + this.endpointCertInSharedRouting = Flags.ENDPOINT_CERT_IN_SHARED_ROUTING.bindTo(flagSource); Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> { try { this.backfillCertificateMetadata(); @@ -82,7 +84,8 @@ public class EndpointCertificateManager { public Optional<EndpointCertificateMetadata> getEndpointCertificateMetadata(Instance instance, ZoneId zone) { - if (!zoneRegistry.zones().directlyRouted().ids().contains(zone)) return Optional.empty(); + boolean endpointCertInSharedRouting = this.endpointCertInSharedRouting.with(FetchVector.Dimension.APPLICATION_ID, instance.id().serializedForm()).value(); + if (!zoneRegistry.zones().directlyRouted().ids().contains(zone) && !endpointCertInSharedRouting) return Optional.empty(); // Re-use existing certificate if already provisioned var endpointCertificateMetadata = @@ -153,7 +156,8 @@ public class EndpointCertificateManager { storedMetaData.certName(), storedMetaData.version(), providerMetadata.request_id(), - providerMetadata.requestedDnsSans()); + providerMetadata.requestedDnsSans(), + Optional.empty()); if (mode == BackfillMode.DRYRUN) { log.log(LogLevel.INFO, "Would update stored metadata " + storedMetaData + " with data from provider: " + backfilledMetadata); @@ -171,9 +175,9 @@ public class EndpointCertificateManager { } private EndpointCertificateMetadata provisionEndpointCertificate(Instance instance) { - List<ZoneId> directlyRoutedZones = zoneRegistry.zones().directlyRouted().zones().stream().map(ZoneApi::getId).collect(Collectors.toUnmodifiableList()); + List<ZoneId> zones = zoneRegistry.zones().controllerUpgraded().zones().stream().map(ZoneApi::getId).collect(Collectors.toUnmodifiableList()); EndpointCertificateMetadata provisionedCertificateMetadata = endpointCertificateProvider - .requestCaSignedCertificate(instance.id(), dnsNamesOf(instance.id(), directlyRoutedZones)); + .requestCaSignedCertificate(instance.id(), dnsNamesOf(instance.id(), zones), Optional.empty()); curator.writeEndpointCertificateMetadata(instance.id(), provisionedCertificateMetadata); return provisionedCertificateMetadata; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java index ff88805f957..5b792f384e5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java @@ -1,12 +1,11 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer; -import com.yahoo.vespa.hosted.controller.restapi.cost.CostCalculator; +import com.yahoo.vespa.hosted.controller.metric.CostCalculator; import java.time.Clock; import java.time.Duration; @@ -34,7 +33,8 @@ public class CostReportMaintainer extends Maintainer { @Override protected void maintain() { - consumer.consume(CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller(), clock, consumer.fixedAllocations(), CloudName.from("yahoo"))); + var csv = CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller(), clock, consumer.fixedAllocations()); + consumer.consume(csv); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java index ec01b3817a7..4cc4ee0386c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java @@ -1,5 +1,5 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.cost; +package com.yahoo.vespa.hosted.controller.metric; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; @@ -28,12 +28,12 @@ import static com.yahoo.yolean.Exceptions.uncheck; public class CostCalculator { private static final double SELF_HOSTED_DISCOUNT = .5; + private static final CloudName cloudName = CloudName.from("yahoo"); public static String resourceShareByPropertyToCsv(NodeRepository nodeRepository, Controller controller, Clock clock, - Map<Property, ResourceAllocation> fixedAllocations, - CloudName cloudName) { + Map<Property, ResourceAllocation> fixedAllocations) { var date = DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("UTC")).format(clock.instant()); @@ -61,14 +61,12 @@ public class CostCalculator { } // Add fixed allocations from config - if (cloudName.equals(CloudName.from("yahoo"))) { - for (var kv : fixedAllocations.entrySet()) { - var property = kv.getKey(); - var allocation = allocationByProperty.getOrDefault(property, ResourceAllocation.ZERO); - var discountedFixedAllocation = kv.getValue().multiply(SELF_HOSTED_DISCOUNT); - allocationByProperty.put(property, allocation.plus(discountedFixedAllocation)); - totalAllocation = totalAllocation.plus(discountedFixedAllocation); - } + for (var kv : fixedAllocations.entrySet()) { + var property = kv.getKey(); + var allocation = allocationByProperty.getOrDefault(property, ResourceAllocation.ZERO); + var discountedFixedAllocation = kv.getValue().multiply(SELF_HOSTED_DISCOUNT); + allocationByProperty.put(property, allocation.plus(discountedFixedAllocation)); + totalAllocation = totalAllocation.plus(discountedFixedAllocation); } return toCsv(allocationByProperty, date, totalAllocation); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index ad2835e301f..eb86b1028e2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -42,7 +42,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.NavigableMap; @@ -521,8 +520,7 @@ public class CuratorDb { } public Optional<EndpointCertificateMetadata> readEndpointCertificateMetadata(ApplicationId applicationId) { - Optional<String> zkData = curator.getData(endpointCertificatePath(applicationId)).map(String::new); - return zkData.map(EndpointCertificateMetadataSerializer::fromJsonOrTlsSecretsKeysString); + return curator.getData(endpointCertificatePath(applicationId)).map(String::new).map(EndpointCertificateMetadataSerializer::fromJsonString); } public Map<ApplicationId, EndpointCertificateMetadata> readAllEndpointCertificateMetadata() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java index 653f224a02b..501d3a06d42 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java @@ -4,6 +4,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; +import com.yahoo.slime.Type; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import java.util.List; @@ -33,6 +34,7 @@ public class EndpointCertificateMetadataSerializer { private final static String versionField = "version"; private final static String requestIdField = "requestId"; private final static String requestedDnsSansField = "requestedDnsSans"; + private final static String issuerField = "issuer"; public static Slime toSlime(EndpointCertificateMetadata metadata) { Slime slime = new Slime(); @@ -51,46 +53,31 @@ public class EndpointCertificateMetadataSerializer { } public static EndpointCertificateMetadata fromSlime(Inspector inspector) { - switch (inspector.type()) { - case STRING: // TODO: Remove once all are transmitted and stored as JSON - return new EndpointCertificateMetadata( - inspector.asString() + "-key", - inspector.asString() + "-cert", - 0 - ); - case OBJECT: { - Optional<String> request_id = inspector.field(requestIdField).valid() ? - Optional.of(inspector.field(requestIdField).asString()) : - Optional.empty(); + if (inspector.type() != Type.OBJECT) + throw new IllegalArgumentException("Unknown format encountered for endpoint certificate metadata!"); + Optional<String> request_id = inspector.field(requestIdField).valid() ? + Optional.of(inspector.field(requestIdField).asString()) : + Optional.empty(); - Optional<List<String>> requestedDnsSans = inspector.field(requestedDnsSansField).valid() ? - Optional.of(IntStream.range(0, inspector.field(requestedDnsSansField).entries()) - .mapToObj(i -> inspector.field(requestedDnsSansField).entry(i).asString()).collect(Collectors.toList())) : - Optional.empty(); + Optional<List<String>> requestedDnsSans = inspector.field(requestedDnsSansField).valid() ? + Optional.of(IntStream.range(0, inspector.field(requestedDnsSansField).entries()) + .mapToObj(i -> inspector.field(requestedDnsSansField).entry(i).asString()).collect(Collectors.toList())) : + Optional.empty(); - return new EndpointCertificateMetadata( - inspector.field(keyNameField).asString(), - inspector.field(certNameField).asString(), - Math.toIntExact(inspector.field(versionField).asLong()), - request_id, - requestedDnsSans - ); - } + Optional<String> issuer = inspector.field(issuerField).valid() ? + Optional.of(inspector.field(issuerField).asString()) : + Optional.empty(); - default: - throw new IllegalArgumentException("Unknown format encountered for endpoint certificate metadata!"); - } + return new EndpointCertificateMetadata( + inspector.field(keyNameField).asString(), + inspector.field(certNameField).asString(), + Math.toIntExact(inspector.field(versionField).asLong()), + request_id, + requestedDnsSans, + issuer); } - public static EndpointCertificateMetadata fromTlsSecretsKeysString(String tlsSecretsKeys) { - return fromSlime(new Slime().setString(tlsSecretsKeys)); - } - - public static EndpointCertificateMetadata fromJsonOrTlsSecretsKeysString(String zkdata) { - if (zkdata.strip().startsWith("{")) { - return fromSlime(SlimeUtils.jsonToSlime(zkdata).get()); - } else { - return fromTlsSecretsKeysString(zkdata); - } + public static EndpointCertificateMetadata fromJsonString(String zkdata) { + return fromSlime(SlimeUtils.jsonToSlime(zkdata).get()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index 9e674134347..1aa229984a8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -31,6 +31,7 @@ import java.util.TreeMap; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.endpointCertificateTimeout; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.outOfCapacity; @@ -346,14 +347,15 @@ class RunSerializer { static String valueOf(RunStatus status) { switch (status) { - case running : return "running"; - case outOfCapacity : return "outOfCapacity"; - case deploymentFailed : return "deploymentFailed"; - case installationFailed : return "installationFailed"; - case testFailure : return "testFailure"; - case error : return "error"; - case success : return "success"; - case aborted : return "aborted"; + case running : return "running"; + case outOfCapacity : return "outOfCapacity"; + case endpointCertificateTimeout : return "endpointCertificateTimeout"; + case deploymentFailed : return "deploymentFailed"; + case installationFailed : return "installationFailed"; + case testFailure : return "testFailure"; + case error : return "error"; + case success : return "success"; + case aborted : return "aborted"; default: throw new AssertionError("No value defined for '" + status + "'!"); } @@ -361,14 +363,15 @@ class RunSerializer { static RunStatus runStatusOf(String status) { switch (status) { - case "running" : return running; - case "outOfCapacity" : return outOfCapacity; - case "deploymentFailed" : return deploymentFailed; - case "installationFailed" : return installationFailed; - case "testFailure" : return testFailure; - case "error" : return error; - case "success" : return success; - case "aborted" : return aborted; + case "running" : return running; + case "outOfCapacity" : return outOfCapacity; + case "endpointCertificateTimeout" : return endpointCertificateTimeout; + case "deploymentFailed" : return deploymentFailed; + case "installationFailed" : return installationFailed; + case "testFailure" : return testFailure; + case "error" : return error; + case "success" : return success; + case "aborted" : return aborted; default: throw new IllegalArgumentException("No run status defined by '" + status + "'!"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 0f4eca6fd58..94d7b120406 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -432,11 +432,28 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse applications(String tenantName, Optional<String> applicationName, HttpRequest request) { TenantName tenant = TenantName.from(tenantName); Slime slime = new Slime(); - Cursor array = slime.setArray(); + Cursor applicationArray = slime.setArray(); for (Application application : controller.applications().asList(tenant)) { - if (applicationName.map(application.id().application().value()::equals).orElse(true)) - for (InstanceName instance : application.instances().keySet()) - toSlime(application.id().instance(instance), array.addObject(), request); + if (applicationName.map(application.id().application().value()::equals).orElse(true)) { + Cursor applicationObject = applicationArray.addObject(); + applicationObject.setString("tenant", application.id().tenant().value()); + applicationObject.setString("application", application.id().application().value()); + applicationObject.setString("url", withPath("/application/v4" + + "/tenant/" + application.id().tenant().value() + + "/application/" + application.id().application().value(), + request.getUri()).toString()); + Cursor instanceArray = applicationObject.setArray("instances"); + for (InstanceName instance : showOnlyProductionInstances(request) ? application.productionInstances().keySet() + : application.instances().keySet()) { + Cursor instanceObject = instanceArray.addObject(); + instanceObject.setString("instance", instance.value()); + instanceObject.setString("url", withPath("/application/v4" + + "/tenant/" + application.id().tenant().value() + + "/application/" + application.id().application().value() + + "/instance/" + instance.value(), + request.getUri()).toString()); + } + } } return new SlimeJsonResponse(slime); } @@ -1044,6 +1061,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // Add zone endpoints defined by routing policies var endpointArray = response.setArray("endpoints"); for (var policy : controller.routingController().policies().get(deploymentId).values()) { + // TODO(mpolden): Always add endpoints from all policies, independent of routing method. This allows removal + // of RoutingGenerator and eliminates the external call to the routing layer below. + if (!controller.routingController().supportsRoutingMethod(RoutingMethod.exclusive, deployment.zone())) continue; if (!policy.status().isActive()) continue; { var endpointObject = endpointArray.addObject(); @@ -1140,6 +1160,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { deployment.activity().lastWritesPerSecond().ifPresent(value -> activity.setDouble("lastWritesPerSecond", value)); // Cost + // TODO(mpolden): Unused, remove this field and related code. DeploymentCost appCost = new DeploymentCost(Map.of()); Cursor costObject = response.setObject("cost"); toSlime(appCost, costObject); @@ -1791,6 +1812,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'."); } + // TODO jonmv: This should list applications, not instances. Cursor applicationArray = object.setArray("applications"); for (Application application : applications) { DeploymentStatus status = controller.jobController().deploymentStatus(application); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 32c2d6ec3d1..0d1dca391bd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -554,15 +554,16 @@ class JobControllerApiHandlerHelper { private static String nameOf(RunStatus status) { switch (status) { - case running: return "running"; - case aborted: return "aborted"; - case error: return "error"; - case testFailure: return "testFailure"; - case outOfCapacity: return "outOfCapacity"; - case installationFailed: return "installationFailed"; - case deploymentFailed: return "deploymentFailed"; - case success: return "success"; - default: throw new IllegalArgumentException("Unexpected status '" + status + "'"); + case running: return "running"; + case aborted: return "aborted"; + case error: return "error"; + case testFailure: return "testFailure"; + case endpointCertificateTimeout: return "endpointCertificateTimeout"; + case outOfCapacity: return "outOfCapacity"; + case installationFailed: return "installationFailed"; + case deploymentFailed: return "deploymentFailed"; + case success: return "success"; + default: throw new IllegalArgumentException("Unexpected status '" + status + "'"); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java deleted file mode 100644 index 8bf3f4a6fd0..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.cost; - -import com.yahoo.config.provision.CloudName; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; -import com.yahoo.restapi.Path; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; -import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.StringResponse; - -import java.time.Clock; -import java.util.Optional; - -import static com.yahoo.jdisc.http.HttpRequest.Method.GET; - -/** - * @author ldalves - */ -public class CostApiHandler extends LoggingRequestHandler { - - private final Controller controller; - private final NodeRepository nodeRepository; - private final CostReportConsumer costReportConsumer; - - public CostApiHandler(Context ctx, Controller controller) { - super(ctx); - this.controller = controller; - this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); - this.costReportConsumer = controller.serviceRegistry().costReportConsumer(); - } - - @Override - public HttpResponse handle(HttpRequest request) { - if (request.getMethod() != GET) { - return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); - } - - Path path = new Path(request.getUri()); - - if (path.matches("/cost/v1/csv")) { - Optional<String> cloudProperty = Optional.ofNullable(request.getProperty("cloud")); - CloudName cloud = cloudProperty.map(CloudName::from).orElse(CloudName.defaultName()); - return new StringResponse(CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller, Clock.systemUTC(), costReportConsumer.fixedAllocations(), cloud)); - } - - return ErrorResponse.notFoundError("Nothing at " + path); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/package-info.java deleted file mode 100644 index a96ae5488fa..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.controller.restapi.cost; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java index 673cb7e82e9..26ccecee3e6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java @@ -29,6 +29,7 @@ import com.yahoo.yolean.Exceptions; import java.net.URI; import java.time.Instant; +import java.util.Comparator; import java.util.List; import java.util.Objects; import java.util.logging.Level; @@ -233,7 +234,10 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { ? application.instances().values() : List.of(application.instances().get(instanceId.instance())); for (var instance : instances) { - var zones = zoneId == null ? instance.deployments().keySet() : List.of(zoneId); + var zones = zoneId == null + ? instance.deployments().keySet().stream().sorted(Comparator.comparing(ZoneId::value)) + .collect(Collectors.toList()) + : List.of(zoneId); for (var zone : zones) { var deploymentId = new DeploymentId(instance.id(), zone); // Include status from rotation diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 50e567b2024..b082f66b70a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -716,7 +716,7 @@ public class ControllerTest { assertEquals(Stream.concat(Stream.of("vznqtz7a5ygwjkbhhj7ymxvlrekgt4l6g.vespa.oath.cloud", "app1.tenant1.global.vespa.oath.cloud", "*.app1.tenant1.global.vespa.oath.cloud"), - tester.controller().zoneRegistry().zones().directlyRouted().ids().stream() + tester.controller().zoneRegistry().zones().controllerUpgraded().ids().stream() .flatMap(zone -> Stream.of("", "*.") .map(prefix -> prefix + "app1.tenant1." + zone.region().value() + (zone.environment() == Environment.prod ? "" : "." + zone.environment().value()) + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index b3a6ef53d2b..c36d4494a82 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -338,6 +338,25 @@ public class JobRunnerTest { } @Test + public void onlySuccessfulRunExpiresThenAnotherFails() { + DeploymentTester tester = new DeploymentTester(); + JobController jobs = tester.controller().jobController(); + var app = tester.newDeploymentContext().submit(); + JobId jobId = new JobId(app.instanceId(), systemTest); + assertFalse(jobs.lastSuccess(jobId).isPresent()); + + app.runJob(systemTest); + assertTrue(jobs.lastSuccess(jobId).isPresent()); + assertEquals(1, jobs.runs(jobId).size()); + + tester.clock().advance(JobController.maxHistoryAge.plusSeconds(1)); + app.submit(); + app.failDeployment(systemTest); + assertFalse(jobs.lastSuccess(jobId).isPresent()); + assertEquals(1, jobs.runs(jobId).size()); + } + + @Test public void timeout() { DeploymentTester tester = new DeploymentTester(); JobController jobs = tester.controller().jobController(); @@ -388,6 +407,7 @@ public class JobRunnerTest { assertEquals(1, metric.getMetric(context::equals, JobMetrics.convergenceFailure).get().intValue()); assertEquals(1, metric.getMetric(context::equals, JobMetrics.deploymentFailure).get().intValue()); assertEquals(1, metric.getMetric(context::equals, JobMetrics.outOfCapacity).get().intValue()); + assertEquals(1, metric.getMetric(context::equals, JobMetrics.endpointCertificateTimeout).get().intValue()); assertEquals(1, metric.getMetric(context::equals, JobMetrics.testFailure).get().intValue()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java index 7428b9901a2..5f8a3eaa98a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java @@ -29,17 +29,10 @@ public class EndpointCertificateMetadataSerializerTest { } @Test - public void deserializeFromString() { - assertEquals( - new EndpointCertificateMetadata("foo-key", "foo-cert", 0), - EndpointCertificateMetadataSerializer.fromJsonOrTlsSecretsKeysString("foo")); - } - - @Test public void deserializeFromJson() { assertEquals( sample, - EndpointCertificateMetadataSerializer.fromJsonOrTlsSecretsKeysString( + EndpointCertificateMetadataSerializer.fromJsonString( "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1}")); } @@ -47,7 +40,7 @@ public class EndpointCertificateMetadataSerializerTest { public void deserializeFromJsonWithRequestMetadata() { assertEquals( sampleWithRequestMetadata, - EndpointCertificateMetadataSerializer.fromJsonOrTlsSecretsKeysString( + EndpointCertificateMetadataSerializer.fromJsonString( "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"]}")); } }
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index da770c9c023..9e03a236f4a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -92,9 +92,6 @@ public class ControllerContainerTest { " <handler id='com.yahoo.vespa.hosted.controller.restapi.os.OsApiHandler'>\n" + " <binding>http://*/os/v1/*</binding>\n" + " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.cost.CostApiHandler'>\n" + - " <binding>http://*/cost/v1/*</binding>\n" + - " </handler>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v2.ZoneApiHandler'>\n" + " <binding>http://*/zone/v2</binding>\n" + " <binding>http://*/zone/v2/*</binding>\n" + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 1bdc3a22c2e..442770ba23e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import ai.vespa.hosted.api.MultiPartStreamer; import ai.vespa.hosted.api.Signatures; -import com.yahoo.application.container.handler.Headers; import com.yahoo.application.container.handler.Request; import com.yahoo.component.Version; import com.yahoo.config.application.api.ValidationId; @@ -14,6 +13,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; @@ -59,6 +59,7 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.RotationStatusUpdater; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; @@ -76,8 +77,6 @@ import org.junit.Test; import java.io.File; import java.math.BigDecimal; import java.net.URI; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; import java.time.YearMonth; @@ -85,6 +84,7 @@ import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Base64; import java.util.Collections; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -92,7 +92,6 @@ import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.function.Supplier; -import java.util.stream.Collectors; import static com.yahoo.application.container.handler.Request.Method.DELETE; import static com.yahoo.application.container.handler.Request.Method.GET; @@ -237,10 +236,10 @@ public class ApplicationApiTest extends ControllerContainerTest { // GET tenant applications tester.assertResponse(request("/application/v4/tenant/tenant1/application/", GET).userIdentity(USER_ID), - new File("instance-list.json")); + new File("application-list.json")); // GET tenant applications (instances of "application1" only) tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/", GET).userIdentity(USER_ID), - new File("instance-list.json")); + new File("application-list.json")); // GET at a tenant, with "&recursive=true&production=true", recurses over no instances yet, as they are not in deployment spec. tester.assertResponse(request("/application/v4/tenant/tenant1/", GET) .userIdentity(USER_ID) @@ -1479,14 +1478,17 @@ public class ApplicationApiTest extends ControllerContainerTest { @Test public void applicationWithRoutingPolicy() { var app = deploymentTester.newDeploymentContext(createTenantAndApplication()); + var zone = ZoneId.from(Environment.prod, RegionName.from("us-west-1")); + deploymentTester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(zone), + EnumSet.of(RoutingMethod.exclusive, RoutingMethod.shared)); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) .instances("instance1") - .region("us-west-1") + .region(zone.region().value()) .build(); app.submit(applicationPackage).deploy(); - app.addRoutingPolicy(ZoneId.from(Environment.prod, RegionName.from("us-west-1")), true); - app.addRoutingPolicy(ZoneId.from(Environment.prod, RegionName.from("us-west-1")), false); + app.addRoutingPolicy(zone, true); + app.addRoutingPolicy(zone, false); // GET application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", GET) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-list.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-list.json new file mode 100644 index 00000000000..2479f127f92 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-list.json @@ -0,0 +1,13 @@ +[ + { + "tenant": "tenant1", + "application":"application1", + "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1", + "instances": [ + { + "instance":"instance1", + "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1" + } + ] + } +] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-list.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-list.json deleted file mode 100644 index 024ca11dbe3..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-list.json +++ /dev/null @@ -1,3 +0,0 @@ -[ - @include(instance-reference.json) -]
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java deleted file mode 100644 index f992a54a114..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.cost; - -import com.yahoo.application.container.handler.Request; -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; -import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; -import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; -import org.junit.Before; -import org.junit.Test; - -/** - * @author andreer - */ -public class CostApiTest extends ControllerContainerTest { - - private static final String responses = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/responses/"; - private static final AthenzIdentity operator = AthenzUser.fromUserId("operatorUser"); - private static final CloudName cloud1 = CloudName.from("yahoo"); - private static final CloudName cloud2 = CloudName.from("cloud2"); - private static final ZoneApi zone1 = ZoneApiMock.newBuilder().withId("prod.us-east-3").with(cloud1).build(); - private static final ZoneApi zone2 = ZoneApiMock.newBuilder().withId("prod.us-west-1").with(cloud1).build(); - private static final ZoneApi zone3 = ZoneApiMock.newBuilder().withId("prod.eu-west-1").with(cloud2).build(); - - private ContainerTester tester; - - @Before - public void before() { - tester = new ContainerTester(container, responses); - tester.serviceRegistry().zoneRegistry().setSystemName(SystemName.cd) - .setZones(zone1, zone2, zone3); - } - - @Test - public void test_api() { - assertResponse(new Request("http://localhost:8080/cost/v1/csv"), - "Date,Property,Reserved Cpu Cores,Reserved Memory GB,Reserved Disk Space GB,Usage Fraction\n", 200); - } - - private void assertResponse(Request request, String body, int statusCode) { - addIdentityToRequest(request, operator); - tester.assertResponse(request, body, statusCode); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json index 9a5d919e9b4..e0b0e5e9b7a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json @@ -4,7 +4,7 @@ "routingMethod": "shared", "instance": "t1:a1:default", "environment": "prod", - "region": "us-west-1", + "region": "us-east-3", "status": "in", "agent": "unknown", "changedAt": "(ignore)" @@ -13,7 +13,7 @@ "routingMethod": "shared", "instance": "t1:a1:default", "environment": "prod", - "region": "us-east-3", + "region": "us-west-1", "status": "in", "agent": "unknown", "changedAt": "(ignore)" diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json index 85db7411c40..1ee4e1b82ba 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json @@ -4,7 +4,7 @@ "routingMethod": "shared", "instance": "t1:a1:default", "environment": "prod", - "region": "us-west-1", + "region": "us-east-3", "status": "in", "agent": "unknown", "changedAt": "(ignore)" @@ -13,7 +13,7 @@ "routingMethod": "shared", "instance": "t1:a1:default", "environment": "prod", - "region": "us-east-3", + "region": "us-west-1", "status": "in", "agent": "unknown", "changedAt": "(ignore)" @@ -22,7 +22,7 @@ "routingMethod": "shared", "instance": "t1:a2:default", "environment": "prod", - "region": "us-west-1", + "region": "us-east-3", "status": "in", "agent": "unknown", "changedAt": "(ignore)" @@ -31,7 +31,7 @@ "routingMethod": "shared", "instance": "t1:a2:default", "environment": "prod", - "region": "us-east-3", + "region": "us-west-1", "status": "in", "agent": "unknown", "changedAt": "(ignore)" diff --git a/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java b/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java index 47599e53ece..8f52c29e84d 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java @@ -144,9 +144,9 @@ public class ComparisonNode implements ExpressionNode { return new ResultList(Result.INVALID); } } else if (oLeft instanceof AttributeNode.VariableValueList) { - return evaluateListAndSingle((AttributeNode.VariableValueList)oLeft, oRight); + return evaluateLhsListAndRhsSingle((AttributeNode.VariableValueList)oLeft, oRight); } else if (oRight instanceof AttributeNode.VariableValueList) { - return evaluateListAndSingle((AttributeNode.VariableValueList)oRight, oLeft); + return evaluateLhsSingleAndRhsList(oLeft, (AttributeNode.VariableValueList)oRight); } return new ResultList(evaluateBool(oLeft, oRight)); } @@ -197,7 +197,7 @@ public class ComparisonNode implements ExpressionNode { } } - private ResultList evaluateListAndSingle(AttributeNode.VariableValueList lhs, Object rhs) { + private ResultList evaluateLhsListAndRhsSingle(AttributeNode.VariableValueList lhs, Object rhs) { if (rhs == null && lhs == null) { return new ResultList(Result.TRUE); } @@ -215,6 +215,24 @@ public class ComparisonNode implements ExpressionNode { return retVal; } + private ResultList evaluateLhsSingleAndRhsList(Object lhs, AttributeNode.VariableValueList rhs) { + if (rhs == null && lhs == null) { + return new ResultList(Result.TRUE); + } + + if (rhs == null || lhs == null) { + return new ResultList(Result.FALSE); + } + + ResultList retVal = new ResultList(); + for (ResultList.VariableValue value : rhs) { + Result result = evaluateBool(lhs, value.getValue()); + retVal.add((FieldPathIteratorHandler.VariableMap)value.getVariables().clone(), result); + } + + return retVal; + } + /** * Evaluate this expression on two operands, given that they are not invalid. * diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java index 0212be8542e..4e591cdfbd4 100644 --- a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java +++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java @@ -704,6 +704,23 @@ public class DocumentSelectorTestCase { } @Test + public void using_non_commutative_comparison_operator_with_field_value_is_well_defined() throws ParseException { + var documents = createDocs(); + // Doc 0 contains 24 in `hint` field. + assertEquals(Result.FALSE, evaluate("25 <= test.hint", documents.get(0))); + assertEquals(Result.TRUE, evaluate("24 <= test.hint", documents.get(0))); + assertEquals(Result.TRUE, evaluate("25 > test.hint", documents.get(0))); + assertEquals(Result.FALSE, evaluate("24 > test.hint", documents.get(0))); + assertEquals(Result.TRUE, evaluate("24 >= test.hint", documents.get(0))); + + assertEquals(Result.FALSE, evaluate("test.hint <= 23", documents.get(0))); + assertEquals(Result.TRUE, evaluate("test.hint <= 24", documents.get(0))); + assertEquals(Result.TRUE, evaluate("test.hint > 23", documents.get(0))); + assertEquals(Result.FALSE, evaluate("test.hint > 24", documents.get(0))); + assertEquals(Result.TRUE, evaluate("test.hint >= 24", documents.get(0))); + } + + @Test public void imported_field_references_are_treated_as_valid_field_with_missing_value() throws ParseException { var documents = createDocs(); assertEquals(Result.TRUE, evaluate("test.my_imported_field == null", documents.get(0))); diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp index c1e5fbecd14..6d446f6f1d7 100644 --- a/document/src/tests/documentselectparsertest.cpp +++ b/document/src/tests/documentselectparsertest.cpp @@ -551,6 +551,22 @@ TEST_F(DocumentSelectParserTest, operators_0) PARSE("30 = 30", *_doc[0], True); } +TEST_F(DocumentSelectParserTest, using_non_commutative_comparison_operator_with_field_value_is_well_defined) { + auto doc = createDoc("testdoctype1", "id:foo:testdoctype1::bar", 24, 0.0, "foo", "bar", 0); + // Document's `headerval` field has value of 24. + PARSE("25 <= testdoctype1.headerval", *doc, False); + PARSE("24 <= testdoctype1.headerval", *doc, True); + PARSE("25 > testdoctype1.headerval", *doc, True); + PARSE("24 > testdoctype1.headerval", *doc, False); + PARSE("24 >= testdoctype1.headerval", *doc, True); + + PARSE("testdoctype1.headerval <= 23", *doc, False); + PARSE("testdoctype1.headerval <= 24", *doc, True); + PARSE("testdoctype1.headerval > 23", *doc, True); + PARSE("testdoctype1.headerval > 24", *doc, False); + PARSE("testdoctype1.headerval >= 24", *doc, True); +} + TEST_F(DocumentSelectParserTest, regex_matching_does_not_bind_anchors_to_newlines) { createDocs(); diff --git a/documentgen-test/etc/complex/book.sd b/documentgen-test/etc/complex/book.sd index 872634bf53b..dd6ccafeab5 100644 --- a/documentgen-test/etc/complex/book.sd +++ b/documentgen-test/etc/complex/book.sd @@ -67,6 +67,10 @@ search book { attribute: tensor(x{}) } } + + import field ref.dummy as my_dummy {} + import field ref.foo as my_foo {} + field sw1 type float { } field didinteger type array<int> { diff --git a/documentgen-test/etc/complex/parent.sd b/documentgen-test/etc/complex/parent.sd index 99a50fdf8ea..50b8e76cf5a 100644 --- a/documentgen-test/etc/complex/parent.sd +++ b/documentgen-test/etc/complex/parent.sd @@ -4,7 +4,10 @@ search parent { document parent { field dummy type string { - + indexing: attribute + } + field foo type string { + indexing: attribute } } } diff --git a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java index 6339416d007..29bee2e9e3e 100644 --- a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java +++ b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java @@ -1028,5 +1028,14 @@ public class DocumentGenPluginTest { assertTrue(book.getDataType().fieldSetAll().contains(posZcurve)); assertTrue(book.getDataType().getFields().contains(posZcurve)); } + + @Test + public void imported_fields_are_enumerated_in_document_type() { + var docType = getBook().getDataType(); + assertEquals(2, docType.getImportedFieldNames().size()); + assertTrue(docType.hasImportedField("my_dummy")); + assertTrue(docType.hasImportedField("my_foo")); + assertFalse(docType.hasImportedField("some_field_that_does_not_exist")); + } } diff --git a/fbench/src/fbench/fbench.cpp b/fbench/src/fbench/fbench.cpp index 91475ce2125..593ae30a0e5 100644 --- a/fbench/src/fbench/fbench.cpp +++ b/fbench/src/fbench/fbench.cpp @@ -86,10 +86,13 @@ FBench::init_crypto_engine(const std::string &ca_certs_file_name, return false; } bool load_failed = false; - vespalib::net::tls::TransportSecurityOptions - tls_opts(maybe_load(ca_certs_file_name, load_failed), - maybe_load(cert_chain_file_name, load_failed), - maybe_load(private_key_file_name, load_failed)); + auto ts_builder = vespalib::net::tls::TransportSecurityOptions::Params(). + ca_certs_pem(maybe_load(ca_certs_file_name, load_failed)). + cert_chain_pem(maybe_load(cert_chain_file_name, load_failed)). + private_key_pem(maybe_load(private_key_file_name, load_failed)). + authorized_peers(vespalib::net::tls::AuthorizedPeers::allow_all_authenticated()). + disable_hostname_validation(true); // TODO configurable or default false! + vespalib::net::tls::TransportSecurityOptions tls_opts(std::move(ts_builder)); if (load_failed) { fprintf(stderr, "failed to load transport security options\n"); return false; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index a1ee71b9a58..32a235f2989 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -55,6 +55,18 @@ public class Flags { "Whether to enable Nessus.", "Takes effect on next host admin tick", HOSTNAME); + public static final UnboundBooleanFlag ENABLE_FLEET_SSHD_CONFIG = defineFeatureFlag( + "enable-fleet-sshd-config", false, + "Whether fleet should manage the /etc/ssh/sshd_config file.", + "Takes effect on next host admin tick.", + HOSTNAME); + + public static final UnboundBooleanFlag FLEET_CANARY = defineFeatureFlag( + "fleet-canary", false, + "Whether the host is a fleet canary.", + "Takes effect on next host admin tick.", + HOSTNAME); + public static final UnboundListFlag<String> DISABLED_HOST_ADMIN_TASKS = defineListFlag( "disabled-host-admin-tasks", List.of(), String.class, "List of host-admin task names (as they appear in the log, e.g. root>main>UpgradeTask) that should be skipped", @@ -62,7 +74,7 @@ public class Flags { HOSTNAME, NODE_TYPE); public static final UnboundStringFlag DOCKER_VERSION = defineStringFlag( - "docker-version", "1.13.1-91.git07f3374", + "docker-version", "1.13.1-102.git7f2769b", "The version of the docker to use of the format VERSION-REL: The YUM package to be installed will be " + "2:docker-VERSION-REL.el7.centos.x86_64 in AWS (and without '.centos' otherwise). " + "If docker-version is not of this format, it must be parseable by YumPackageName::fromString.", @@ -223,6 +235,11 @@ public class Flags { "Override the Docker image to use for deployments. This must containing the image name only, without tag", "Takes effect on next host-admin tick", APPLICATION_ID); + public static final UnboundBooleanFlag ENDPOINT_CERT_IN_SHARED_ROUTING = defineFeatureFlag( + "endpoint-cert-in-shared-routing", false, + "Whether to provision and use endpoint certs for apps in shared routing zones", + "Takes effect on next deployment of the application", APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java index 2d54c716f8d..aeb08e042a1 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java @@ -5,11 +5,11 @@ import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.ConnectorConfig; import org.apache.http.Header; import org.apache.http.HttpEntity; +import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.conn.ssl.NoopHostnameVerifier; import org.apache.http.conn.ssl.TrustAllStrategy; -import org.apache.http.impl.NoConnectionReuseStrategy; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.ssl.SSLContexts; @@ -19,12 +19,14 @@ import org.eclipse.jetty.server.handler.HandlerWrapper; import org.eclipse.jetty.util.ssl.SslContextFactory; import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLException; import javax.servlet.ServletException; import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.security.GeneralSecurityException; +import java.time.Duration; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -127,9 +129,14 @@ class HealthCheckProxyHandler extends HandlerWrapper { } CloseableHttpResponse requestStatusHtml() throws IOException { - HttpGet request = new HttpGet("https://localhost:" + port + HEALTH_CHECK_PATH); - request.setHeader("Connection", "Close"); - return client().execute(request); + try { + HttpGet request = new HttpGet("https://localhost:" + port + HEALTH_CHECK_PATH); + return client().execute(request); + } catch (SSLException e) { + log.log(Level.SEVERE, "SSL connection failed. Closing existing client, a new client will be created on next request", e); + close(); + throw e; + } } // Client construction must be delayed to ensure that the SslContextFactory is started before calling getSslContext(). @@ -139,11 +146,17 @@ class HealthCheckProxyHandler extends HandlerWrapper { if (client == null) { client = HttpClientBuilder.create() .disableAutomaticRetries() - .setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE) + .setMaxConnPerRoute(4) .setSSLContext(getSslContext(sslContextFactory)) .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) .setUserTokenHandler(context -> null) // https://stackoverflow.com/a/42112034/1615280 .setUserAgent("health-check-proxy-client") + .setDefaultRequestConfig( + RequestConfig.custom() + .setConnectTimeout((int) Duration.ofSeconds(4).toMillis()) + .setConnectionRequestTimeout((int) Duration.ofSeconds(4).toMillis()) + .setSocketTimeout((int) Duration.ofSeconds(8).toMillis()) + .build()) .build(); } } @@ -175,6 +188,7 @@ class HealthCheckProxyHandler extends HandlerWrapper { synchronized (this) { if (client != null) { client.close(); + client = null; } } } diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java index 88db5c99de9..eb292199ea2 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java @@ -5,6 +5,7 @@ import com.yahoo.security.KeyUtils; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.tls.AuthorizationMode; import com.yahoo.security.tls.DefaultTlsContext; +import com.yahoo.security.tls.HostnameVerification; import com.yahoo.security.tls.PeerAuthentication; import com.yahoo.security.tls.TlsContext; import com.yahoo.security.tls.policy.AuthorizedPeers; @@ -51,7 +52,7 @@ public class TlsContextBasedProviderTest { BigInteger.ONE) .build(); return new DefaultTlsContext( - List.of(certificate), keyPair.getPrivate(), List.of(certificate), new AuthorizedPeers(Set.of()), AuthorizationMode.ENFORCE, PeerAuthentication.NEED); + List.of(certificate), keyPair.getPrivate(), List.of(certificate), new AuthorizedPeers(Set.of()), AuthorizationMode.ENFORCE, PeerAuthentication.NEED, HostnameVerification.ENABLED); } private static class SimpleTlsContextBasedProvider extends TlsContextBasedProvider { diff --git a/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java b/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java index 7474220d4e7..a140e87713c 100644 --- a/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java +++ b/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java @@ -21,7 +21,8 @@ public class TlsCryptoEngine implements CryptoEngine { @Override public TlsCryptoSocket createClientCryptoSocket(SocketChannel channel, Spec spec) { - SSLEngine sslEngine = tlsContext.createSslEngine(); + String peerHost = spec.host() != null ? spec.host() : "localhost"; // Use localhost for wildcard address + SSLEngine sslEngine = tlsContext.createSslEngine(peerHost, spec.port()); sslEngine.setUseClientMode(true); return new TlsCryptoSocket(channel, sslEngine); } diff --git a/jrt/tests/com/yahoo/jrt/CryptoUtils.java b/jrt/tests/com/yahoo/jrt/CryptoUtils.java index e7e4eea568d..95ea581cb90 100644 --- a/jrt/tests/com/yahoo/jrt/CryptoUtils.java +++ b/jrt/tests/com/yahoo/jrt/CryptoUtils.java @@ -5,6 +5,7 @@ import com.yahoo.security.KeyUtils; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.tls.AuthorizationMode; import com.yahoo.security.tls.DefaultTlsContext; +import com.yahoo.security.tls.HostnameVerification; import com.yahoo.security.tls.PeerAuthentication; import com.yahoo.security.tls.TlsContext; import com.yahoo.security.tls.policy.AuthorizedPeers; @@ -35,21 +36,23 @@ class CryptoUtils { static final KeyPair keyPair = KeyUtils.generateKeypair(EC); static final X509Certificate certificate = X509CertificateBuilder - .fromKeypair(keyPair, new X500Principal("CN=dummy"), EPOCH, Instant.now().plus(1, DAYS), SHA256_WITH_ECDSA, generateRandomSerialNumber()) + .fromKeypair(keyPair, new X500Principal("CN=localhost"), EPOCH, Instant.now().plus(1, DAYS), SHA256_WITH_ECDSA, generateRandomSerialNumber()) .build(); static final AuthorizedPeers authorizedPeers = new AuthorizedPeers( singleton( new PeerPolicy( - "dummy-policy", + "localhost-policy", singleton( - new Role("dummy-role")), + new Role("localhost-role")), singletonList( new RequiredPeerCredential( - Field.CN, new HostGlobPattern("dummy")))))); + Field.CN, new HostGlobPattern("localhost")))))); static TlsContext createTestTlsContext() { - return new DefaultTlsContext(singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers, AuthorizationMode.ENFORCE, PeerAuthentication.NEED); + return new DefaultTlsContext( + singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers, + AuthorizationMode.ENFORCE, PeerAuthentication.NEED, HostnameVerification.ENABLED); } } diff --git a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp index 22550a19383..bf98bbd75ef 100644 --- a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp +++ b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp @@ -88,7 +88,7 @@ CfHandler::doConfigure() if (fp != NULL) { fprintf(fp, "[default]\n"); fprintf(fp, "host = %s\n", getenv("VESPA_HOSTNAME")); - fprintf(fp, "_meta = vespa_tenant::%s vespa_application::%s vespa_instance::%s\n", getenv("VESPA_TENANT"), getenv("VESPA_APPLICATION"), getenv("VESPA_INSTANCE")); + fprintf(fp, "_meta = vespa_tenant::%s vespa_app::%s.%s\n", getenv("VESPA_TENANT"), getenv("VESPA_APPLICATION"), getenv("VESPA_INSTANCE")); fclose(fp); rename(tmpPath.c_str(), path.c_str()); } diff --git a/metrics-proxy/CMakeLists.txt b/metrics-proxy/CMakeLists.txt index b43019e9ba6..7587159165d 100644 --- a/metrics-proxy/CMakeLists.txt +++ b/metrics-proxy/CMakeLists.txt @@ -6,6 +6,7 @@ install_config_definition(src/main/resources/configdefinitions/consumers.def ai. install_config_definition(src/main/resources/configdefinitions/monitoring.def ai.vespa.metricsproxy.core.monitoring.def) install_config_definition(src/main/resources/configdefinitions/metrics-nodes.def ai.vespa.metricsproxy.http.application.metrics-nodes.def) install_config_definition(src/main/resources/configdefinitions/node-dimensions.def ai.vespa.metricsproxy.metric.dimensions.node-dimensions.def) +install_config_definition(src/main/resources/configdefinitions/node-info.def ai.vespa.metricsproxy.http.metrics.node-info.def) install_config_definition(src/main/resources/configdefinitions/rpc-connector.def ai.vespa.metricsproxy.rpc.rpc-connector.def) install_config_definition(src/main/resources/configdefinitions/vespa-services.def ai.vespa.metricsproxy.service.vespa-services.def) install_config_definition(src/main/resources/configdefinitions/telegraf.def ai.vespa.metricsproxy.telegraf.telegraf.def) diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml index f72ad75c6af..355f420c2a4 100644 --- a/metrics-proxy/pom.xml +++ b/metrics-proxy/pom.xml @@ -132,6 +132,10 @@ <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> </dependency> + <dependency> + <groupId>org.apache.velocity</groupId> + <artifactId>velocity</artifactId> + </dependency> <!-- test scope --> diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java new file mode 100644 index 00000000000..1c8401d003a --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java @@ -0,0 +1,83 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.telegraf; + +import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; +import com.yahoo.system.execution.ProcessExecutor; +import org.apache.velocity.VelocityContext; +import org.apache.velocity.app.VelocityEngine; + +import java.io.FileWriter; +import java.io.InputStreamReader; +import java.io.Reader; +import java.io.Writer; +import java.util.logging.Logger; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * @author olaa + */ +public class Telegraf extends AbstractComponent { + + private static final String TELEGRAF_CONFIG_PATH = "/etc/telegraf/telegraf.conf"; + private static final String TELEGRAF_CONFIG_TEMPLATE_PATH = "templates/telegraf.conf.vm"; + private final TelegrafRegistry telegrafRegistry; + + private static final Logger logger = Logger.getLogger(Telegraf.class.getName()); + + @Inject + public Telegraf(TelegrafRegistry telegrafRegistry, TelegrafConfig telegrafConfig) { + this.telegrafRegistry = telegrafRegistry; + telegrafRegistry.addInstance(this); + writeConfig(telegrafConfig, uncheck(() -> new FileWriter(TELEGRAF_CONFIG_PATH))); + restartTelegraf(); + } + + protected static void writeConfig(TelegrafConfig telegrafConfig, Writer writer) { + VelocityContext context = new VelocityContext(); + context.put("intervalSeconds", telegrafConfig.intervalSeconds()); + context.put("cloudwatchPlugins", telegrafConfig.cloudWatch()); + // TODO: Add node cert if hosted + + VelocityEngine velocityEngine = new VelocityEngine(); + velocityEngine.init(); + velocityEngine.evaluate(context, writer, "TelegrafConfigWriter", getTemplateReader()); + uncheck(writer::close); + } + + private void restartTelegraf() { + logger.info("Restarting Telegraf"); + executeCommand("service telegraf restart"); + } + + private void stopTelegraf() { + logger.info("Stopping Telegraf"); + executeCommand("service telegraf stop"); + } + + private void executeCommand(String command) { + ProcessExecutor processExecutor = new ProcessExecutor + .Builder(10) + .successExitCodes(0) + .build(); + uncheck(() -> processExecutor.execute(command)) + .orElseThrow(() -> new RuntimeException("Timed out running command: " + command)); + } + + @SuppressWarnings("ConstantConditions") + private static Reader getTemplateReader() { + return new InputStreamReader(Telegraf.class.getClassLoader() + .getResourceAsStream(TELEGRAF_CONFIG_TEMPLATE_PATH) + ); + + } + + @Override + public void deconstruct() { + telegrafRegistry.removeInstance(this); + if (telegrafRegistry.isEmpty()) { + stopTelegraf(); + } + } +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java new file mode 100644 index 00000000000..dbc08a96777 --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java @@ -0,0 +1,33 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.telegraf; + +import com.yahoo.log.LogLevel; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.logging.Logger; + +/** + * @author olaa + */ +public class TelegrafRegistry { + + private static final List<Telegraf> telegrafInstances = Collections.synchronizedList(new ArrayList<>()); + + private static final Logger logger = Logger.getLogger(TelegrafRegistry.class.getName()); + + public void addInstance(Telegraf telegraf) { + logger.log(LogLevel.DEBUG, () -> "Adding Telegraf instance to registry: " + telegraf); + telegrafInstances.add(telegraf); + } + + public void removeInstance(Telegraf telegraf) { + logger.log(LogLevel.DEBUG, () -> "Removing Telegraf instance from registry: " + telegraf); + telegrafInstances.remove(telegraf); + } + + public boolean isEmpty() { + return telegrafInstances.isEmpty(); + } +} diff --git a/metrics-proxy/src/main/resources/configdefinitions/telegraf.def b/metrics-proxy/src/main/resources/configdefinitions/telegraf.def index f3b5db35d52..6abbd7921b5 100644 --- a/metrics-proxy/src/main/resources/configdefinitions/telegraf.def +++ b/metrics-proxy/src/main/resources/configdefinitions/telegraf.def @@ -5,9 +5,8 @@ package=ai.vespa.metricsproxy.telegraf intervalSeconds int default=60 -# The consumer to get metrics for -vespa.consumer string default="default" - +# The Vespa metrics consumer to get metrics for +cloudWatch[].consumer string cloudWatch[].region string default="us-east-1" cloudWatch[].namespace string diff --git a/metrics-proxy/src/main/resources/templates/telegraf.conf.vm b/metrics-proxy/src/main/resources/templates/telegraf.conf.vm new file mode 100644 index 00000000000..c427ee1ce4b --- /dev/null +++ b/metrics-proxy/src/main/resources/templates/telegraf.conf.vm @@ -0,0 +1,44 @@ +# Configuration for telegraf agent +[agent] + interval = "${intervalSeconds}s" + round_interval = true + metric_batch_size = 1000 + metric_buffer_limit = 10000 + collection_jitter = "0s" + flush_interval = "${intervalSeconds}s" + flush_jitter = "0s" + precision = "" + logtarget = "file" + logfile = "/var/log/telegraf/telegraf.log" + logfile_rotation_interval = "1d" + logfile_rotation_max_size = "20MB" + logfile_rotation_max_archives = 5 + +#foreach( $cloudwatch in $cloudwatchPlugins ) +# Configuration for AWS CloudWatch output. +[[outputs.cloudwatch]] + region = "$cloudwatch.region()" + namespace = "$cloudwatch.namespace()" +#if( $cloudwatch.accessKeyName() != "" ) + access_key = "$cloudwatch.accessKeyName()" + secret_key = "$cloudwatch.secretKeyName()" +#elseif( $cloudwatch.profile() != "" ) + profile = "$cloudwatch.profile()" +#end + tagexclude = ["vespa_consumer"] + [outputs.cloudwatch.tagpass] + vespa_consumer = ["$cloudwatch.consumer()"] + +# Configuration for Vespa input plugin +[[inputs.vespa]] + url = "http://localhost:19092/metrics/v2/values?consumer=$cloudwatch.consumer()" + [inputs.vespa.tags] + vespa_consumer = "$cloudwatch.consumer()" +#* TODO: Add node cert if hosted +#if( $isHosted ) + tls_cert = "${VESPA_CERTIFICATE_PATH}" + tls_key = "${VESPA_KEY_PATH}" + insecure_skip_verify = true +#end +*### +#end
\ No newline at end of file diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/telegraf/TelegrafTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/telegraf/TelegrafTest.java new file mode 100644 index 00000000000..00cbb4bf033 --- /dev/null +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/telegraf/TelegrafTest.java @@ -0,0 +1,42 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.telegraf; + +import ai.vespa.metricsproxy.TestUtil; +import org.junit.Test; + +import java.io.StringWriter; + +import static org.junit.Assert.*; + +/** + * @author olaa + */ +public class TelegrafTest { + + + @Test + public void test_writing_correct_telegraf_plugin_config() { + TelegrafConfig telegrafConfig = new TelegrafConfig.Builder() + .cloudWatch( + new TelegrafConfig.CloudWatch.Builder() + .accessKeyName("accessKey1") + .namespace("namespace1") + .secretKeyName("secretKey1") + .region("us-east-1") + .consumer("consumer1") + ) + .cloudWatch( + new TelegrafConfig.CloudWatch.Builder() + .namespace("namespace2") + .profile("awsprofile") + .region("us-east-2") + .consumer("consumer2") + ) + .intervalSeconds(300) + .build(); + StringWriter stringWriter = new StringWriter(); + Telegraf.writeConfig(telegrafConfig, stringWriter); + String expectedConfig = TestUtil.getFileContents( "telegraf-config-with-two-cloudwatch-plugins.txt"); + assertEquals(expectedConfig, stringWriter.toString()); + } +} diff --git a/metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt b/metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt new file mode 100644 index 00000000000..85656465901 --- /dev/null +++ b/metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt @@ -0,0 +1,46 @@ +# Configuration for telegraf agent +[agent] + interval = "300s" + round_interval = true + metric_batch_size = 1000 + metric_buffer_limit = 10000 + collection_jitter = "0s" + flush_interval = "300s" + flush_jitter = "0s" + precision = "" + logtarget = "file" + logfile = "/var/log/telegraf/telegraf.log" + logfile_rotation_interval = "1d" + logfile_rotation_max_size = "20MB" + logfile_rotation_max_archives = 5 + +# Configuration for AWS CloudWatch output. +[[outputs.cloudwatch]] + region = "us-east-1" + namespace = "namespace1" + access_key = "accessKey1" + secret_key = "secretKey1" + tagexclude = ["vespa_consumer"] + [outputs.cloudwatch.tagpass] + vespa_consumer = ["consumer1"] + +# Configuration for Vespa input plugin +[[inputs.vespa]] + url = "http://localhost:19092/metrics/v2/values?consumer=consumer1" + [inputs.vespa.tags] + vespa_consumer = "consumer1" +# Configuration for AWS CloudWatch output. +[[outputs.cloudwatch]] + region = "us-east-2" + namespace = "namespace2" + profile = "awsprofile" + tagexclude = ["vespa_consumer"] + [outputs.cloudwatch.tagpass] + vespa_consumer = ["consumer2"] + +# Configuration for Vespa input plugin +[[inputs.vespa]] + url = "http://localhost:19092/metrics/v2/values?consumer=consumer2" + [inputs.vespa.tags] + vespa_consumer = "consumer2" + diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java index cb209d710c8..e6d6f9463d3 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java @@ -114,6 +114,10 @@ public class NodeAdminStateUpdater { // To avoid node agents stalling for too long, we'll force unfrozen ticks now. adjustNodeAgentsToRunFromNodeRepository(); nodeAdmin.setFrozen(false); + + NodeState currentNodeState = nodeRepository.getNode(hostHostname).state(); + if (currentNodeState == NodeState.active) orchestrator.resume(hostHostname); + throw new ConvergenceException("Timed out trying to freeze all nodes: will force an unfrozen tick"); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java index bb6d53d3304..f75621e18a0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.provisioning.NodePrioritizer; import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceComparator; import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost; @@ -44,13 +45,15 @@ public class DynamicProvisioningMaintainer extends Maintainer { private static final ApplicationId preprovisionAppId = ApplicationId.from("hosted-vespa", "tenant-host", "preprovision"); private final HostProvisioner hostProvisioner; + private final HostResourcesCalculator hostResourcesCalculator; private final BooleanFlag dynamicProvisioningEnabled; private final ListFlag<PreprovisionCapacity> preprovisionCapacityFlag; - DynamicProvisioningMaintainer(NodeRepository nodeRepository, Duration interval, - HostProvisioner hostProvisioner, FlagSource flagSource) { + DynamicProvisioningMaintainer(NodeRepository nodeRepository, Duration interval, HostProvisioner hostProvisioner, + HostResourcesCalculator hostResourcesCalculator, FlagSource flagSource) { super(nodeRepository, interval); this.hostProvisioner = hostProvisioner; + this.hostResourcesCalculator = hostResourcesCalculator; this.dynamicProvisioningEnabled = Flags.ENABLE_DYNAMIC_PROVISIONING.bindTo(flagSource); this.preprovisionCapacityFlag = Flags.PREPROVISION_CAPACITY.bindTo(flagSource); } @@ -68,17 +71,14 @@ public class DynamicProvisioningMaintainer extends Maintainer { } void updateProvisioningNodes(NodeList nodes, Mutex lock) { - Map<String, Node> provisionedHostsByHostname = nodes.state(Node.State.provisioned).nodeType(NodeType.host) - .asList().stream() - .collect(Collectors.toMap(Node::hostname, Function.identity())); - - Map<Node, Set<Node>> nodesByProvisionedParent = nodes.asList().stream() - .filter(node -> node.parentHostname().map(provisionedHostsByHostname::containsKey).orElse(false)) + Map<String, Set<Node>> nodesByProvisionedParentHostname = nodes.nodeType(NodeType.tenant).asList().stream() + .filter(node -> node.parentHostname().isPresent()) .collect(Collectors.groupingBy( - node -> provisionedHostsByHostname.get(node.parentHostname().get()), + node -> node.parentHostname().get(), Collectors.toSet())); - nodesByProvisionedParent.forEach((host, children) -> { + nodes.state(Node.State.provisioned).nodeType(NodeType.host).forEach(host -> { + Set<Node> children = nodesByProvisionedParentHostname.getOrDefault(host.hostname(), Set.of()); try { List<Node> updatedNodes = hostProvisioner.provision(host, children); nodeRepository().write(updatedNodes, lock); @@ -112,7 +112,7 @@ public class DynamicProvisioningMaintainer extends Maintainer { NodeResources resources = it.next(); removableHosts.stream() .filter(host -> NodePrioritizer.ALLOCATABLE_HOST_STATES.contains(host.state())) - .filter(host -> host.flavor().resources().satisfies(resources)) + .filter(host -> hostResourcesCalculator.availableCapacityOf(host.flavor().name(), host.flavor().resources()).satisfies(resources)) .min(Comparator.comparingInt(n -> n.flavor().cost())) .ifPresent(host -> { removableHosts.remove(host); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index 0f363991310..e2b70608d58 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -119,14 +119,12 @@ public class LoadBalancerExpirer extends Maintainer { /** Apply operation to all load balancers that exist in given state, while holding lock */ private void withLoadBalancersIn(LoadBalancer.State state, Consumer<LoadBalancer> operation) { - try (var legacyLock = db.lockLoadBalancers()) { - for (var id : db.readLoadBalancerIds()) { - try (var lock = db.lockLoadBalancers(id.application())) { - var loadBalancer = db.readLoadBalancer(id); - if (loadBalancer.isEmpty()) continue; // Load balancer was removed during loop - if (loadBalancer.get().state() != state) continue; // Wrong state - operation.accept(loadBalancer.get()); - } + for (var id : db.readLoadBalancerIds()) { + try (var lock = db.lockLoadBalancers(id.application())) { + var loadBalancer = db.readLoadBalancer(id); + if (loadBalancer.isEmpty()) continue; // Load balancer was removed during loop + if (loadBalancer.get().state() != state) continue; // Wrong state + operation.accept(loadBalancer.get()); } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index ae8f8b052db..063b5ad2c2a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -81,7 +81,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { loadBalancerExpirer = provisionServiceProvider.getLoadBalancerService().map(lbService -> new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService)); dynamicProvisioningMaintainer = provisionServiceProvider.getHostProvisioner().map(hostProvisioner -> - new DynamicProvisioningMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, flagSource)); + new DynamicProvisioningMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, provisionServiceProvider.getHostResourcesCalculator(), flagSource)); capacityReportMaintainer = new CapacityReportMaintainer(nodeRepository, metric, defaults.capacityReportInterval); osUpgradeActivator = new OsUpgradeActivator(nodeRepository, defaults.osUpgradeActivatorInterval); rebalancer = new Rebalancer(deployer, nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), provisionServiceProvider.getHostProvisioner(), metric, clock, defaults.rebalancerInterval); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 0a8575578ce..f211ea9eac5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -521,11 +521,6 @@ public class CuratorDatabaseClient { transaction.commit(); } - // TODO(mpolden): Remove this and all usages once migration to per-application lock is complete - public Lock lockLoadBalancers() { - return lock(lockRoot.append("loadBalancersLock"), defaultLockTimeout); - } - public Lock lockLoadBalancers(ApplicationId application) { return lock(lockRoot.append("loadBalancersLock2").append(application.serializedForm()), defaultLockTimeout); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java index a609103ac89..fb76dc54d1a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java @@ -26,12 +26,6 @@ public class DockerHostCapacity { this.hostResourcesCalculator = Objects.requireNonNull(hostResourcesCalculator, "hostResourcesCalculator must be non-null"); } - /** Returns the allocation skew of this host */ - public double skew(Node host) { - NodeResources free = freeCapacityOf(host, false); - return Node.skew(host.flavor().resources(), free); - } - int compareWithoutInactive(Node hostA, Node hostB) { int result = compare(freeCapacityOf(hostB, true), freeCapacityOf(hostA, true)); if (result != 0) return result; @@ -72,7 +66,7 @@ public class DockerHostCapacity { NodeResources freeCapacityOf(Node host, boolean excludeInactive) { // Only hosts have free capacity if (!host.type().canRun(NodeType.tenant)) return new NodeResources(0, 0, 0, 0); - NodeResources hostResources = hostResourcesCalculator.availableCapacityOf(host.flavor().resources()); + NodeResources hostResources = hostResourcesCalculator.availableCapacityOf(host.flavor().name(), host.flavor().resources()); return allNodes.childrenOf(host).asList().stream() .filter(node -> !(excludeInactive && isInactiveOrRetired(node))) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java index 05915b82bae..b5c4478cd5a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java @@ -30,7 +30,7 @@ public class EmptyProvisionServiceProvider implements ProvisionServiceProvider { public static class NoopHostResourcesCalculator implements HostResourcesCalculator { @Override - public NodeResources availableCapacityOf(NodeResources hostResources) { + public NodeResources availableCapacityOf(String flavorName, NodeResources hostResources) { return hostResources; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 3cb6f848f7b..5753bbb3c5a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -62,7 +62,8 @@ public class GroupPreparer { List<Node> surplusActiveNodes, MutableInteger highestIndex, int spareCount, int wantedGroups) { boolean dynamicProvisioningEnabled = hostProvisioner.isPresent() && dynamicProvisioningEnabledFlag .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()) - .value() && preprovisionCapacityFlag.value().isEmpty(); + .value(); + boolean allocateFully = dynamicProvisioningEnabled && preprovisionCapacityFlag.value().isEmpty(); try (Mutex lock = nodeRepository.lock(application)) { @@ -73,7 +74,7 @@ public class GroupPreparer { LockedNodeList nodeList = nodeRepository.list(allocationLock); NodePrioritizer prioritizer = new NodePrioritizer(nodeList, application, cluster, requestedNodes, spareCount, wantedGroups, nodeRepository.nameResolver(), - hostResourcesCalculator, dynamicProvisioningEnabled); + hostResourcesCalculator, allocateFully); prioritizer.addApplicationNodes(); prioritizer.addSurplusNodes(surplusActiveNodes); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java index c5808a53837..a5570dbf169 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java @@ -9,6 +9,6 @@ import com.yahoo.config.provision.NodeResources; public interface HostResourcesCalculator { /** Calculates the resources that are reserved for host level processes and returns the remainder. */ - NodeResources availableCapacityOf(NodeResources hostResources); + NodeResources availableCapacityOf(String flavorName, NodeResources hostResources); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index d4d5b46dfdf..26f8cffa519 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -51,12 +51,10 @@ public class LoadBalancerProvisioner { this.db = nodeRepository.database(); this.service = service; // Read and write all load balancers to make sure they are stored in the latest version of the serialization format - try (var legacyLock = db.lockLoadBalancers()) { - for (var id : db.readLoadBalancerIds()) { - try (var lock = db.lockLoadBalancers(id.application())) { - var loadBalancer = db.readLoadBalancer(id); - loadBalancer.ifPresent(db::writeLoadBalancer); - } + for (var id : db.readLoadBalancerIds()) { + try (var lock = db.lockLoadBalancers(id.application())) { + var loadBalancer = db.readLoadBalancer(id); + loadBalancer.ifPresent(db::writeLoadBalancer); } } } @@ -75,10 +73,8 @@ public class LoadBalancerProvisioner { if (requestedNodes.type() != NodeType.tenant) return; // Nothing to provision for this node type if (!cluster.type().isContainer()) return; // Nothing to provision for this cluster type if (application.instance().isTester()) return; // Do not provision for tester instances - try (var legacyLock = db.lockLoadBalancers()) { - try (var lock = db.lockLoadBalancers(application)) { - provision(application, cluster.id(), false, lock); - } + try (var lock = db.lockLoadBalancers(application)) { + provision(application, cluster.id(), false, lock); } } @@ -94,17 +90,15 @@ public class LoadBalancerProvisioner { */ public void activate(ApplicationId application, Set<ClusterSpec> clusters, @SuppressWarnings("unused") Mutex applicationLock, NestedTransaction transaction) { - try (var legacyLock = db.lockLoadBalancers()) { - try (var lock = db.lockLoadBalancers(application)) { - var containerClusters = containerClusterOf(clusters); - for (var clusterId : containerClusters) { - // Provision again to ensure that load balancer instance is re-configured with correct nodes - provision(application, clusterId, true, lock); - } - // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed - var surplusLoadBalancers = surplusLoadBalancersOf(application, containerClusters); - deactivate(surplusLoadBalancers, transaction); + try (var lock = db.lockLoadBalancers(application)) { + var containerClusters = containerClusterOf(clusters); + for (var clusterId : containerClusters) { + // Provision again to ensure that load balancer instance is re-configured with correct nodes + provision(application, clusterId, true, lock); } + // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed + var surplusLoadBalancers = surplusLoadBalancersOf(application, containerClusters); + deactivate(surplusLoadBalancers, transaction); } } @@ -114,10 +108,8 @@ public class LoadBalancerProvisioner { */ public void deactivate(ApplicationId application, NestedTransaction transaction) { try (var applicationLock = nodeRepository.lock(application)) { - try (var legacyLock = db.lockLoadBalancers()) { - try (var lock = db.lockLoadBalancers(application)) { - deactivate(nodeRepository.loadBalancers(application).asList(), transaction); - } + try (var lock = db.lockLoadBalancers(application)) { + deactivate(nodeRepository.loadBalancers(application).asList(), transaction); } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index a0e9141f753..1ef23209369 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -46,13 +46,14 @@ public class NodePrioritizer { private final boolean isDocker; private final boolean isAllocatingForReplacement; private final boolean isTopologyChange; - private final boolean dynamicProvisioningEnabled; + /** If set, a host can only have nodes by single tenant and does not allow in-place resizing. */ + private final boolean allocateFully; private final int currentClusterSize; private final Set<Node> spareHosts; NodePrioritizer(LockedNodeList allNodes, ApplicationId application, ClusterSpec clusterSpec, NodeSpec nodeSpec, int spares, int wantedGroups, NameResolver nameResolver, HostResourcesCalculator hostResourcesCalculator, - boolean dynamicProvisioningEnabled) { + boolean allocateFully) { this.allNodes = allNodes; this.capacity = new DockerHostCapacity(allNodes, hostResourcesCalculator); this.requestedNodes = nodeSpec; @@ -60,7 +61,7 @@ public class NodePrioritizer { this.application = application; this.nameResolver = nameResolver; this.spareHosts = findSpareHosts(allNodes, capacity, spares); - this.dynamicProvisioningEnabled = dynamicProvisioningEnabled; + this.allocateFully = allocateFully; NodeList nodesInCluster = allNodes.owner(application).type(clusterSpec.type()).cluster(clusterSpec.id()); NodeList nonRetiredNodesInCluster = nodesInCluster.not().retired(); @@ -125,7 +126,7 @@ public class NodePrioritizer { .filter(node -> node.type() != NodeType.host || ALLOCATABLE_HOST_STATES.contains(node.state())) .filter(node -> node.reservedTo().isEmpty() || node.reservedTo().get().equals(application.tenant())); - if (dynamicProvisioningEnabled) { + if (allocateFully) { Set<String> candidateHostnames = candidates.asList().stream() .filter(node -> node.type() == NodeType.tenant) .filter(node -> node.allocation() @@ -213,7 +214,7 @@ public class NodePrioritizer { builder.parent(parent).freeParentCapacity(parentCapacity); if (!isNewNode) - builder.resizable(!dynamicProvisioningEnabled && requestedNodes.canResize( + builder.resizable(!allocateFully && requestedNodes.canResize( node.flavor().resources(), parentCapacity, isTopologyChange, currentClusterSize)); if (spareHosts.contains(parent)) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index 9dd8de6d306..36f876fb6bb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -26,9 +26,11 @@ import com.yahoo.vespa.hosted.provision.node.Status; import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; +import org.junit.Before; import org.junit.Test; import java.time.Duration; @@ -49,6 +51,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -64,18 +67,23 @@ public class DynamicProvisioningMaintainerTest { private final HostProvisionerTester tester = new HostProvisionerTester(); private final HostProvisioner hostProvisioner = mock(HostProvisioner.class); + private final HostResourcesCalculator hostResourcesCalculator = mock(HostResourcesCalculator.class); private final InMemoryFlagSource flagSource = new InMemoryFlagSource() .withBooleanFlag(Flags.ENABLE_DYNAMIC_PROVISIONING.id(), true) .withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(), PreprovisionCapacity.class); private final DynamicProvisioningMaintainer maintainer = new DynamicProvisioningMaintainer( - tester.nodeRepository, Duration.ofDays(1), hostProvisioner, flagSource); + tester.nodeRepository, Duration.ofDays(1), hostProvisioner, hostResourcesCalculator, flagSource); @Test public void delegates_to_host_provisioner_and_writes_back_result() { addNodes(); + Node host3 = tester.nodeRepository.getNode("host3").orElseThrow(); Node host4 = tester.nodeRepository.getNode("host4").orElseThrow(); Node host41 = tester.nodeRepository.getNode("host4-1").orElseThrow(); - assertTrue(Stream.of(host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty)); + assertTrue(Stream.of(host3, host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty)); + + Node host3new = host3.with(host3.ipConfig().with(Set.of("::5"))); + when(hostProvisioner.provision(eq(host3), eq(Set.of()))).thenReturn(List.of(host3new)); Node host4new = host4.with(host4.ipConfig().with(Set.of("::2"))); Node host41new = host41.with(host4.ipConfig().with(Set.of("::4", "10.0.0.1"))); @@ -83,8 +91,10 @@ public class DynamicProvisioningMaintainerTest { maintainer.updateProvisioningNodes(tester.nodeRepository.list(), () -> {}); verify(hostProvisioner).provision(eq(host4), eq(Set.of(host41))); + verify(hostProvisioner).provision(eq(host3), eq(Set.of())); verifyNoMoreInteractions(hostProvisioner); + assertEquals(Optional.of(host3new), tester.nodeRepository.getNode("host3")); assertEquals(Optional.of(host4new), tester.nodeRepository.getNode("host4")); assertEquals(Optional.of(host41new), tester.nodeRepository.getNode("host4-1")); } @@ -127,7 +137,7 @@ public class DynamicProvisioningMaintainerTest { @Test public void provision_deficit_and_deprovision_excess() { - flagSource.withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(new PreprovisionCapacity(1, 3, 2, 1), new PreprovisionCapacity(2, 3, 2, 2)), PreprovisionCapacity.class); + flagSource.withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(new PreprovisionCapacity(2, 4, 8, 1), new PreprovisionCapacity(2, 3, 2, 2)), PreprovisionCapacity.class); addNodes(); maintainer.convergeToCapacity(tester.nodeRepository.list()); @@ -150,6 +160,15 @@ public class DynamicProvisioningMaintainerTest { verifyNoMoreInteractions(hostProvisioner); } + @Before + public void setup() { + doAnswer(invocation -> { + String flavorName = invocation.getArgument(0, String.class); + if ("default".equals(flavorName)) return new NodeResources(2, 4, 8, 1); + return invocation.getArguments()[1]; + }).when(hostResourcesCalculator).availableCapacityOf(any(), any()); + } + public void addNodes() { List.of(createNode("host1", Optional.empty(), NodeType.host, Node.State.active, Optional.of(tenantHostApp)), createNode("host1-1", Optional.of("host1"), NodeType.tenant, Node.State.reserved, Optional.of(tenantApp)), @@ -157,6 +176,7 @@ public class DynamicProvisioningMaintainerTest { createNode("host2", Optional.empty(), NodeType.host, Node.State.failed, Optional.of(tenantApp)), createNode("host2-1", Optional.of("host2"), NodeType.tenant, Node.State.failed, Optional.empty()), + createNode("host3", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()), createNode("host4", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java index d1a330a3bd6..d0c678bdf45 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java @@ -152,7 +152,7 @@ public class RebalancerTest { private static class IdentityHostResourcesCalculator implements HostResourcesCalculator { @Override - public NodeResources availableCapacityOf(NodeResources hostResources) { + public NodeResources availableCapacityOf(String flavorName, NodeResources hostResources) { return hostResources; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java index 7d9ac230771..ba9a04573e1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java @@ -37,7 +37,7 @@ public class DockerHostCapacityTest { @Before public void setup() { - doAnswer(invocation -> invocation.getArguments()[0]).when(hostResourcesCalculator).availableCapacityOf(any()); + doAnswer(invocation -> invocation.getArguments()[1]).when(hostResourcesCalculator).availableCapacityOf(any(), any()); // Create flavors NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("host", "docker", "docker2"); @@ -95,9 +95,9 @@ public class DockerHostCapacityTest { capacity.freeCapacityOf(host3, false)); doAnswer(invocation -> { - NodeResources totalHostResources = (NodeResources) invocation.getArguments()[0]; + NodeResources totalHostResources = (NodeResources) invocation.getArguments()[1]; return totalHostResources.subtract(new NodeResources(1, 2, 3, 0.5, NodeResources.DiskSpeed.any)); - }).when(hostResourcesCalculator).availableCapacityOf(any()); + }).when(hostResourcesCalculator).availableCapacityOf(any(), any()); assertEquals(new NodeResources(4, 2, 5, 1.5, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), capacity.freeCapacityOf(host1, false)); diff --git a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/GetHostResponse.java b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/GetHostResponse.java index 3f14579dfba..2a582020bb3 100644 --- a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/GetHostResponse.java +++ b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/GetHostResponse.java @@ -18,6 +18,7 @@ public class GetHostResponse { public static final String FIELD_NAME_HOSTNAME = "hostname"; public static final String FIELD_NAME_STATE = "state"; + public static final String FIELD_NAME_SUSPENDED_SINCE = "suspendedSince"; public static final String FIELD_NAME_APPLICATION_URL = "applicationUrl"; public static final String FIELD_NAME_SERVICES = "services"; @@ -25,15 +26,18 @@ public class GetHostResponse { private final String state; private final String applicationUrl; private final List<HostService> services; + private final String suspendedSince; @JsonCreator public GetHostResponse( @JsonProperty(FIELD_NAME_HOSTNAME) String hostname, @JsonProperty(FIELD_NAME_STATE) String state, + @JsonProperty(FIELD_NAME_SUSPENDED_SINCE) String suspendedSince, @JsonProperty(FIELD_NAME_APPLICATION_URL) String applicationUrl, @JsonProperty(FIELD_NAME_SERVICES) List<HostService> services) { this.hostname = hostname; this.state = state; + this.suspendedSince = suspendedSince; this.applicationUrl = applicationUrl; this.services = services; } @@ -48,6 +52,11 @@ public class GetHostResponse { return state; } + @JsonProperty(FIELD_NAME_SUSPENDED_SINCE) + public String suspendedSince() { + return suspendedSince; + } + @JsonProperty(FIELD_NAME_APPLICATION_URL) public String applicationUrl() { return applicationUrl; @@ -65,12 +74,13 @@ public class GetHostResponse { GetHostResponse that = (GetHostResponse) o; return Objects.equals(hostname, that.hostname) && Objects.equals(state, that.state) && + Objects.equals(suspendedSince, that.suspendedSince) && Objects.equals(applicationUrl, that.applicationUrl) && Objects.equals(services, that.services); } @Override public int hashCode() { - return Objects.hash(hostname, state, applicationUrl, services); + return Objects.hash(hostname, state, suspendedSince, applicationUrl, services); } } diff --git a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/WireHostInfo.java b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/WireHostInfo.java new file mode 100644 index 00000000000..39c93291bad --- /dev/null +++ b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/WireHostInfo.java @@ -0,0 +1,38 @@ +package com.yahoo.vespa.orchestrator.restapi.wire; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.time.Instant; +import java.util.Objects; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class WireHostInfo { + private static final String HOST_STATUS_FIELD = "hostStatus"; + private static final String SUSPENDED_SINCE_FIELD = "suspendedSince"; + + private final String hostStatus; + private final String suspendedSinceUtcOrNull; + + /** + * @param hostStatus The host status, e.g. NO_REMARKS. + * @param suspendedSinceUtcOrNull The time the host was suspended in the format + * {@link Instant#toString()}, or null if not suspended + * (NO_REMARKS). + */ + @JsonCreator + public WireHostInfo(@JsonProperty(HOST_STATUS_FIELD) String hostStatus, + @JsonProperty(SUSPENDED_SINCE_FIELD) String suspendedSinceUtcOrNull) { + this.hostStatus = Objects.requireNonNull(hostStatus); + this.suspendedSinceUtcOrNull = suspendedSinceUtcOrNull; + } + + @JsonProperty(HOST_STATUS_FIELD) + public String hostStatus() { return hostStatus; } + + @JsonProperty(SUSPENDED_SINCE_FIELD) + public String getSuspendedSinceUtcOrNull() { return suspendedSinceUtcOrNull; } +} diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Host.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Host.java index bda9505d72b..8f5f00af7a0 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Host.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Host.java @@ -4,23 +4,23 @@ package com.yahoo.vespa.orchestrator; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceInstance; -import com.yahoo.vespa.orchestrator.status.HostStatus; +import com.yahoo.vespa.orchestrator.status.HostInfo; import java.util.List; public class Host { private final HostName hostName; - private final HostStatus hostStatus; + private final HostInfo hostInfo; private final ApplicationInstanceReference applicationInstanceReference; private final List<ServiceInstance> serviceInstances; public Host(HostName hostName, - HostStatus hostStatus, + HostInfo hostInfo, ApplicationInstanceReference applicationInstanceReference, List<ServiceInstance> serviceInstances) { this.hostName = hostName; - this.hostStatus = hostStatus; + this.hostInfo = hostInfo; this.applicationInstanceReference = applicationInstanceReference; this.serviceInstances = serviceInstances; } @@ -29,8 +29,8 @@ public class Host { return hostName; } - public HostStatus getHostStatus() { - return hostStatus; + public HostInfo getHostInfo() { + return hostInfo; } public ApplicationInstanceReference getApplicationInstanceReference() { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java index 414548f8bdc..fbe6864274c 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java @@ -114,9 +114,10 @@ public class OrchestratorImpl implements Orchestrator { .filter(serviceInstance -> hostName.equals(serviceInstance.hostName())) .collect(Collectors.toList()); + HostInfo hostInfo = statusService.getHostInfo(applicationInstance.reference(), hostName); HostStatus hostStatus = getNodeStatus(applicationInstance.reference(), hostName); - return new Host(hostName, hostStatus, applicationInstance.reference(), serviceInstances); + return new Host(hostName, hostInfo, applicationInstance.reference(), serviceInstances); } @Override diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java index 4bb93ffa3cb..fc5c5eb5004 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java @@ -31,6 +31,7 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; import java.net.URI; +import java.time.Instant; import java.util.List; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -72,7 +73,8 @@ public class HostResource implements HostApi { return new GetHostResponse( host.getHostName().s(), - host.getHostStatus().name(), + host.getHostInfo().status().name(), + host.getHostInfo().suspendedSince().map(Instant::toString).orElse(null), applicationUri.toString(), hostServices); } catch (UncheckedTimeoutException e) { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java index 7b8a74d7fe2..fbb8f445db0 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java @@ -14,6 +14,8 @@ import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.orchestrator.InstanceLookupService; import com.yahoo.vespa.orchestrator.OrchestratorUtil; import com.yahoo.vespa.orchestrator.restapi.wire.SlobrokEntryResponse; +import com.yahoo.vespa.orchestrator.restapi.wire.WireHostInfo; +import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.orchestrator.status.HostInfos; import com.yahoo.vespa.orchestrator.status.StatusService; import com.yahoo.vespa.service.manager.MonitorManager; @@ -29,9 +31,10 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import java.time.Instant; import java.util.List; -import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.stream.Collectors; import static com.yahoo.vespa.orchestrator.OrchestratorUtil.getHostsUsedByApplicationInstance; @@ -82,13 +85,23 @@ public class InstanceResource { .orElseThrow(() -> new WebApplicationException(Response.status(Response.Status.NOT_FOUND).build())); HostInfos hostInfos = statusService.getHostInfosByApplicationResolver().apply(applicationInstance.reference()); - Map<HostName, String> hostStatusMap = getHostsUsedByApplicationInstance(applicationInstance) - .stream() - .collect(Collectors.toMap(hostName -> hostName, - hostName -> hostInfos.getOrNoRemarks(hostName).status().asString())); + TreeMap<HostName, WireHostInfo> hostStatusMap = + getHostsUsedByApplicationInstance(applicationInstance) + .stream() + .collect(Collectors.toMap( + hostName -> hostName, + hostName -> hostInfoToWire(hostInfos.getOrNoRemarks(hostName)), + (u, v) -> { throw new IllegalStateException(); }, + TreeMap::new)); return InstanceStatusResponse.create(applicationInstance, hostStatusMap); } + private WireHostInfo hostInfoToWire(HostInfo hostInfo) { + String hostStatusString = hostInfo.status().asString(); + String suspendedSinceUtcOrNull = hostInfo.suspendedSince().map(Instant::toString).orElse(null); + return new WireHostInfo(hostStatusString, suspendedSinceUtcOrNull); + } + @GET @Path("/{instanceId}/slobrok") @Produces(MediaType.APPLICATION_JSON) diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java index 068423f7d24..313c73e5c68 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java @@ -4,10 +4,12 @@ package com.yahoo.vespa.orchestrator.resources; import com.fasterxml.jackson.annotation.JsonProperty; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.orchestrator.status.HostInfo; +import com.yahoo.vespa.orchestrator.restapi.wire.WireHostInfo; import java.util.Map; import java.util.Objects; +import java.util.TreeMap; +import java.util.stream.Collectors; /* * @author andreer @@ -15,16 +17,16 @@ import java.util.Objects; public class InstanceStatusResponse { private final ApplicationInstance applicationInstance; - private final Map<HostName, String> hostStates; + private final TreeMap<HostName, WireHostInfo> hostInfos; - private InstanceStatusResponse(ApplicationInstance applicationInstance, Map<HostName, String> hostStates) { + private InstanceStatusResponse(ApplicationInstance applicationInstance, TreeMap<HostName, WireHostInfo> hostInfos) { this.applicationInstance = applicationInstance; - this.hostStates = hostStates; + this.hostInfos = hostInfos; } public static InstanceStatusResponse create( ApplicationInstance applicationInstance, - Map<HostName, String> hostStates) { + TreeMap<HostName, WireHostInfo> hostStates) { return new InstanceStatusResponse(applicationInstance, hostStates); } @@ -35,14 +37,24 @@ public class InstanceStatusResponse { @JsonProperty("hostStates") public Map<HostName, String> hostStates() { - return hostStates; + // TODO: Remove this once all clients have been moved to hostStatus. + return hostInfos.entrySet().stream() + .collect(Collectors.toMap( + entry -> entry.getKey(), + entry -> entry.getValue().hostStatus() + )); + } + + @JsonProperty("hostInfos") + public TreeMap<HostName, WireHostInfo> hostInfos() { + return hostInfos; } @Override public String toString() { return "InstanceStatusResponse{" + "applicationInstance=" + applicationInstance + - ", hostStates=" + hostStates + + ", hostInfos=" + hostInfos + '}'; } @@ -52,11 +64,11 @@ public class InstanceStatusResponse { if (o == null || getClass() != o.getClass()) return false; InstanceStatusResponse that = (InstanceStatusResponse) o; return Objects.equals(applicationInstance, that.applicationInstance) && - Objects.equals(hostStates, that.hostStates); + Objects.equals(hostInfos, that.hostInfos); } @Override public int hashCode() { - return Objects.hash(applicationInstance, hostStates); + return Objects.hash(applicationInstance, hostInfos); } } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java index a2c99b86ae2..77ec824da54 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java @@ -435,7 +435,8 @@ public class OrchestratorImplTest { Host host = orchestrator.getHost(hostName); assertEquals(reference, host.getApplicationInstanceReference()); assertEquals(hostName, host.getHostName()); - assertEquals(HostStatus.ALLOWED_TO_BE_DOWN, host.getHostStatus()); + assertEquals(HostStatus.ALLOWED_TO_BE_DOWN, host.getHostInfo().status()); + assertTrue(host.getHostInfo().suspendedSince().isPresent()); assertEquals(2, host.getServiceInstances().size()); } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java index b19f96a5867..dc26c1a3770 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java @@ -37,6 +37,7 @@ import com.yahoo.vespa.orchestrator.restapi.wire.GetHostResponse; import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostRequest; import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostResponse; import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse; +import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.orchestrator.status.HostStatus; import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; import com.yahoo.vespa.orchestrator.status.StatusService; @@ -348,7 +349,7 @@ public class HostResourceTest { Host host = new Host( hostName, - HostStatus.ALLOWED_TO_BE_DOWN, + HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.EPOCH), new ApplicationInstanceReference( new TenantId("tenantId"), new ApplicationInstanceId("applicationId")), @@ -358,6 +359,7 @@ public class HostResourceTest { assertEquals("https://foo.com/bar", response.applicationUrl()); assertEquals("hostname", response.hostname()); assertEquals("ALLOWED_TO_BE_DOWN", response.state()); + assertEquals("1970-01-01T00:00:00Z", response.suspendedSince()); assertEquals(1, response.services().size()); assertEquals("clusterId", response.services().get(0).clusterId); assertEquals("configId", response.services().get(0).configId); diff --git a/parent/pom.xml b/parent/pom.xml index b1ca2539ef5..68ee698e7a5 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -735,8 +735,8 @@ <properties> <antlr.version>3.5.2</antlr.version> <antlr4.version>4.5</antlr4.version> - <apache.httpclient.version>4.5.10</apache.httpclient.version> - <apache.httpcore.version>4.4.12</apache.httpcore.version> + <apache.httpclient.version>4.5.11</apache.httpclient.version> + <apache.httpcore.version>4.4.13</apache.httpcore.version> <asm.version>7.0</asm.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> <athenz.version>1.8.44</athenz.version> diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.cpp b/searchcommon/src/vespa/searchcommon/attribute/config.cpp index 53e57fd9c66..b4e05875820 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/config.cpp +++ b/searchcommon/src/vespa/searchcommon/attribute/config.cpp @@ -17,7 +17,8 @@ Config::Config() : _growStrategy(), _compactionStrategy(), _predicateParams(), - _tensorType(vespalib::eval::ValueType::error_type()) + _tensorType(vespalib::eval::ValueType::error_type()), + _hnsw_index_params() { } @@ -34,7 +35,8 @@ Config::Config(BasicType bt, CollectionType ct, bool fastSearch_, bool huge_) _growStrategy(), _compactionStrategy(), _predicateParams(), - _tensorType(vespalib::eval::ValueType::error_type()) + _tensorType(vespalib::eval::ValueType::error_type()), + _hnsw_index_params() { } @@ -60,7 +62,8 @@ Config::operator==(const Config &b) const _compactionStrategy == b._compactionStrategy && _predicateParams == b._predicateParams && (_basicType.type() != BasicType::Type::TENSOR || - _tensorType == b._tensorType); + _tensorType == b._tensorType) && + _hnsw_index_params == b._hnsw_index_params; } } diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.h b/searchcommon/src/vespa/searchcommon/attribute/config.h index 2f767061f7a..836fcfed84a 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/config.h +++ b/searchcommon/src/vespa/searchcommon/attribute/config.h @@ -4,15 +4,21 @@ #include "basictype.h" #include "collectiontype.h" +#include "hnsw_index_params.h" #include "predicate_params.h" -#include <vespa/searchcommon/common/growstrategy.h> #include <vespa/searchcommon/common/compaction_strategy.h> +#include <vespa/searchcommon/common/growstrategy.h> #include <vespa/eval/eval/value_type.h> +#include <optional> namespace search::attribute { -class Config -{ +/** + * Configuration for an attribute vector. + * + * Used to determine which implementation to instantiate. + */ +class Config { public: Config(); Config(BasicType bt, CollectionType ct = CollectionType::SINGLE, @@ -29,6 +35,7 @@ public: bool huge() const { return _huge; } const PredicateParams &predicateParams() const { return _predicateParams; } vespalib::eval::ValueType tensorType() const { return _tensorType; } + const std::optional<HnswIndexParams>& hnsw_index_params() const { return _hnsw_index_params; } /** * Check if attribute posting list can consist of a bitvector in @@ -60,6 +67,10 @@ public: _tensorType = tensorType_in; return *this; } + Config& set_hnsw_index_params(const HnswIndexParams& params) { + _hnsw_index_params = params; + return *this; + } /** * Enable attribute posting list to consist of a bitvector in @@ -107,6 +118,7 @@ private: CompactionStrategy _compactionStrategy; PredicateParams _predicateParams; vespalib::eval::ValueType _tensorType; + std::optional<HnswIndexParams> _hnsw_index_params; }; } diff --git a/searchcommon/src/vespa/searchcommon/attribute/hnsw_index_params.h b/searchcommon/src/vespa/searchcommon/attribute/hnsw_index_params.h new file mode 100644 index 00000000000..9e98a8c5fb7 --- /dev/null +++ b/searchcommon/src/vespa/searchcommon/attribute/hnsw_index_params.h @@ -0,0 +1,32 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace search::attribute { + +/** + * Configuration parameters for a hnsw index used together with a 1-dimensional indexed tensor + * for approximate nearest neighbor search. + */ +class HnswIndexParams { +private: + uint32_t _max_links_per_node; + uint32_t _neighbors_to_explore_at_insert; + +public: + HnswIndexParams(uint32_t max_links_per_node_in, + uint32_t neighbors_to_explore_at_insert_in) + : _max_links_per_node(max_links_per_node_in), + _neighbors_to_explore_at_insert(neighbors_to_explore_at_insert_in) + {} + + uint32_t max_links_per_node() const { return _max_links_per_node; } + uint32_t neighbors_to_explore_at_insert() const { return _neighbors_to_explore_at_insert; } + + bool operator==(const HnswIndexParams& rhs) const { + return _max_links_per_node == rhs._max_links_per_node && + _neighbors_to_explore_at_insert == rhs._neighbors_to_explore_at_insert; + } +}; + +} diff --git a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp index 7d09b2aa0b8..850a967ed3d 100644 --- a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp +++ b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp @@ -278,6 +278,22 @@ AttributeManagerTest::testConfigConvert() AttributeVector::Config out = ConfigConverter::convert(a); EXPECT_EQUAL("tensor(x[5])", out.tensorType().to_spec()); } + { // hnsw index params (enabled) + CACA a; + a.index.hnsw.enabled = true; + a.index.hnsw.maxlinkspernode = 32; + a.index.hnsw.neighborstoexploreatinsert = 300; + auto out = ConfigConverter::convert(a); + EXPECT_TRUE(out.hnsw_index_params().has_value()); + EXPECT_EQUAL(32u, out.hnsw_index_params().value().max_links_per_node()); + EXPECT_EQUAL(300u, out.hnsw_index_params().value().neighbors_to_explore_at_insert()); + } + { // hnsw index params (disabled) + CACA a; + a.index.hnsw.enabled = false; + auto out = ConfigConverter::convert(a); + EXPECT_FALSE(out.hnsw_index_params().has_value()); + } } bool gt_attribute(const attribute::IAttributeVector * a, const attribute::IAttributeVector * b) { diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 7e0fcdc0ccc..5089743a54a 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -1,34 +1,48 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/base/exceptions.h> -#include <vespa/searchlib/tensor/tensor_attribute.h> -#include <vespa/searchlib/tensor/generic_tensor_attribute.h> -#include <vespa/searchlib/tensor/dense_tensor_attribute.h> -#include <vespa/searchlib/attribute/attributeguard.h> -#include <vespa/eval/tensor/tensor.h> -#include <vespa/eval/tensor/dense/dense_tensor.h> #include <vespa/eval/tensor/default_tensor_engine.h> -#include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib/data/fileheader.h> +#include <vespa/eval/tensor/dense/dense_tensor.h> +#include <vespa/eval/tensor/tensor.h> #include <vespa/fastos/file.h> +#include <vespa/searchlib/attribute/attributeguard.h> +#include <vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h> +#include <vespa/searchlib/tensor/dense_tensor_attribute.h> +#include <vespa/searchlib/tensor/doc_vector_access.h> +#include <vespa/searchlib/tensor/generic_tensor_attribute.h> +#include <vespa/searchlib/tensor/hnsw_index.h> +#include <vespa/searchlib/tensor/nearest_neighbor_index.h> +#include <vespa/searchlib/tensor/nearest_neighbor_index_factory.h> +#include <vespa/searchlib/tensor/tensor_attribute.h> +#include <vespa/vespalib/data/fileheader.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/vespalib/testkit/test_kit.h> + #include <vespa/log/log.h> LOG_SETUP("tensorattribute_test"); using document::WrongTensorTypeException; -using search::tensor::TensorAttribute; -using search::tensor::DenseTensorAttribute; -using search::tensor::GenericTensorAttribute; using search::AttributeGuard; using search::AttributeVector; -using vespalib::eval::ValueType; +using search::attribute::HnswIndexParams; +using search::tensor::DefaultNearestNeighborIndexFactory; +using search::tensor::DenseTensorAttribute; +using search::tensor::DocVectorAccess; +using search::tensor::GenericTensorAttribute; +using search::tensor::HnswIndex; +using search::tensor::NearestNeighborIndex; +using search::tensor::NearestNeighborIndexFactory; +using search::tensor::TensorAttribute; using vespalib::eval::TensorSpec; -using vespalib::tensor::Tensor; -using vespalib::tensor::DenseTensor; +using vespalib::eval::ValueType; using vespalib::tensor::DefaultTensorEngine; +using vespalib::tensor::DenseTensor; +using vespalib::tensor::Tensor; + +using DoubleVector = std::vector<double>; -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { static bool operator==(const Tensor &lhs, const Tensor &rhs) { @@ -36,10 +50,10 @@ static bool operator==(const Tensor &lhs, const Tensor &rhs) } } -} vespalib::string sparseSpec("tensor(x{},y{})"); vespalib::string denseSpec("tensor(x[2],y[3])"); +vespalib::string vec_2d_spec("tensor(x[2])"); Tensor::UP createTensor(const TensorSpec &spec) { auto value = DefaultTensorEngine::ref().from_spec(spec); @@ -52,6 +66,78 @@ Tensor::UP createTensor(const TensorSpec &spec) { return Tensor::UP(tensor); } +TensorSpec +vec_2d(double x0, double x1) +{ + return TensorSpec(vec_2d_spec).add({{"x", 0}}, x0).add({{"x", 1}}, x1); +} + +class MockNearestNeighborIndex : public NearestNeighborIndex { +private: + using Entry = std::pair<uint32_t, DoubleVector>; + using EntryVector = std::vector<Entry>; + + const DocVectorAccess& _vectors; + EntryVector _adds; + EntryVector _removes; + +public: + MockNearestNeighborIndex(const DocVectorAccess& vectors) + : _vectors(vectors), + _adds(), + _removes() + { + } + void clear() { + _adds.clear(); + _removes.clear(); + } + void expect_empty_add() const { + EXPECT_TRUE(_adds.empty()); + } + void expect_add(uint32_t exp_docid, const DoubleVector& exp_vector) const { + EXPECT_EQUAL(1u, _adds.size()); + EXPECT_EQUAL(exp_docid, _adds.back().first); + EXPECT_EQUAL(exp_vector, _adds.back().second); + } + void expect_adds(const EntryVector &exp_adds) const { + EXPECT_EQUAL(exp_adds, _adds); + } + void expect_empty_remove() const { + EXPECT_TRUE(_removes.empty()); + } + void expect_remove(uint32_t exp_docid, const DoubleVector& exp_vector) const { + EXPECT_EQUAL(1u, _removes.size()); + EXPECT_EQUAL(exp_docid, _removes.back().first); + EXPECT_EQUAL(exp_vector, _removes.back().second); + } + void add_document(uint32_t docid) override { + auto vector = _vectors.get_vector(docid).typify<double>(); + _adds.emplace_back(docid, DoubleVector(vector.begin(), vector.end())); + } + void remove_document(uint32_t docid) override { + auto vector = _vectors.get_vector(docid).typify<double>(); + _removes.emplace_back(docid, DoubleVector(vector.begin(), vector.end())); + } + std::vector<Neighbor> find_top_k(uint32_t k, vespalib::tensor::TypedCells vector, uint32_t explore_k) const override { + (void) k; + (void) vector; + (void) explore_k; + return std::vector<Neighbor>(); + } +}; + +class MockNearestNeighborIndexFactory : public NearestNeighborIndexFactory { + + std::unique_ptr<NearestNeighborIndex> make(const DocVectorAccess& vectors, + ValueType::CellType cell_type, + const search::attribute::HnswIndexParams& params) const override { + (void) params; + assert(cell_type == ValueType::CellType::DOUBLE); + return std::make_unique<MockNearestNeighborIndex>(vectors); + } +}; + struct Fixture { using BasicType = search::attribute::BasicType; @@ -61,16 +147,20 @@ struct Fixture Config _cfg; vespalib::string _name; vespalib::string _typeSpec; + std::unique_ptr<NearestNeighborIndexFactory> _index_factory; std::shared_ptr<TensorAttribute> _tensorAttr; std::shared_ptr<AttributeVector> _attr; bool _denseTensors; bool _useDenseTensorAttribute; Fixture(const vespalib::string &typeSpec, - bool useDenseTensorAttribute = false) + bool useDenseTensorAttribute = false, + bool enable_hnsw_index = false, + bool use_mock_index = false) : _cfg(BasicType::TENSOR, CollectionType::SINGLE), _name("test"), _typeSpec(typeSpec), + _index_factory(std::make_unique<DefaultNearestNeighborIndexFactory>()), _tensorAttr(), _attr(), _denseTensors(false), @@ -80,20 +170,40 @@ struct Fixture if (_cfg.tensorType().is_dense()) { _denseTensors = true; } + if (enable_hnsw_index) { + _cfg.set_hnsw_index_params(HnswIndexParams(4, 20)); + if (use_mock_index) { + _index_factory = std::make_unique<MockNearestNeighborIndexFactory>(); + } + } _tensorAttr = makeAttr(); _attr = _tensorAttr; _attr->addReservedDoc(); } + ~Fixture() {} std::shared_ptr<TensorAttribute> makeAttr() { if (_useDenseTensorAttribute) { assert(_denseTensors); - return std::make_shared<DenseTensorAttribute>(_name, _cfg); + return std::make_shared<DenseTensorAttribute>(_name, _cfg, *_index_factory); } else { return std::make_shared<GenericTensorAttribute>(_name, _cfg); } } + const DenseTensorAttribute& as_dense_tensor() const { + auto result = dynamic_cast<const DenseTensorAttribute*>(_tensorAttr.get()); + assert(result != nullptr); + return *result; + } + + MockNearestNeighborIndex& mock_index() { + assert(as_dense_tensor().nearest_neighbor_index() != nullptr); + auto mock_index = dynamic_cast<const MockNearestNeighborIndex*>(as_dense_tensor().nearest_neighbor_index()); + assert(mock_index != nullptr); + return *const_cast<MockNearestNeighborIndex*>(mock_index); + } + void ensureSpace(uint32_t docId) { while (_attr->getNumDocs() <= docId) { uint32_t newDocId = 0u; @@ -108,7 +218,15 @@ struct Fixture _attr->commit(); } - void setTensor(uint32_t docId, const Tensor &tensor) { + void set_tensor(uint32_t docid, const TensorSpec &spec) { + set_tensor_internal(docid, *createTensor(spec)); + } + + void set_empty_tensor(uint32_t docid) { + set_tensor_internal(docid, *_tensorAttr->getEmptyTensor()); + } + + void set_tensor_internal(uint32_t docId, const Tensor &tensor) { ensureSpace(docId); _tensorAttr->setTensor(docId, tensor); _attr->commit(); @@ -119,27 +237,18 @@ struct Fixture return _attr->getStatus(); } - void - assertGetNoTensor(uint32_t docId) { + void assertGetNoTensor(uint32_t docId) { AttributeGuard guard(_attr); Tensor::UP actTensor = _tensorAttr->getTensor(docId); EXPECT_FALSE(actTensor); } - void - assertGetTensor(const Tensor &expTensor, uint32_t docId) - { + void assertGetTensor(const TensorSpec &expSpec, uint32_t docId) { + Tensor::UP expTensor = createTensor(expSpec); AttributeGuard guard(_attr); Tensor::UP actTensor = _tensorAttr->getTensor(docId); EXPECT_TRUE(static_cast<bool>(actTensor)); - EXPECT_EQUAL(expTensor, *actTensor); - } - - void - assertGetTensor(const TensorSpec &expSpec, uint32_t docId) - { - Tensor::UP expTensor = createTensor(expSpec); - assertGetTensor(*expTensor, docId); + EXPECT_EQUAL(*expTensor, *actTensor); } void save() { @@ -154,23 +263,20 @@ struct Fixture EXPECT_TRUE(loadok); } - Tensor::UP expDenseTensor3() const - { - return createTensor(TensorSpec(denseSpec) - .add({{"x", 0}, {"y", 1}}, 11) - .add({{"x", 1}, {"y", 2}}, 0)); + TensorSpec expDenseTensor3() const { + return TensorSpec(denseSpec) + .add({{"x", 0}, {"y", 1}}, 11) + .add({{"x", 1}, {"y", 2}}, 0); } - Tensor::UP expDenseFillTensor() const - { - return createTensor(TensorSpec(denseSpec) - .add({{"x", 0}, {"y", 0}}, 5) - .add({{"x", 1}, {"y", 2}}, 0)); + TensorSpec expDenseFillTensor() const { + return TensorSpec(denseSpec) + .add({{"x", 0}, {"y", 0}}, 5) + .add({{"x", 1}, {"y", 2}}, 0); } - Tensor::UP expEmptyDenseTensor() const - { - return createTensor(TensorSpec(denseSpec)); + TensorSpec expEmptyDenseTensor() const { + return TensorSpec(denseSpec); } vespalib::string expEmptyDenseTensorSpec() const { @@ -200,21 +306,21 @@ Fixture::testSetTensorValue() EXPECT_EQUAL(5u, _attr->getNumDocs()); EXPECT_EQUAL(5u, _attr->getCommittedDocIdLimit()); TEST_DO(assertGetNoTensor(4)); - EXPECT_EXCEPTION(setTensor(4, *createTensor(TensorSpec("double"))), + EXPECT_EXCEPTION(set_tensor(4, TensorSpec("double")), WrongTensorTypeException, "but other tensor type is 'double'"); TEST_DO(assertGetNoTensor(4)); - setTensor(4, *_tensorAttr->getEmptyTensor()); + set_empty_tensor(4); if (_denseTensors) { - TEST_DO(assertGetTensor(*expEmptyDenseTensor(), 4)); - setTensor(3, *expDenseTensor3()); - TEST_DO(assertGetTensor(*expDenseTensor3(), 3)); + TEST_DO(assertGetTensor(expEmptyDenseTensor(), 4)); + set_tensor(3, expDenseTensor3()); + TEST_DO(assertGetTensor(expDenseTensor3(), 3)); } else { TEST_DO(assertGetTensor(TensorSpec(sparseSpec), 4)); - setTensor(3, *createTensor(TensorSpec(sparseSpec) - .add({{"x", ""}, {"y", ""}}, 11))); + set_tensor(3, TensorSpec(sparseSpec) + .add({{"x", ""}, {"y", ""}}, 11)); TEST_DO(assertGetTensor(TensorSpec(sparseSpec) - .add({{"x", ""}, {"y", ""}}, 11), 3)); + .add({{"x", ""}, {"y", ""}}, 11), 3)); } TEST_DO(assertGetNoTensor(2)); TEST_DO(clearTensor(3)); @@ -225,23 +331,23 @@ void Fixture::testSaveLoad() { ensureSpace(4); - setTensor(4, *_tensorAttr->getEmptyTensor()); + set_empty_tensor(4); if (_denseTensors) { - setTensor(3, *expDenseTensor3()); + set_tensor(3, expDenseTensor3()); } else { - setTensor(3, *createTensor(TensorSpec(sparseSpec) - .add({{"x", ""}, {"y", "1"}}, 11))); + set_tensor(3, TensorSpec(sparseSpec) + .add({{"x", ""}, {"y", "1"}}, 11)); } TEST_DO(save()); TEST_DO(load()); EXPECT_EQUAL(5u, _attr->getNumDocs()); EXPECT_EQUAL(5u, _attr->getCommittedDocIdLimit()); if (_denseTensors) { - TEST_DO(assertGetTensor(*expDenseTensor3(), 3)); - TEST_DO(assertGetTensor(*expEmptyDenseTensor(), 4)); + TEST_DO(assertGetTensor(expDenseTensor3(), 3)); + TEST_DO(assertGetTensor(expEmptyDenseTensor(), 4)); } else { TEST_DO(assertGetTensor(TensorSpec(sparseSpec) - .add({{"x", ""}, {"y", "1"}}, 11), 3)); + .add({{"x", ""}, {"y", "1"}}, 11), 3)); TEST_DO(assertGetTensor(TensorSpec(sparseSpec), 4)); } TEST_DO(assertGetNoTensor(2)); @@ -256,29 +362,28 @@ Fixture::testCompaction() return; } ensureSpace(4); - Tensor::UP emptytensor = _tensorAttr->getEmptyTensor(); - Tensor::UP emptyxytensor = createTensor(TensorSpec(sparseSpec)); - Tensor::UP simpletensor = createTensor(TensorSpec(sparseSpec) - .add({{"x", ""}, {"y", "1"}}, 11)); - Tensor::UP filltensor = createTensor(TensorSpec(sparseSpec) - .add({{"x", ""}, {"y", ""}}, 5)); + TensorSpec empty_xy_tensor(sparseSpec); + TensorSpec simple_tensor = TensorSpec(sparseSpec) + .add({{"x", ""}, {"y", "1"}}, 11); + TensorSpec fill_tensor = TensorSpec(sparseSpec) + .add({{"x", ""}, {"y", ""}}, 5); if (_denseTensors) { - emptyxytensor = expEmptyDenseTensor(); - simpletensor = expDenseTensor3(); - filltensor = expDenseFillTensor(); + empty_xy_tensor = expEmptyDenseTensor(); + simple_tensor = expDenseTensor3(); + fill_tensor = expDenseFillTensor(); } - setTensor(4, *emptytensor); - setTensor(3, *simpletensor); - setTensor(2, *filltensor); + set_empty_tensor(4); + set_tensor(3, simple_tensor); + set_tensor(2, fill_tensor); clearTensor(2); - setTensor(2, *filltensor); + set_tensor(2, fill_tensor); search::attribute::Status oldStatus = getStatus(); search::attribute::Status newStatus = oldStatus; uint64_t iter = 0; uint64_t iterLimit = 100000; for (; iter < iterLimit; ++iter) { clearTensor(2); - setTensor(2, *filltensor); + set_tensor(2, fill_tensor); newStatus = getStatus(); if (newStatus.getUsed() < oldStatus.getUsed()) { break; @@ -290,9 +395,9 @@ Fixture::testCompaction() "iter = %" PRIu64 ", memory usage %" PRIu64 ", -> %" PRIu64, iter, oldStatus.getUsed(), newStatus.getUsed()); TEST_DO(assertGetNoTensor(1)); - TEST_DO(assertGetTensor(*filltensor, 2)); - TEST_DO(assertGetTensor(*simpletensor, 3)); - TEST_DO(assertGetTensor(*emptyxytensor, 4)); + TEST_DO(assertGetTensor(fill_tensor, 2)); + TEST_DO(assertGetTensor(simple_tensor, 3)); + TEST_DO(assertGetTensor(empty_xy_tensor, 4)); } void @@ -357,4 +462,73 @@ TEST("Test dense tensors with dense tensor attribute") testAll([]() { return std::make_shared<Fixture>(denseSpec, true); }); } +TEST_F("Hnsw index is NOT instantiated in dense tensor attribute by default", + Fixture(vec_2d_spec, true, false)) +{ + const auto& tensor = f.as_dense_tensor(); + EXPECT_TRUE(tensor.nearest_neighbor_index() == nullptr); +} + +TEST_F("Hnsw index is instantiated in dense tensor attribute when specified in config", + Fixture(vec_2d_spec, true, true)) +{ + const auto& tensor = f.as_dense_tensor(); + ASSERT_TRUE(tensor.nearest_neighbor_index() != nullptr); + auto hnsw_index = dynamic_cast<const HnswIndex*>(tensor.nearest_neighbor_index()); + ASSERT_TRUE(hnsw_index != nullptr); + + const auto& cfg = hnsw_index->config(); + EXPECT_EQUAL(8u, cfg.max_links_at_level_0()); + EXPECT_EQUAL(4u, cfg.max_links_at_hierarchic_levels()); + EXPECT_EQUAL(20u, cfg.neighbors_to_explore_at_construction()); + EXPECT_TRUE(cfg.heuristic_select_neighbors()); +} + +class DenseTensorAttributeMockIndex : public Fixture { +public: + DenseTensorAttributeMockIndex() : Fixture(vec_2d_spec, true, true, true) {} +}; + +TEST_F("setTensor() updates nearest neighbor index", DenseTensorAttributeMockIndex) +{ + auto& index = f.mock_index(); + + f.set_tensor(1, vec_2d(3, 5)); + index.expect_add(1, {3, 5}); + index.expect_empty_remove(); + index.clear(); + + // Replaces previous value. + f.set_tensor(1, vec_2d(7, 9)); + index.expect_remove(1, {3, 5}); + index.expect_add(1, {7, 9}); +} + +TEST_F("clearDoc() updates nearest neighbor index", DenseTensorAttributeMockIndex) +{ + auto& index = f.mock_index(); + + // Nothing to clear. + f.clearTensor(1); + index.expect_empty_remove(); + index.expect_empty_add(); + + // Clears previous value. + f.set_tensor(1, vec_2d(3, 5)); + index.clear(); + f.clearTensor(1); + index.expect_remove(1, {3, 5}); + index.expect_empty_add(); +} + +TEST_F("onLoad() updates nearest neighbor index", DenseTensorAttributeMockIndex) +{ + f.set_tensor(1, vec_2d(3, 5)); + f.set_tensor(2, vec_2d(7, 9)); + f.save(); + f.load(); + auto& index = f.mock_index(); + index.expect_adds({{1, {3, 5}}, {2, {7, 9}}}); +} + TEST_MAIN() { TEST_RUN_ALL(); vespalib::unlink("test.dat"); } diff --git a/searchlib/src/tests/fef/resolver/resolver_test.cpp b/searchlib/src/tests/fef/resolver/resolver_test.cpp index 955c072810f..7591d028620 100644 --- a/searchlib/src/tests/fef/resolver/resolver_test.cpp +++ b/searchlib/src/tests/fef/resolver/resolver_test.cpp @@ -4,10 +4,13 @@ #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/fef/test/indexenvironment.h> #include <vespa/searchlib/features/valuefeature.h> +#include <vespa/searchlib/features/rankingexpressionfeature.h> #include <vespa/log/log.h> LOG_SETUP("resolver_test"); +using search::features::RankingExpressionBlueprint; + namespace search { namespace fef { @@ -58,6 +61,7 @@ class Test : public vespalib::TestApp { private: BlueprintFactory _factory; void requireThatWeGetUniqueBlueprints(); + void require_that_bad_input_is_handled(); public: Test(); ~Test(); @@ -69,6 +73,7 @@ Test::Test() : { _factory.addPrototype(Blueprint::SP(new BaseBlueprint())); _factory.addPrototype(Blueprint::SP(new CombineBlueprint())); + _factory.addPrototype(std::make_shared<RankingExpressionBlueprint>()); } Test::~Test() {} @@ -85,12 +90,28 @@ Test::requireThatWeGetUniqueBlueprints() EXPECT_TRUE(dynamic_cast<CombineBlueprint *>(spec[1].blueprint.get()) != NULL); } +void +Test::require_that_bad_input_is_handled() +{ + test::IndexEnvironment ienv; + ienv.getProperties().add(indexproperties::eval::LazyExpressions::NAME, "false"); + ienv.getProperties().add("rankingExpression(badinput).rankingScript", "base.foobad + base.bar"); + BlueprintResolver::SP res(new BlueprintResolver(_factory, ienv)); + res->addSeed("rankingExpression(badinput)"); + EXPECT_FALSE(res->compile()); + const BlueprintResolver::ExecutorSpecList & spec = res->getExecutorSpecs(); + EXPECT_EQUAL(2u, spec.size()); + EXPECT_TRUE(dynamic_cast<BaseBlueprint *>(spec[0].blueprint.get()) != nullptr); + EXPECT_TRUE(dynamic_cast<RankingExpressionBlueprint *>(spec[1].blueprint.get()) != nullptr); +} + int Test::Main() { TEST_INIT("resolver_test"); requireThatWeGetUniqueBlueprints(); + require_that_bad_input_is_handled(); TEST_DONE(); } diff --git a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp index 7bc582ab442..691e80aeb9f 100644 --- a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp +++ b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp @@ -12,6 +12,7 @@ #include <vespa/searchlib/queryeval/simpleresult.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/searchlib/queryeval/nns_index_iterator.h> #include <vespa/log/log.h> LOG_SETUP("nearest_neighbor_test"); @@ -190,4 +191,70 @@ TEST("require that NearestNeighborIterator sets expected rawscore") { TEST_DO(verify_iterator_sets_expected_rawscore(denseSpecFloat, denseSpecDouble)); } +TEST("require that NnsIndexIterator works as expected") { + std::vector<NnsIndexIterator::Hit> hits{{2,4.0}, {3,9.0}, {5,1.0}, {8,16.0}, {9,36.0}}; + auto md = MatchData::makeTestInstance(2, 2); + auto &tfmd = *(md->resolveTermField(0)); + auto search = NnsIndexIterator::create(true, tfmd, hits); + uint32_t docid = 1; + search->initFullRange(); + bool match = search->seek(docid); + EXPECT_FALSE(match); + EXPECT_FALSE(search->isAtEnd()); + EXPECT_EQUAL(2u, search->getDocId()); + docid = 2; + match = search->seek(docid); + EXPECT_TRUE(match); + EXPECT_FALSE(search->isAtEnd()); + EXPECT_EQUAL(docid, search->getDocId()); + search->unpack(docid); + EXPECT_EQUAL(2.0, tfmd.getRawScore()); + + docid = 3; + match = search->seek(docid); + EXPECT_TRUE(match); + EXPECT_FALSE(search->isAtEnd()); + EXPECT_EQUAL(docid, search->getDocId()); + search->unpack(docid); + EXPECT_EQUAL(3.0, tfmd.getRawScore()); + + docid = 4; + match = search->seek(docid); + EXPECT_FALSE(match); + EXPECT_FALSE(search->isAtEnd()); + EXPECT_EQUAL(5u, search->getDocId()); + + docid = 6; + match = search->seek(docid); + EXPECT_FALSE(match); + EXPECT_FALSE(search->isAtEnd()); + EXPECT_EQUAL(8u, search->getDocId()); + docid = 8; + search->unpack(docid); + EXPECT_EQUAL(4.0, tfmd.getRawScore()); + docid = 9; + match = search->seek(docid); + EXPECT_TRUE(match); + EXPECT_FALSE(search->isAtEnd()); + docid = 10; + match = search->seek(docid); + EXPECT_FALSE(match); + EXPECT_TRUE(search->isAtEnd()); + + docid = 4; + search->initRange(docid, 7); + match = search->seek(docid); + EXPECT_FALSE(match); + EXPECT_FALSE(search->isAtEnd()); + EXPECT_EQUAL(5u, search->getDocId()); + docid = 5; + search->unpack(docid); + EXPECT_EQUAL(1.0, tfmd.getRawScore()); + EXPECT_FALSE(search->isAtEnd()); + docid = 6; + match = search->seek(docid); + EXPECT_FALSE(match); + EXPECT_TRUE(search->isAtEnd()); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp index 081361b2fdc..1204ae1e9bc 100644 --- a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -48,8 +48,7 @@ using HnswIndexUP = std::unique_ptr<HnswIndex>; class HnswIndexTest : public ::testing::Test { public: FloatVectors vectors; - FloatSqEuclideanDistance distance_func; - LevelGenerator level_generator; + LevelGenerator* level_generator; HnswIndexUP index; HnswIndexTest() @@ -59,14 +58,17 @@ public: { vectors.set(1, {2, 2}).set(2, {3, 2}).set(3, {2, 3}) .set(4, {1, 2}).set(5, {8, 3}).set(6, {7, 2}) - .set(7, {3, 5}); + .set(7, {3, 5}).set(8, {0, 3}).set(9, {4, 5}); } void init(bool heuristic_select_neighbors) { - index = std::make_unique<HnswIndex>(vectors, distance_func, level_generator, + auto generator = std::make_unique<LevelGenerator>(); + level_generator = generator.get(); + index = std::make_unique<HnswIndex>(vectors, std::make_unique<FloatSqEuclideanDistance>(), + std::move(generator), HnswIndex::Config(2, 1, 10, heuristic_select_neighbors)); } void add_document(uint32_t docid, uint32_t max_level = 0) { - level_generator.level = max_level; + level_generator->level = max_level; index->add_document(docid); } void remove_document(uint32_t docid) { @@ -86,6 +88,26 @@ public: ASSERT_EQ(exp_levels.size(), act_node.size()); EXPECT_EQ(exp_levels, act_node.levels()); } + void expect_top_3(uint32_t docid, std::vector<uint32_t> exp_hits) { + uint32_t k = 3; + auto qv = vectors.get_vector(docid); + auto rv = index->top_k_candidates(qv, k).peek(); + std::sort(rv.begin(), rv.end(), LesserDistance()); + size_t idx = 0; + for (const auto & hit : rv) { + if (idx < exp_hits.size()) { + EXPECT_EQ(hit.docid, exp_hits[idx++]); + } + } + if (exp_hits.size() == k) { + std::vector<uint32_t> expected_by_docid = exp_hits; + std::sort(expected_by_docid.begin(), expected_by_docid.end()); + auto got_by_docid = index->find_top_k(k, qv, k); + for (idx = 0; idx < k; ++idx) { + EXPECT_EQ(expected_by_docid[idx], got_by_docid[idx].docid); + } + } + } }; @@ -134,6 +156,16 @@ TEST_F(HnswIndexTest, 2d_vectors_inserted_in_level_0_graph_with_simple_select_ne expect_level_0(5, {2, 3, 6}); expect_level_0(6, {2, 5}); expect_level_0(7, {2, 3}); + + expect_top_3(1, {1}); + expect_top_3(2, {2, 1, 3}); + expect_top_3(3, {3}); + expect_top_3(4, {4, 1, 3}); + expect_top_3(5, {5, 6, 2}); + expect_top_3(6, {6, 5, 2}); + expect_top_3(7, {7, 3, 2}); + expect_top_3(8, {4, 3, 1}); + expect_top_3(9, {7, 3, 2}); } TEST_F(HnswIndexTest, 2d_vectors_inserted_and_removed) diff --git a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp index 535e81fc032..10e1a1edb52 100644 --- a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp @@ -73,6 +73,10 @@ ConfigConverter::convert(const AttributesConfig::Attribute & cfg) predicateParams.setBounds(cfg.lowerbound, cfg.upperbound); predicateParams.setDensePostingListThreshold(cfg.densepostinglistthreshold); retval.setPredicateParams(predicateParams); + if (cfg.index.hnsw.enabled) { + retval.set_hnsw_index_params(HnswIndexParams(cfg.index.hnsw.maxlinkspernode, + cfg.index.hnsw.neighborstoexploreatinsert)); + } if (retval.basicType().type() == BasicType::Type::TENSOR) { if (!cfg.tensortype.empty()) { retval.setTensorType(ValueType::from_spec(cfg.tensortype)); diff --git a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp index cd2cd949a91..82a69009e82 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp +++ b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp @@ -85,9 +85,21 @@ struct Compiler : public Blueprint::DependencyHandler { } compile_error = true; } + fixup_feature_map(); return FeatureRef(); } + void fixup_feature_map() { + auto itr = feature_map.begin(); + while (itr != feature_map.end()) { + if (itr->second.executor >= spec_list.size()) { + itr = feature_map.erase(itr); + } else { + ++itr; + } + } + } + FeatureRef verify_type(const FeatureNameParser &parser, FeatureRef ref, Accept accept_type) { const auto &spec = spec_list[ref.executor]; bool is_object = spec.output_types[ref.output]; diff --git a/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt b/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt index de2919443ff..0dcb0393473 100644 --- a/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt @@ -32,6 +32,7 @@ vespa_add_library(searchlib_queryeval OBJECT nearest_neighbor_blueprint.cpp nearest_neighbor_iterator.cpp nearsearch.cpp + nns_index_iterator.cpp orsearch.cpp predicate_blueprint.cpp predicate_search.cpp diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp index 8be6263221a..f9bce4bf7d1 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp @@ -3,6 +3,7 @@ #include "emptysearch.h" #include "nearest_neighbor_blueprint.h" #include "nearest_neighbor_iterator.h" +#include "nns_index_iterator.h" #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/eval/tensor/dense/dense_tensor_view.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> @@ -17,20 +18,47 @@ NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& f _attr_tensor(attr_tensor), _query_tensor(std::move(query_tensor)), _target_num_hits(target_num_hits), - _distance_heap(target_num_hits) + _distance_heap(target_num_hits), + _found_hits() { setEstimate(HitEstimate(_attr_tensor.getNumDocs(), false)); } NearestNeighborBlueprint::~NearestNeighborBlueprint() = default; +void +NearestNeighborBlueprint::perform_top_k() +{ + auto nns_index = _attr_tensor.nearest_neighbor_index(); + if (nns_index) { + auto lhs_type = _query_tensor->fast_type(); + auto rhs_type = _attr_tensor.getTensorType(); + // XXX deal with different cell types later + if (lhs_type == rhs_type) { + auto lhs = _query_tensor->cellsRef(); + uint32_t k = _target_num_hits; + uint32_t explore_k = k + 100; // XXX hardcoded for now + _found_hits = nns_index->find_top_k(k, lhs, explore_k); + } + } +} + +void +NearestNeighborBlueprint::fetchPostings(const ExecuteInfo &execInfo) { + if (execInfo.isStrict()) { + perform_top_k(); + } +} + std::unique_ptr<SearchIterator> NearestNeighborBlueprint::createLeafSearch(const search::fef::TermFieldMatchDataArray& tfmda, bool strict) const { assert(tfmda.size() == 1); fef::TermFieldMatchData &tfmd = *tfmda[0]; // always search in only one field + if (strict && ! _found_hits.empty()) { + return NnsIndexIterator::create(strict, tfmd, _found_hits); + } const vespalib::tensor::DenseTensorView &qT = *_query_tensor; - return NearestNeighborIterator::create(strict, tfmd, qT, _attr_tensor, _distance_heap); } diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h index 019f8e31842..ab4413c487a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h @@ -3,6 +3,7 @@ #include "blueprint.h" #include "nearest_neighbor_distance_heap.h" +#include <vespa/searchlib/tensor/nearest_neighbor_index.h> namespace vespalib::tensor { class DenseTensorView; } namespace search::tensor { class DenseTensorAttribute; } @@ -21,7 +22,9 @@ private: std::unique_ptr<vespalib::tensor::DenseTensorView> _query_tensor; uint32_t _target_num_hits; mutable NearestNeighborDistanceHeap _distance_heap; + std::vector<search::tensor::NearestNeighborIndex::Neighbor> _found_hits; + void perform_top_k(); public: NearestNeighborBlueprint(const queryeval::FieldSpec& field, const tensor::DenseTensorAttribute& attr_tensor, @@ -38,6 +41,7 @@ public: bool strict) const override; void visitMembers(vespalib::ObjectVisitor& visitor) const override; bool always_needs_unpack() const override; + void fetchPostings(const ExecuteInfo &execInfo) override; }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp new file mode 100644 index 00000000000..7ee985a0ba5 --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp @@ -0,0 +1,70 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "nns_index_iterator.h" +#include <vespa/searchlib/tensor/nearest_neighbor_index.h> +#include <cmath> + +using Hit = search::tensor::NearestNeighborIndex::Neighbor; + +namespace search::queryeval { + +/** + * Search iterator for K nearest neighbor matching, + * where the actual search is done up front and this class + * just iterates over a vector held by the blueprint. + **/ +class NeighborVectorIterator : public NnsIndexIterator +{ +private: + fef::TermFieldMatchData &_tfmd; + const std::vector<Hit> &_hits; + uint32_t _idx; + double _last_sq_dist; +public: + NeighborVectorIterator(fef::TermFieldMatchData &tfmd, + const std::vector<Hit> &hits) + : _tfmd(tfmd), + _hits(hits), + _idx(0), + _last_sq_dist(0.0) + {} + + void initRange(uint32_t begin_id, uint32_t end_id) override { + SearchIterator::initRange(begin_id, end_id); + _idx = 0; + } + + void doSeek(uint32_t docId) override { + while (_idx < _hits.size()) { + uint32_t hit_id = _hits[_idx].docid; + if (hit_id < docId) { + ++_idx; + } else if (hit_id < getEndId()) { + setDocId(hit_id); + _last_sq_dist = _hits[_idx].distance; + return; + } else { + _idx = _hits.size(); + } + } + setAtEnd(); + } + + void doUnpack(uint32_t docId) override { + _tfmd.setRawScore(docId, sqrt(_last_sq_dist)); + } + + Trinary is_strict() const override { return Trinary::True; } +}; + +std::unique_ptr<NnsIndexIterator> +NnsIndexIterator::create( + bool strict, + fef::TermFieldMatchData &tfmd, + const std::vector<Hit> &hits) +{ + assert(strict); + return std::make_unique<NeighborVectorIterator>(tfmd, hits); +} + +} // namespace diff --git a/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h new file mode 100644 index 00000000000..62fa49aac46 --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h @@ -0,0 +1,21 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "searchiterator.h" +#include <vespa/searchlib/fef/termfieldmatchdata.h> +#include <vespa/searchlib/tensor/nearest_neighbor_index.h> + +namespace search::queryeval { + +class NnsIndexIterator : public SearchIterator +{ +public: + using Hit = search::tensor::NearestNeighborIndex::Neighbor; + static std::unique_ptr<NnsIndexIterator> create( + bool strict, + fef::TermFieldMatchData &tfmd, + const std::vector<Hit> &hits); +}; + +} // namespace diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt index 9175168248c..0bdcd53af77 100644 --- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt @@ -1,16 +1,18 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchlib_tensor OBJECT SOURCES + default_nearest_neighbor_index_factory.cpp dense_tensor_attribute.cpp dense_tensor_attribute_saver.cpp dense_tensor_store.cpp generic_tensor_attribute.cpp + generic_tensor_attribute_saver.cpp generic_tensor_store.cpp hnsw_index.cpp imported_tensor_attribute_vector.cpp imported_tensor_attribute_vector_read_guard.cpp + nearest_neighbor_index.cpp tensor_attribute.cpp - generic_tensor_attribute_saver.cpp tensor_store.cpp DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp new file mode 100644 index 00000000000..68efe6417c0 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp @@ -0,0 +1,51 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "default_nearest_neighbor_index_factory.h" +#include "distance_functions.h" +#include "hnsw_index.h" +#include "random_level_generator.h" +#include <vespa/searchcommon/attribute/config.h> + +namespace search::tensor { + +using vespalib::eval::ValueType; + +namespace { + +class LevelZeroGenerator : public RandomLevelGenerator { + uint32_t max_level() override { return 0; } +}; + +DistanceFunction::UP +make_distance_function(ValueType::CellType cell_type) +{ + if (cell_type == ValueType::CellType::FLOAT) { + return std::make_unique<SquaredEuclideanDistance<float>>(); + } else { + return std::make_unique<SquaredEuclideanDistance<double>>(); + } +} + +RandomLevelGenerator::UP +make_random_level_generator() +{ + // TODO: Make generator that results in hierarchical graph. + return std::make_unique<LevelZeroGenerator>(); +} + +} + +std::unique_ptr<NearestNeighborIndex> +DefaultNearestNeighborIndexFactory::make(const DocVectorAccess& vectors, + vespalib::eval::ValueType::CellType cell_type, + const search::attribute::HnswIndexParams& params) const +{ + HnswIndex::Config cfg(params.max_links_per_node() * 2, + params.max_links_per_node(), + params.neighbors_to_explore_at_insert(), + true); + return std::make_unique<HnswIndex>(vectors, make_distance_function(cell_type), make_random_level_generator(), cfg); +} + +} + diff --git a/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h new file mode 100644 index 00000000000..ea784efdb51 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h @@ -0,0 +1,19 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "nearest_neighbor_index_factory.h" + +namespace search::tensor { + +/** + * Factory that instantiates the production hnsw index. + */ +class DefaultNearestNeighborIndexFactory : public NearestNeighborIndexFactory { +public: + std::unique_ptr<NearestNeighborIndex> make(const DocVectorAccess& vectors, + vespalib::eval::ValueType::CellType cell_type, + const search::attribute::HnswIndexParams& params) const override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index a2b9f136ed9..171340e07f1 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -2,6 +2,7 @@ #include "dense_tensor_attribute.h" #include "dense_tensor_attribute_saver.h" +#include "nearest_neighbor_index.h" #include "tensor_attribute.hpp" #include <vespa/eval/tensor/tensor.h> #include <vespa/eval/tensor/dense/mutable_dense_tensor_view.h> @@ -55,11 +56,23 @@ TensorReader::is_present() { } -DenseTensorAttribute::DenseTensorAttribute(vespalib::stringref baseFileName, - const Config &cfg) +void +DenseTensorAttribute::consider_remove_from_index(DocId docid) +{ + if (_index && _refVector[docid].valid()) { + _index->remove_document(docid); + } +} + +DenseTensorAttribute::DenseTensorAttribute(vespalib::stringref baseFileName, const Config& cfg, + const NearestNeighborIndexFactory& index_factory) : TensorAttribute(baseFileName, cfg, _denseTensorStore), - _denseTensorStore(cfg.tensorType()) + _denseTensorStore(cfg.tensorType()), + _index() { + if (cfg.hnsw_index_params().has_value()) { + _index = index_factory.make(*this, cfg.tensorType().cell_type(), cfg.hnsw_index_params().value()); + } } @@ -69,12 +82,23 @@ DenseTensorAttribute::~DenseTensorAttribute() _tensorStore.clearHoldLists(); } +uint32_t +DenseTensorAttribute::clearDoc(DocId docId) +{ + consider_remove_from_index(docId); + return TensorAttribute::clearDoc(docId); +} + void DenseTensorAttribute::setTensor(DocId docId, const Tensor &tensor) { checkTensorType(tensor); + consider_remove_from_index(docId); EntryRef ref = _denseTensorStore.setTensor(tensor); setTensorRef(docId, ref); + if (_index) { + _index->add_document(docId); + } } @@ -120,6 +144,11 @@ DenseTensorAttribute::onLoad() auto raw = _denseTensorStore.allocRawBuffer(); tensorReader.readTensor(raw.data, _denseTensorStore.getBufSize()); _refVector.push_back(raw.ref); + if (_index) { + // This ensures that get_vector() (via getTensor()) is able to find the newly added tensor. + setCommittedDocIdLimit(lid + 1); + _index->add_document(lid); + } } else { _refVector.push_back(EntryRef()); } @@ -154,4 +183,12 @@ DenseTensorAttribute::getVersion() const return DENSE_TENSOR_ATTRIBUTE_VERSION; } +vespalib::tensor::TypedCells +DenseTensorAttribute::get_vector(uint32_t docid) const +{ + MutableDenseTensorView tensor_view(_denseTensorStore.type()); + getTensor(docid, tensor_view); + return tensor_view.cellsRef(); +} + } diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h index 593741cef39..f9a8a81b56b 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h @@ -2,35 +2,47 @@ #pragma once -#include "tensor_attribute.h" +#include "default_nearest_neighbor_index_factory.h" #include "dense_tensor_store.h" +#include "doc_vector_access.h" +#include "tensor_attribute.h" +#include <memory> -namespace vespalib { namespace tensor { class MutableDenseTensorView; }} +namespace vespalib::tensor { class MutableDenseTensorView; } -namespace search { +namespace search::tensor { -namespace tensor { +class NearestNeighborIndex; /** * Attribute vector class used to store dense tensors for all * documents in memory. */ -class DenseTensorAttribute : public TensorAttribute -{ +class DenseTensorAttribute : public TensorAttribute, public DocVectorAccess { +private: DenseTensorStore _denseTensorStore; + std::unique_ptr<NearestNeighborIndex> _index; + + void consider_remove_from_index(DocId docid); + public: - DenseTensorAttribute(vespalib::stringref baseFileName, const Config &cfg); + DenseTensorAttribute(vespalib::stringref baseFileName, const Config& cfg, + const NearestNeighborIndexFactory& index_factory = DefaultNearestNeighborIndexFactory()); virtual ~DenseTensorAttribute(); - virtual void setTensor(DocId docId, const Tensor &tensor) override; - virtual std::unique_ptr<Tensor> getTensor(DocId docId) const override; - virtual void getTensor(DocId docId, vespalib::tensor::MutableDenseTensorView &tensor) const override; - virtual bool onLoad() override; - virtual std::unique_ptr<AttributeSaver> onInitSave(vespalib::stringref fileName) override; - virtual void compactWorst() override; - virtual uint32_t getVersion() const override; + // Implements TensorAttribute + uint32_t clearDoc(DocId docId) override; + void setTensor(DocId docId, const Tensor &tensor) override; + std::unique_ptr<Tensor> getTensor(DocId docId) const override; + void getTensor(DocId docId, vespalib::tensor::MutableDenseTensorView &tensor) const override; + bool onLoad() override; + std::unique_ptr<AttributeSaver> onInitSave(vespalib::stringref fileName) override; + void compactWorst() override; + uint32_t getVersion() const override; + + // Implements DocVectorAccess + vespalib::tensor::TypedCells get_vector(uint32_t docid) const override; + + const NearestNeighborIndex* nearest_neighbor_index() const { return _index.get(); } }; - -} // namespace search::tensor - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function.h b/searchlib/src/vespa/searchlib/tensor/distance_function.h index 8dfb77ddccb..b682824c805 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_function.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_function.h @@ -2,6 +2,8 @@ #pragma once +#include <memory> + namespace vespalib::tensor { struct TypedCells; } namespace search::tensor { @@ -14,6 +16,7 @@ namespace search::tensor { */ class DistanceFunction { public: + using UP = std::unique_ptr<DistanceFunction>; virtual ~DistanceFunction() {} virtual double calc(const vespalib::tensor::TypedCells& lhs, const vespalib::tensor::TypedCells& rhs) const = 0; }; diff --git a/searchlib/src/vespa/searchlib/tensor/distance_functions.h b/searchlib/src/vespa/searchlib/tensor/distance_functions.h index 1e8727e92aa..494d1a859b6 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_functions.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_functions.h @@ -3,6 +3,7 @@ #pragma once #include "distance_function.h" +#include <vespa/eval/tensor/dense/typed_cells.h> namespace search::tensor { diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index 9600f2fd9d4..0d308206761 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -44,7 +44,7 @@ HnswIndex::max_links_for_level(uint32_t level) const uint32_t HnswIndex::make_node_for_document(uint32_t docid) { - uint32_t max_level = _level_generator.max_level(); + uint32_t max_level = _level_generator->max_level(); // TODO: Add capping on num_levels uint32_t num_levels = max_level + 1; // Note: The level array instance lives as long as the document is present in the index. @@ -170,11 +170,11 @@ double HnswIndex::calc_distance(const TypedCells& lhs, uint32_t rhs_docid) const { auto rhs = get_vector(rhs_docid); - return _distance_func.calc(lhs, rhs); + return _distance_func->calc(lhs, rhs); } HnswCandidate -HnswIndex::find_nearest_in_layer(const TypedCells& input, const HnswCandidate& entry_point, uint32_t level) +HnswIndex::find_nearest_in_layer(const TypedCells& input, const HnswCandidate& entry_point, uint32_t level) const { HnswCandidate nearest = entry_point; bool keep_searching = true; @@ -192,7 +192,7 @@ HnswIndex::find_nearest_in_layer(const TypedCells& input, const HnswCandidate& e } void -HnswIndex::search_layer(const TypedCells& input, uint32_t neighbors_to_find, FurthestPriQ& best_neighbors, uint32_t level) +HnswIndex::search_layer(const TypedCells& input, uint32_t neighbors_to_find, FurthestPriQ& best_neighbors, uint32_t level) const { NearestPriQ candidates; // TODO: Add proper handling of visited set. @@ -227,11 +227,11 @@ HnswIndex::search_layer(const TypedCells& input, uint32_t neighbors_to_find, Fur } } -HnswIndex::HnswIndex(const DocVectorAccess& vectors, const DistanceFunction& distance_func, - RandomLevelGenerator& level_generator, const Config& cfg) +HnswIndex::HnswIndex(const DocVectorAccess& vectors, DistanceFunction::UP distance_func, + RandomLevelGenerator::UP level_generator, const Config& cfg) : _vectors(vectors), - _distance_func(distance_func), - _level_generator(level_generator), + _distance_func(std::move(distance_func)), + _level_generator(std::move(level_generator)), _cfg(cfg), _node_refs(), _nodes(make_default_node_store_config()), @@ -310,6 +310,49 @@ HnswIndex::remove_document(uint32_t docid) _node_refs[docid].store_release(invalid); } +struct NeighborsByDocId { + bool operator() (const NearestNeighborIndex::Neighbor &lhs, + const NearestNeighborIndex::Neighbor &rhs) + { + return (lhs.docid < rhs.docid); + } +}; + +std::vector<NearestNeighborIndex::Neighbor> +HnswIndex::find_top_k(uint32_t k, TypedCells vector, uint32_t explore_k) const +{ + std::vector<Neighbor> result; + FurthestPriQ candidates = top_k_candidates(vector, std::max(k, explore_k)); + while (candidates.size() > k) { + candidates.pop(); + } + result.reserve(candidates.size()); + for (const HnswCandidate & hit : candidates.peek()) { + result.emplace_back(hit.docid, hit.distance); + } + std::sort(result.begin(), result.end(), NeighborsByDocId()); + return result; +} + +FurthestPriQ +HnswIndex::top_k_candidates(const TypedCells &vector, uint32_t k) const +{ + FurthestPriQ best_neighbors; + if (_entry_level < 0) { + return best_neighbors; + } + double entry_dist = calc_distance(vector, _entry_docid); + HnswCandidate entry_point(_entry_docid, entry_dist); + int search_level = _entry_level; + while (search_level > 0) { + entry_point = find_nearest_in_layer(vector, entry_point, search_level); + --search_level; + } + best_neighbors.push(entry_point); + search_layer(vector, k, best_neighbors, 0); + return best_neighbors; +} + HnswNode HnswIndex::get_node(uint32_t docid) const { diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h index 15ad4ebf07b..800b88923b5 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -2,10 +2,12 @@ #pragma once +#include "distance_function.h" #include "doc_vector_access.h" #include "hnsw_index_utils.h" #include "hnsw_node.h" #include "nearest_neighbor_index.h" +#include "random_level_generator.h" #include <vespa/eval/tensor/dense/typed_cells.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/vespalib/datastore/array_store.h> @@ -15,9 +17,6 @@ namespace search::tensor { -class DistanceFunction; -class RandomLevelGenerator; - /** * Implementation of a hierarchical navigable small world graph (HNSW) * that is used for approximate K-nearest neighbor search. @@ -82,8 +81,8 @@ protected: using TypedCells = vespalib::tensor::TypedCells; const DocVectorAccess& _vectors; - const DistanceFunction& _distance_func; - RandomLevelGenerator& _level_generator; + DistanceFunction::UP _distance_func; + RandomLevelGenerator::UP _level_generator; Config _cfg; NodeRefVector _node_refs; NodeStore _nodes; @@ -124,16 +123,20 @@ protected: /** * Performs a greedy search in the given layer to find the candidate that is nearest the input vector. */ - HnswCandidate find_nearest_in_layer(const TypedCells& input, const HnswCandidate& entry_point, uint32_t level); - void search_layer(const TypedCells& input, uint32_t neighbors_to_find, FurthestPriQ& found_neighbors, uint32_t level); + HnswCandidate find_nearest_in_layer(const TypedCells& input, const HnswCandidate& entry_point, uint32_t level) const; + void search_layer(const TypedCells& input, uint32_t neighbors_to_find, FurthestPriQ& found_neighbors, uint32_t level) const; public: - HnswIndex(const DocVectorAccess& vectors, const DistanceFunction& distance_func, - RandomLevelGenerator& level_generator, const Config& cfg); + HnswIndex(const DocVectorAccess& vectors, DistanceFunction::UP distance_func, + RandomLevelGenerator::UP level_generator, const Config& cfg); ~HnswIndex() override; + const Config& config() const { return _cfg; } + void add_document(uint32_t docid) override; void remove_document(uint32_t docid) override; + std::vector<Neighbor> find_top_k(uint32_t k, TypedCells vector, uint32_t explore_k) const override; + FurthestPriQ top_k_candidates(const TypedCells &vector, uint32_t k) const; // TODO: Add support for generation handling and cleanup (transfer_hold_lists, trim_hold_lists) diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.cpp b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.cpp new file mode 100644 index 00000000000..f31230af381 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.cpp @@ -0,0 +1,3 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "nearest_neighbor_index.h" diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h index 2167157f6cb..f933af0147e 100644 --- a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h +++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h @@ -3,6 +3,8 @@ #pragma once #include <cstdint> +#include <vector> +#include <vespa/eval/tensor/dense/typed_cells.h> namespace search::tensor { @@ -11,9 +13,20 @@ namespace search::tensor { */ class NearestNeighborIndex { public: + struct Neighbor { + uint32_t docid; + double distance; + Neighbor(uint32_t id, double dist) + : docid(id), distance(dist) + {} + Neighbor() : docid(0), distance(0.0) {} + }; virtual ~NearestNeighborIndex() {} virtual void add_document(uint32_t docid) = 0; virtual void remove_document(uint32_t docid) = 0; + virtual std::vector<Neighbor> find_top_k(uint32_t k, + vespalib::tensor::TypedCells vector, + uint32_t explore_k) const = 0; }; } diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_factory.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_factory.h new file mode 100644 index 00000000000..c09403df5e0 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_factory.h @@ -0,0 +1,26 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/value_type.h> +#include <memory> + +namespace search::attribute { class HnswIndexParams; } + +namespace search::tensor { + +class DocVectorAccess; +class NearestNeighborIndex; + +/** + * Factory interface used to instantiate an index used for (approximate) nearest neighbor search. + */ +class NearestNeighborIndexFactory { +public: + virtual ~NearestNeighborIndexFactory() {} + virtual std::unique_ptr<NearestNeighborIndex> make(const DocVectorAccess& vectors, + vespalib::eval::ValueType::CellType cell_type, + const search::attribute::HnswIndexParams& params) const = 0; +}; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/random_level_generator.h b/searchlib/src/vespa/searchlib/tensor/random_level_generator.h index 0fcac977d9d..0f4c7c34445 100644 --- a/searchlib/src/vespa/searchlib/tensor/random_level_generator.h +++ b/searchlib/src/vespa/searchlib/tensor/random_level_generator.h @@ -2,6 +2,8 @@ #pragma once +#include <memory> + namespace search::tensor { /** @@ -9,6 +11,7 @@ namespace search::tensor { */ class RandomLevelGenerator { public: + using UP = std::unique_ptr<RandomLevelGenerator>; virtual ~RandomLevelGenerator() {} virtual uint32_t max_level() = 0; }; diff --git a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp index 9019a212f3f..dff3acc5b89 100644 --- a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp +++ b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp @@ -246,6 +246,7 @@ TEST_F(MatchedElementsFilterTest, filters_elements_in_array_field_value_when_inp expect_filtered("array_in_doc", {0, 1, 2}, "[{'name':'a','weight':3}," "{'name':'b','weight':5}," "{'name':'c','weight':7}]"); + expect_filtered("array_in_doc", {0, 1, 100}, "[]"); } TEST_F(MatchedElementsFilterTest, struct_field_mapper_is_setup_for_array_field_value) @@ -276,6 +277,7 @@ TEST_F(MatchedElementsFilterTest, filters_elements_in_map_field_value_when_input expect_filtered("map_in_doc", {0, 1, 2}, "[{'key':'a','value':{'name':'a','weight':3}}," "{'key':'b','value':{'name':'b','weight':5}}," "{'key':'c','value':{'name':'c','weight':7}}]"); + expect_filtered("map_in_doc", {0, 1, 100}, "[]"); } TEST_F(MatchedElementsFilterTest, struct_field_mapper_is_setup_for_map_field_value) diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp index 7368a199569..ada14bf17f5 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp @@ -389,9 +389,11 @@ private: MapFieldValueInserter map_inserter(_inserter, _tokenize); if (filter_matching_elements()) { assert(v.has_no_erased_keys()); - for (uint32_t id_to_keep : (*_matching_elems)) { - auto entry = v[id_to_keep]; - map_inserter.insert_entry(*entry.first, *entry.second); + if (!_matching_elems->empty() && _matching_elems->back() < v.size()) { + for (uint32_t id_to_keep : (*_matching_elems)) { + auto entry = v[id_to_keep]; + map_inserter.insert_entry(*entry.first, *entry.second); + } } } else { for (const auto &entry : v) { @@ -406,8 +408,10 @@ private: ArrayInserter ai(a); SlimeFiller conv(ai, _tokenize); if (filter_matching_elements()) { - for (uint32_t id_to_keep : (*_matching_elems)) { - value[id_to_keep].accept(conv); + if (!_matching_elems->empty() && _matching_elems->back() < value.size()) { + for (uint32_t id_to_keep : (*_matching_elems)) { + value[id_to_keep].accept(conv); + } } } else { for (const FieldValue &fv : value) { diff --git a/security-tools/pom.xml b/security-tools/pom.xml index 38b14ce957f..195e2d06311 100644 --- a/security-tools/pom.xml +++ b/security-tools/pom.xml @@ -57,6 +57,7 @@ <exclude>META-INF/*.SF</exclude> <exclude>META-INF/*.DSA</exclude> <exclude>META-INF/*.RSA</exclude> + <exclude>META-INF/versions/*/module-info.class</exclude> </excludes> </filter> </filters> diff --git a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java index 367d7b9dd83..c314d17e018 100644 --- a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java +++ b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java @@ -54,6 +54,9 @@ public class Main { MixedMode mixedMode = TransportSecurityUtils.getInsecureMixedMode(envVars); if (options.isPresent() && mixedMode != MixedMode.PLAINTEXT_CLIENT_MIXED_SERVER) { outputVariables.put(OutputVariable.TLS_ENABLED, "1"); + if (options.get().isHostnameValidationDisabled()) { + outputVariables.put(OutputVariable.DISABLE_HOSTNAME_VALIDATION, "1"); + } options.get().getCaCertificatesFile() .ifPresent(caCertFile -> outputVariables.put(OutputVariable.CA_CERTIFICATE, caCertFile.toString())); options.get().getCertificatesFile() diff --git a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/OutputVariable.java b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/OutputVariable.java index dd248d05aac..9a90a145f30 100644 --- a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/OutputVariable.java +++ b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/OutputVariable.java @@ -10,7 +10,8 @@ enum OutputVariable { TLS_ENABLED("VESPA_TLS_ENABLED", "Set to '1' if TLS is enabled in Vespa"), CA_CERTIFICATE("VESPA_TLS_CA_CERT", "Path to CA certificates file"), CERTIFICATE("VESPA_TLS_CERT", "Path to certificate file"), - PRIVATE_KEY("VESPA_TLS_PRIVATE_KEY", "Path to private key file"); + PRIVATE_KEY("VESPA_TLS_PRIVATE_KEY", "Path to private key file"), + DISABLE_HOSTNAME_VALIDATION("VESPA_TLS_HOSTNAME_VALIDATION_DISABLED", "Set to '1' if TLS hostname validation is disabled"); private final String variableName; private final String description; diff --git a/security-tools/src/main/sh/vespa-curl-wrapper b/security-tools/src/main/sh/vespa-curl-wrapper index e286e121f64..b4fd9224a8a 100755 --- a/security-tools/src/main/sh/vespa-curl-wrapper +++ b/security-tools/src/main/sh/vespa-curl-wrapper @@ -88,6 +88,11 @@ then CURL_PARAMETERS=("${CURL_PARAMETERS[@]/http:/https:}") fi +if [ -n "${VESPA_TLS_HOSTNAME_VALIDATION_DISABLED}" ] +then + CURL_PARAMETERS=("--insecure" "${CURL_PARAMETERS[@]}") +fi + if [ -n "${VESPA_TLS_CA_CERT}" ] then CURL_PARAMETERS=("--cacert" "${VESPA_TLS_CA_CERT}" "${CURL_PARAMETERS[@]}") diff --git a/security-tools/src/test/java/com/yahoo/vespa/security/tool/securityenv/MainTest.java b/security-tools/src/test/java/com/yahoo/vespa/security/tool/securityenv/MainTest.java index b563ebd14f4..45626820f4d 100644 --- a/security-tools/src/test/java/com/yahoo/vespa/security/tool/securityenv/MainTest.java +++ b/security-tools/src/test/java/com/yahoo/vespa/security/tool/securityenv/MainTest.java @@ -106,6 +106,7 @@ public class MainTest { TransportSecurityOptions options = new TransportSecurityOptions.Builder() .withCertificates(Paths.get("/path/to/certificate"), Paths.get("/path/to/key")) .withCaCertificates(Paths.get("/path/to/cacerts")) + .withHostnameValidationDisabled(true) .build(); Path configFile = tmpFolder.newFile().toPath(); options.toJsonFile(configFile); diff --git a/security-tools/src/test/resources/bash-output.txt b/security-tools/src/test/resources/bash-output.txt index c07c667af47..182dc177d42 100644 --- a/security-tools/src/test/resources/bash-output.txt +++ b/security-tools/src/test/resources/bash-output.txt @@ -2,3 +2,4 @@ VESPA_TLS_ENABLED="1"; export VESPA_TLS_ENABLED; VESPA_TLS_CA_CERT="/path/to/cacerts"; export VESPA_TLS_CA_CERT; VESPA_TLS_CERT="/path/to/certificate"; export VESPA_TLS_CERT; VESPA_TLS_PRIVATE_KEY="/path/to/key"; export VESPA_TLS_PRIVATE_KEY; +VESPA_TLS_HOSTNAME_VALIDATION_DISABLED="1"; export VESPA_TLS_HOSTNAME_VALIDATION_DISABLED; diff --git a/security-tools/src/test/resources/csh-output.txt b/security-tools/src/test/resources/csh-output.txt index 2b6716de92b..2e6cd886c26 100644 --- a/security-tools/src/test/resources/csh-output.txt +++ b/security-tools/src/test/resources/csh-output.txt @@ -2,3 +2,4 @@ setenv VESPA_TLS_ENABLED "1"; setenv VESPA_TLS_CA_CERT "/path/to/cacerts"; setenv VESPA_TLS_CERT "/path/to/certificate"; setenv VESPA_TLS_PRIVATE_KEY "/path/to/key"; +setenv VESPA_TLS_HOSTNAME_VALIDATION_DISABLED "1"; diff --git a/security-tools/src/test/resources/expected-help-output.txt b/security-tools/src/test/resources/expected-help-output.txt index 7d125fe15a2..33ad3b6d232 100644 --- a/security-tools/src/test/resources/expected-help-output.txt +++ b/security-tools/src/test/resources/expected-help-output.txt @@ -9,3 +9,5 @@ The output may include the following variables: - 'VESPA_TLS_CA_CERT': Path to CA certificates file - 'VESPA_TLS_CERT': Path to certificate file - 'VESPA_TLS_PRIVATE_KEY': Path to private key file + - 'VESPA_TLS_HOSTNAME_VALIDATION_DISABLED': Set to '1' if TLS hostname +validation is disabled diff --git a/security-tools/src/test/resources/no-security-output.txt b/security-tools/src/test/resources/no-security-output.txt index 3467f1316b5..257a2747ee2 100644 --- a/security-tools/src/test/resources/no-security-output.txt +++ b/security-tools/src/test/resources/no-security-output.txt @@ -2,3 +2,4 @@ unset VESPA_TLS_ENABLED; unset VESPA_TLS_CA_CERT; unset VESPA_TLS_CERT; unset VESPA_TLS_PRIVATE_KEY; +unset VESPA_TLS_HOSTNAME_VALIDATION_DISABLED; diff --git a/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java b/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java index d2b98fd20d9..f3932c84a17 100644 --- a/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java +++ b/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java @@ -35,6 +35,7 @@ public class SslContextBuilder { private TrustManagerFactory trustManagerFactory = TrustManagerUtils::createDefaultX509TrustManager; private KeyManagerFactory keyManagerFactory = KeyManagerUtils::createDefaultX509KeyManager; private X509ExtendedKeyManager keyManager; + private X509ExtendedTrustManager trustManager; public SslContextBuilder() {} @@ -121,15 +122,25 @@ public class SslContextBuilder { return this; } + /** + * Note: Callee is responsible for configuring the trust manager. + * Any truststore configured by {@link #withTrustStore(KeyStore)} or the other overloads will be ignored. + */ + public SslContextBuilder withTrustManager(X509ExtendedTrustManager trustManager) { + this.trustManager = trustManager; + return this; + } + public SSLContext build() { try { SSLContext sslContext = SSLContext.getInstance(TlsContext.SSL_CONTEXT_VERSION); - TrustManager[] trustManagers = new TrustManager[] { trustManagerFactory.createTrustManager(trustStoreSupplier.get()) }; + X509ExtendedTrustManager trustManager = this.trustManager != null + ? this.trustManager + : trustManagerFactory.createTrustManager(trustStoreSupplier.get()); X509ExtendedKeyManager keyManager = this.keyManager != null ? this.keyManager : keyManagerFactory.createKeyManager(keyStoreSupplier.get(), keyStorePassword); - KeyManager[] keyManagers = new KeyManager[] {keyManager}; - sslContext.init(keyManagers, trustManagers, null); + sslContext.init(new KeyManager[] {keyManager}, new TrustManager[] {trustManager}, null); return sslContext; } catch (GeneralSecurityException e) { throw new RuntimeException(e); diff --git a/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java index f746480b126..28854c59b2c 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java @@ -12,11 +12,9 @@ import com.yahoo.security.tls.policy.AuthorizedPeers; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLParameters; -import javax.net.ssl.X509ExtendedTrustManager; import java.io.IOException; import java.io.UncheckedIOException; import java.lang.ref.WeakReference; -import java.nio.file.Files; import java.nio.file.Path; import java.security.KeyStore; import java.time.Duration; @@ -110,12 +108,14 @@ public class ConfigFileBasedTlsContext implements TlsContext { MutableX509TrustManager mutableTrustManager, MutableX509KeyManager mutableKeyManager, PeerAuthentication peerAuthentication) { + + HostnameVerification hostnameVerification = options.isHostnameValidationDisabled() ? HostnameVerification.DISABLED : HostnameVerification.ENABLED; + PeerAuthorizerTrustManager authorizerTrustManager = options.getAuthorizedPeers() + .map(authorizedPeers -> new PeerAuthorizerTrustManager(authorizedPeers, mode, hostnameVerification, mutableTrustManager)) + .orElseGet(() -> new PeerAuthorizerTrustManager(new AuthorizedPeers(com.yahoo.vespa.jdk8compat.Set.of()), AuthorizationMode.DISABLE, hostnameVerification, mutableTrustManager)); SSLContext sslContext = new SslContextBuilder() .withKeyManager(mutableKeyManager) - .withTrustManagerFactory( - ignoredTruststore -> options.getAuthorizedPeers() - .map(authorizedPeers -> (X509ExtendedTrustManager) new PeerAuthorizerTrustManager(authorizedPeers, mode, mutableTrustManager)) - .orElseGet(() -> new PeerAuthorizerTrustManager(new AuthorizedPeers(com.yahoo.vespa.jdk8compat.Set.of()), AuthorizationMode.DISABLE, mutableTrustManager))) + .withTrustManager(authorizerTrustManager) .build(); List<String> acceptedCiphers = options.getAcceptedCiphers(); Set<String> ciphers = acceptedCiphers.isEmpty() ? TlsContext.ALLOWED_CIPHER_SUITES : new HashSet<>(acceptedCiphers); diff --git a/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java index c3f10a464a5..def3e49be4d 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java @@ -34,8 +34,9 @@ public class DefaultTlsContext implements TlsContext { List<X509Certificate> caCertificates, AuthorizedPeers authorizedPeers, AuthorizationMode mode, - PeerAuthentication peerAuthentication) { - this(createSslContext(certificates, privateKey, caCertificates, authorizedPeers, mode), peerAuthentication); + PeerAuthentication peerAuthentication, + HostnameVerification hostnameVerification) { + this(createSslContext(certificates, privateKey, caCertificates, authorizedPeers, mode, hostnameVerification), peerAuthentication); } public DefaultTlsContext(SSLContext sslContext, PeerAuthentication peerAuthentication) { @@ -120,7 +121,8 @@ public class DefaultTlsContext implements TlsContext { PrivateKey privateKey, List<X509Certificate> caCertificates, AuthorizedPeers authorizedPeers, - AuthorizationMode mode) { + AuthorizationMode mode, + HostnameVerification hostnameVerification) { SslContextBuilder builder = new SslContextBuilder(); if (!certificates.isEmpty()) { builder.withKeyStore(privateKey, certificates); @@ -129,12 +131,12 @@ public class DefaultTlsContext implements TlsContext { builder.withTrustStore(caCertificates); } if (authorizedPeers != null) { - builder.withTrustManagerFactory(truststore -> new PeerAuthorizerTrustManager(authorizedPeers, mode, truststore)); + builder.withTrustManagerFactory(truststore -> new PeerAuthorizerTrustManager(authorizedPeers, mode, hostnameVerification, truststore)); } else { - builder.withTrustManagerFactory(truststore -> new PeerAuthorizerTrustManager(new AuthorizedPeers(com.yahoo.vespa.jdk8compat.Set.of()), AuthorizationMode.DISABLE, truststore)); + builder.withTrustManagerFactory(truststore -> new PeerAuthorizerTrustManager( + new AuthorizedPeers(com.yahoo.vespa.jdk8compat.Set.of()), AuthorizationMode.DISABLE, hostnameVerification, truststore)); } return builder.build(); } - } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/HostnameVerification.java b/security-utils/src/main/java/com/yahoo/security/tls/HostnameVerification.java new file mode 100644 index 00000000000..a41edc6dc44 --- /dev/null +++ b/security-utils/src/main/java/com/yahoo/security/tls/HostnameVerification.java @@ -0,0 +1,7 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.security.tls; + +/** + * @author bjorncs + */ +public enum HostnameVerification { ENABLED, DISABLED } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManager.java b/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManager.java index 3ddd0861f39..03358190e8a 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManager.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManager.java @@ -3,6 +3,7 @@ package com.yahoo.security.tls.authz; import com.yahoo.security.X509CertificateUtils; import com.yahoo.security.tls.AuthorizationMode; +import com.yahoo.security.tls.HostnameVerification; import com.yahoo.security.tls.TrustManagerUtils; import com.yahoo.security.tls.policy.AuthorizedPeers; @@ -14,7 +15,6 @@ import java.net.Socket; import java.security.KeyStore; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; -import java.util.Objects; import java.util.Optional; import java.util.logging.Logger; @@ -33,15 +33,23 @@ public class PeerAuthorizerTrustManager extends X509ExtendedTrustManager { private final PeerAuthorizer authorizer; private final X509ExtendedTrustManager defaultTrustManager; private final AuthorizationMode mode; + private final HostnameVerification hostnameVerification; - public PeerAuthorizerTrustManager(AuthorizedPeers authorizedPeers, AuthorizationMode mode, X509ExtendedTrustManager defaultTrustManager) { + public PeerAuthorizerTrustManager(AuthorizedPeers authorizedPeers, + AuthorizationMode mode, + HostnameVerification hostnameVerification, + X509ExtendedTrustManager defaultTrustManager) { this.authorizer = new PeerAuthorizer(authorizedPeers); this.mode = mode; + this.hostnameVerification = hostnameVerification; this.defaultTrustManager = defaultTrustManager; } - public PeerAuthorizerTrustManager(AuthorizedPeers authorizedPeers, AuthorizationMode mode, KeyStore truststore) { - this(authorizedPeers, mode, TrustManagerUtils.createDefaultX509TrustManager(truststore)); + public PeerAuthorizerTrustManager(AuthorizedPeers authorizedPeers, + AuthorizationMode mode, + HostnameVerification hostnameVerification, + KeyStore truststore) { + this(authorizedPeers, mode, hostnameVerification, TrustManagerUtils.createDefaultX509TrustManager(truststore)); } @Override @@ -58,28 +66,26 @@ public class PeerAuthorizerTrustManager extends X509ExtendedTrustManager { @Override public void checkClientTrusted(X509Certificate[] chain, String authType, Socket socket) throws CertificateException { - overrideHostnameVerification(socket); defaultTrustManager.checkClientTrusted(chain, authType, socket); authorizePeer(chain[0], authType, true, null); } @Override public void checkServerTrusted(X509Certificate[] chain, String authType, Socket socket) throws CertificateException { - overrideHostnameVerification(socket); + overrideHostnameVerificationForClient(socket); defaultTrustManager.checkServerTrusted(chain, authType, socket); authorizePeer(chain[0], authType, false, null); } @Override public void checkClientTrusted(X509Certificate[] chain, String authType, SSLEngine sslEngine) throws CertificateException { - overrideHostnameVerification(sslEngine); defaultTrustManager.checkClientTrusted(chain, authType, sslEngine); authorizePeer(chain[0], authType, true, sslEngine); } @Override public void checkServerTrusted(X509Certificate[] chain, String authType, SSLEngine sslEngine) throws CertificateException { - overrideHostnameVerification(sslEngine); + overrideHostnameVerificationForClient(sslEngine); defaultTrustManager.checkServerTrusted(chain, authType, sslEngine); authorizePeer(chain[0], authType, false, sslEngine); } @@ -121,31 +127,44 @@ public class PeerAuthorizerTrustManager extends X509ExtendedTrustManager { certificate.getSubjectX500Principal(), X509CertificateUtils.getSubjectAlternativeNames(certificate), authType, isVerifyingClient); } - private static void overrideHostnameVerification(SSLEngine engine) { + private void overrideHostnameVerificationForClient(SSLEngine engine) { SSLParameters params = engine.getSSLParameters(); - if (overrideHostnameVerification(params)) { + if (overrideHostnameVerificationForClient(params)) { engine.setSSLParameters(params); } } - private static void overrideHostnameVerification(Socket socket) { + private void overrideHostnameVerificationForClient(Socket socket) { if (socket instanceof SSLSocket) { SSLSocket sslSocket = (SSLSocket) socket; SSLParameters params = sslSocket.getSSLParameters(); - if (overrideHostnameVerification(params)) { + if (overrideHostnameVerificationForClient(params)) { sslSocket.setSSLParameters(params); } } } - // Disable the default hostname verification that is performed by underlying trust manager when 'HTTPS' is used as endpoint identification algorithm. - // Some http clients, notably the new http client in Java 11, does not allow user configuration of the endpoint algorithm or custom HostnameVerifier. - private static boolean overrideHostnameVerification(SSLParameters params) { - if (Objects.equals("HTTPS", params.getEndpointIdentificationAlgorithm())) { - params.setEndpointIdentificationAlgorithm(""); - return true; + // Overrides the endpoint identification algorithm specified in the ssl parameters of the ssl engine/socket. + // The underlying trust manager will perform hostname verification if endpoint identification algorithm is set to 'HTTPS'. + // Returns true if the parameter instance was modified + private boolean overrideHostnameVerificationForClient(SSLParameters params) { + String configuredAlgorithm = params.getEndpointIdentificationAlgorithm(); + switch (hostnameVerification) { + case ENABLED: + if (!"HTTPS".equals(configuredAlgorithm)) { + params.setEndpointIdentificationAlgorithm("HTTPS"); + return true; + } + return false; + case DISABLED: + if (configuredAlgorithm != null && !configuredAlgorithm.isEmpty()) { + params.setEndpointIdentificationAlgorithm(""); // disable any configured endpoint identification algorithm + return true; + } + return false; + default: + throw new IllegalStateException("Unknown host verification type: " + hostnameVerification); } - return false; } } diff --git a/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java b/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java index 727a64ae934..00928187f55 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java @@ -46,7 +46,9 @@ public class DefaultTlsContextTest { singletonList(new RequiredPeerCredential(RequiredPeerCredential.Field.CN, new HostGlobPattern("dummy")))))); DefaultTlsContext tlsContext = - new DefaultTlsContext(singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers, AuthorizationMode.ENFORCE, PeerAuthentication.NEED); + new DefaultTlsContext( + singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers, + AuthorizationMode.ENFORCE, PeerAuthentication.NEED, HostnameVerification.ENABLED); SSLEngine sslEngine = tlsContext.createSslEngine(); assertThat(sslEngine).isNotNull(); diff --git a/standalone-container/vespa-standalone-container.spec b/standalone-container/vespa-standalone-container.spec index 98df4f71406..571b8b0ff4c 100644 --- a/standalone-container/vespa-standalone-container.spec +++ b/standalone-container/vespa-standalone-container.spec @@ -72,7 +72,6 @@ cp vespajlib/target/vespajlib.jar "$jars_dir" # Copy from submodules, so must be done separately cp zookeeper-server/zookeeper-server-common/target/zookeeper-server-common-jar-with-dependencies.jar "$jars_dir" -cp zookeeper-server/zookeeper-server-3.4/target/zookeeper-server-3.4-jar-with-dependencies.jar "$jars_dir" cp zookeeper-server/zookeeper-server-3.5/target/zookeeper-server-3.5-jar-with-dependencies.jar "$jars_dir" # Symlink to default version ln -s zookeeper-server-3.5-jar-with-dependencies.jar "$jars_dir"/zookeeper-server-jar-with-dependencies.jar diff --git a/tenant-base/pom.xml b/tenant-base/pom.xml index 22e1bb0e9ca..a3fac22a93d 100644 --- a/tenant-base/pom.xml +++ b/tenant-base/pom.xml @@ -253,6 +253,30 @@ </plugins> </build> </profile> + + <profile> <!-- Alias vespaversion with a more descriptive vespa.compile.version --> + <id>set-vespa-compile-version</id> + <activation> + <property> + <name>vespa.compile.version</name> + </property> + </activation> + <properties> + <vespaversion>${vespa.compile.version}</vespaversion> + </properties> + </profile> + + <profile> <!-- Alias vespaVersion with a more descriptive vespa.runtime.version --> + <id>set-vespa-runtime-version</id> + <activation> + <property> + <name>vespa.runtime.version</name> + </property> + </activation> + <properties> + <vespaVersion>${vespa.runtime.version}</vespaVersion> + </properties> + </profile> </profiles> <build> diff --git a/vbench/src/vbench/vbench/vbench.cpp b/vbench/src/vbench/vbench/vbench.cpp index 4f6efadfbdd..58854af705e 100644 --- a/vbench/src/vbench/vbench/vbench.cpp +++ b/vbench/src/vbench/vbench/vbench.cpp @@ -29,11 +29,13 @@ CryptoEngine::SP setup_crypto(const vespalib::slime::Inspector &tls) { if (!tls.valid()) { return std::make_shared<vespalib::NullCryptoEngine>(); } - vespalib::net::tls::TransportSecurityOptions - tls_opts(maybe_load(tls["ca-certificates"]), - maybe_load(tls["certificates"]), - maybe_load(tls["private-key"])); - return std::make_shared<vespalib::TlsCryptoEngine>(tls_opts); + auto ts_builder = vespalib::net::tls::TransportSecurityOptions::Params(). + ca_certs_pem(maybe_load(tls["ca-certificates"])). + cert_chain_pem(maybe_load(tls["certificates"])). + private_key_pem(maybe_load(tls["private-key"])). + authorized_peers(vespalib::net::tls::AuthorizedPeers::allow_all_authenticated()). + disable_hostname_validation(true); // TODO configurable or default false! + return std::make_shared<vespalib::TlsCryptoEngine>(vespalib::net::tls::TransportSecurityOptions(std::move(ts_builder))); } } // namespace vbench::<unnamed> diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java index d95eddac57f..95b6528bd77 100644 --- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java @@ -465,7 +465,8 @@ public class DocumentGenMojo extends AbstractMojo { exportStructTypeGetter(docType.getName()+".body", docType.allBody().getFields(), out, 1, "getBodyStructType", "com.yahoo.document.StructDataType"); Collection<Field> allUniqueFields = getAllUniqueFields(multiExtends, docType.getAllFields()); - exportExtendedStructTypeGetter(className, docType.getName(), allUniqueFields, docType.getFieldSets(),out, 1, "getDocumentType", "com.yahoo.document.DocumentType"); + exportExtendedStructTypeGetter(className, docType.getName(), allUniqueFields, docType.getFieldSets(), + docType.getImportedFieldNames(), out, 1, "getDocumentType", "com.yahoo.document.DocumentType"); exportCopyConstructor(className, out, 1, true); exportFieldsAndAccessors(className, "com.yahoo.document.Document".equals(superType) ? allUniqueFields : docType.getFields(), out, 1, true); @@ -627,10 +628,21 @@ public class DocumentGenMojo extends AbstractMojo { } out.write(ind(ind) + "ret.addFieldSets(fieldSets);\n"); } + private static void exportImportedFields(Set<String> importedFieldNames, Writer out, int ind) throws IOException { + out.write(ind(ind) + "java.util.Set<java.lang.String> importedFieldNames = new java.util.HashSet<>();\n"); + for (String importedField : importedFieldNames) { + out.write(ind(ind) + "importedFieldNames.add(\"" + importedField + "\");\n"); + } + } private static void exportExtendedStructTypeGetter(String className, String name, Collection<Field> fields, Set<FieldSet> fieldSets, - Writer out, int ind, String methodName, String retType) throws IOException { + Set<String> importedFieldNames, Writer out, int ind, String methodName, String retType) throws IOException { out.write(ind(ind)+"private static "+retType+" "+methodName+"() {\n"); - out.write(ind(ind+1)+retType+" ret = new "+retType+"(\""+name+"\");\n"); + if (importedFieldNames != null) { + exportImportedFields(importedFieldNames, out, ind + 1); + out.write(ind(ind+1)+retType+" ret = new "+retType+"(\"" + name + "\", importedFieldNames);\n"); + } else { + out.write(ind(ind+1)+retType+" ret = new "+retType+"(\""+name+"\");\n"); + } for (Field f : fields) { if (f.getDataType().equals(DataType.STRING)) { addExtendedStringField(className, f, out, ind + 1); @@ -783,7 +795,7 @@ public class DocumentGenMojo extends AbstractMojo { ind(ind+2)+"super("+structClassName+".type);\n" + ind(ind+1)+"}\n\n"); exportCopyConstructor(structClassName, out, ind+1, false); - exportExtendedStructTypeGetter(structClassName, structType.getName(), structType.getFields(), null, out, ind+1, "getStructType", "com.yahoo.document.StructDataType"); + exportExtendedStructTypeGetter(structClassName, structType.getName(), structType.getFields(), null, null, out, ind+1, "getStructType", "com.yahoo.document.StructDataType"); exportAssign(structType, structClassName, out, ind+1); exportFieldsAndAccessors(structClassName, structType.getFields(), out, ind+1, true); diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java index 0e202d1f348..59953fbe002 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java @@ -70,7 +70,6 @@ public class Runner { Optional.ofNullable(commandLineArgs.getFile()), commandLineArgs.getAddRootElementToXml()); - int intervalOfLogging = commandLineArgs.getVerbose() ? commandLineArgs.getWhenVerboseEnabledPrintMessageForEveryXDocuments() : Integer.MAX_VALUE; @@ -86,13 +85,15 @@ public class Runner { if (commandLineArgs.getVerbose()) { System.err.println(feedClient.getStatsAsJson()); - double fileSizeMb = ((double) new File(commandLineArgs.getFile()).length()) / 1024.0 / 1024.0; double transferTimeSec = ((double) sendTotalTimeMs) / 1000.0; - System.err.println("Sent " + fileSizeMb + " MB in " + transferTimeSec + " seconds."); - System.err.println("Speed: " + ((fileSizeMb / transferTimeSec) * 8.0) + " Mbits/sec, + HTTP overhead " + - "(not taking compression into account)"); if (transferTimeSec > 0) { - System.err.printf("Docs/sec %.3f%n\n", numSent.get() / transferTimeSec); + System.err.printf("Docs/sec %.3f%n", numSent.get() / transferTimeSec); + } + if (commandLineArgs.getFile() != null) { + double fileSizeMb = ((double) new File(commandLineArgs.getFile()).length()) / 1024.0 / 1024.0; + System.err.println("Sent " + fileSizeMb + " MB in " + transferTimeSec + " seconds."); + System.err.println("Speed: " + ((fileSizeMb / transferTimeSec) * 8.0) + " Mbits/sec, + HTTP overhead " + + "(not taking compression into account)"); } } callback.printProgress(); diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java index ed079442440..90f7a76b356 100644 --- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java @@ -61,8 +61,7 @@ public class TestRunnerHandler extends LoggingRequestHandler { private HttpResponse handleGET(HttpRequest request) { String path = request.getUri().getPath(); - // TODO: Migrate to /tester/v1/log when /tester/v1/log2 is not in use anymore (and remove /tester/v1/log2) - if (path.equals("/tester/v1/log") || path.equals("/tester/v1/log2")) { + if (path.equals("/tester/v1/log")) { return new SlimeJsonResponse(logToSlime(testRunner.getLog(request.hasProperty("after") ? Long.parseLong(request.getProperty("after")) : -1))); @@ -91,7 +90,7 @@ public class TestRunnerHandler extends LoggingRequestHandler { path = path.substring(0, path.length() - 1); int lastSlash = path.lastIndexOf("/"); if (lastSlash < 0) return path; - return path.substring(lastSlash + 1, path.length()); + return path.substring(lastSlash + 1); } static Slime logToSlime(Collection<LogRecord> log) { diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/Http.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/Http.pm index 2dbf475f2a7..d907e89fa54 100644 --- a/vespaclient/src/perl/lib/Yahoo/Vespa/Http.pm +++ b/vespaclient/src/perl/lib/Yahoo/Vespa/Http.pm @@ -100,7 +100,10 @@ sub initialize { # () my $tls_enabled = $ENV{'VESPA_TLS_ENABLED'}; if (defined $tls_enabled and $tls_enabled eq '1') { $BROWSER->ssl_opts( SSL_version => 'TLSv12'); - $BROWSER->ssl_opts( verify_hostname => 0); + my $hostname_verification_disabled = $ENV{'VESPA_TLS_HOSTNAME_VALIDATION_DISABLED'}; + if (defined $hostname_verification_disabled and $hostname_verification_disabled eq '1') { + $BROWSER->ssl_opts( verify_hostname => 0); + } $BROWSER->ssl_opts( SSL_cipher_list => 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-CHACHA20-POLY1305:TLS13-AES-128-GCM-SHA256:TLS13-AES-256-GCM-SHA384:TLS13-CHACHA20-POLY1305-SHA256' ); } if (defined $ENV{'VESPA_TLS_CA_CERT'}) { diff --git a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp index 78838ce2cd2..54c8c19fc64 100644 --- a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp +++ b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp @@ -57,11 +57,23 @@ void print_decode_result(const char* mode, const DecodeResult& res) { decode_state_to_str(res.state)); } +TransportSecurityOptions ts_from_pems(vespalib::stringref ca_certs_pem, + vespalib::stringref cert_chain_pem, + vespalib::stringref private_key_pem) +{ + auto ts_builder = TransportSecurityOptions::Params(). + ca_certs_pem(ca_certs_pem). + cert_chain_pem(cert_chain_pem). + private_key_pem(private_key_pem). + authorized_peers(AuthorizedPeers::allow_all_authenticated()); + return TransportSecurityOptions(std::move(ts_builder)); +} + struct Fixture { TransportSecurityOptions tls_opts; std::shared_ptr<TlsContext> tls_ctx; - std::unique_ptr<CryptoCodec> client; - std::unique_ptr<CryptoCodec> server; + std::unique_ptr<OpenSslCryptoCodecImpl> client; + std::unique_ptr<OpenSslCryptoCodecImpl> server; SmartBuffer client_to_server; SmartBuffer server_to_client; @@ -77,16 +89,21 @@ struct Fixture { static TransportSecurityOptions create_options_without_own_peer_cert() { auto source_opts = vespalib::test::make_tls_options_for_testing(); - return TransportSecurityOptions(source_opts.ca_certs_pem(), "", ""); + return ts_from_pems(source_opts.ca_certs_pem(), "", ""); } - static std::unique_ptr<CryptoCodec> create_openssl_codec( - const TransportSecurityOptions& opts, CryptoCodec::Mode mode) { + static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec( + const TransportSecurityOptions& opts, CryptoCodec::Mode mode, const SocketSpec& peer_spec) { auto ctx = TlsContext::create_default_context(opts, AuthorizationMode::Enforce); - return create_openssl_codec(ctx, mode); + return create_openssl_codec(ctx, mode, peer_spec); + } + + static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec( + const TransportSecurityOptions& opts, CryptoCodec::Mode mode) { + return create_openssl_codec(opts, mode, SocketSpec::invalid); } - static std::unique_ptr<CryptoCodec> create_openssl_codec( + static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec( const TransportSecurityOptions& opts, std::shared_ptr<CertificateVerificationCallback> cert_verify_callback, CryptoCodec::Mode mode) { @@ -94,21 +111,30 @@ struct Fixture { return create_openssl_codec(ctx, mode); } - static std::unique_ptr<CryptoCodec> create_openssl_codec( - const std::shared_ptr<TlsContext>& ctx, CryptoCodec::Mode mode) { + static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec( + const std::shared_ptr<TlsContext>& ctx, CryptoCodec::Mode mode, const SocketSpec& peer_spec) { auto ctx_impl = std::dynamic_pointer_cast<impl::OpenSslTlsContextImpl>(ctx); - return std::make_unique<impl::OpenSslCryptoCodecImpl>(std::move(ctx_impl), SocketAddress(), mode); + if (mode == CryptoCodec::Mode::Client) { + return OpenSslCryptoCodecImpl::make_client_codec(std::move(ctx_impl), peer_spec, SocketAddress()); + } else { + return OpenSslCryptoCodecImpl::make_server_codec(std::move(ctx_impl), SocketAddress()); + } } - EncodeResult do_encode(CryptoCodec& codec, Output& buffer, vespalib::stringref plaintext) { + static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec( + const std::shared_ptr<TlsContext>& ctx, CryptoCodec::Mode mode) { + return create_openssl_codec(ctx, mode, SocketSpec::invalid); + } + + static EncodeResult do_encode(CryptoCodec& codec, Output& buffer, vespalib::stringref plaintext) { auto out = buffer.reserve(codec.min_encode_buffer_size()); auto enc_res = codec.encode(plaintext.data(), plaintext.size(), out.data, out.size); buffer.commit(enc_res.bytes_produced); return enc_res; } - DecodeResult do_decode(CryptoCodec& codec, Input& buffer, vespalib::string& out, - size_t max_bytes_produced, size_t max_bytes_consumed) { + static DecodeResult do_decode(CryptoCodec& codec, Input& buffer, vespalib::string& out, + size_t max_bytes_produced, size_t max_bytes_consumed) { auto in = buffer.obtain(); out.resize(max_bytes_produced); auto to_consume = std::min(in.size, max_bytes_consumed); @@ -382,13 +408,13 @@ l9pLv1vrujrPEC78cyIQe2x55wf3pRoaDg== -----END EC PRIVATE KEY-----)"; TEST_F("client with certificate signed by untrusted CA is rejected by server", Fixture) { - TransportSecurityOptions client_opts(unknown_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem); + auto client_opts = ts_from_pems(unknown_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem); f.client = f.create_openssl_codec(client_opts, CryptoCodec::Mode::Client); EXPECT_FALSE(f.handshake()); } TEST_F("server with certificate signed by untrusted CA is rejected by client", Fixture) { - TransportSecurityOptions server_opts(unknown_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem); + auto server_opts = ts_from_pems(unknown_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem); f.server = f.create_openssl_codec(server_opts, CryptoCodec::Mode::Server); EXPECT_FALSE(f.handshake()); } @@ -396,8 +422,8 @@ TEST_F("server with certificate signed by untrusted CA is rejected by client", F TEST_F("Can specify multiple trusted CA certs in transport options", Fixture) { auto& base_opts = f.tls_opts; auto multi_ca_pem = base_opts.ca_certs_pem() + "\n" + unknown_ca_pem; - TransportSecurityOptions multi_ca_using_ca_1(multi_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem); - TransportSecurityOptions multi_ca_using_ca_2(multi_ca_pem, base_opts.cert_chain_pem(), base_opts.private_key_pem()); + auto multi_ca_using_ca_1 = ts_from_pems(multi_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem); + auto multi_ca_using_ca_2 = ts_from_pems(multi_ca_pem, base_opts.cert_chain_pem(), base_opts.private_key_pem()); // Let client be signed by CA 1, server by CA 2. Both have the two CAs in their trust store // so this should allow for a successful handshake. f.client = f.create_openssl_codec(multi_ca_using_ca_1, CryptoCodec::Mode::Client); @@ -446,7 +472,7 @@ struct CertFixture : Fixture { return {std::move(cert), std::move(key)}; } - static std::unique_ptr<CryptoCodec> create_openssl_codec_with_authz_mode( + static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec_with_authz_mode( const TransportSecurityOptions& opts, std::shared_ptr<CertificateVerificationCallback> cert_verify_callback, CryptoCodec::Mode codec_mode, @@ -455,33 +481,52 @@ struct CertFixture : Fixture { return create_openssl_codec(ctx, codec_mode); } + TransportSecurityOptions::Params ts_builder_from(const CertKeyWrapper& ck) const { + return TransportSecurityOptions::Params(). + ca_certs_pem(root_ca.cert->to_pem()). + cert_chain_pem(ck.cert->to_pem()). + private_key_pem(ck.key->private_to_pem()); + } + void reset_client_with_cert_opts(const CertKeyWrapper& ck, AuthorizedPeers authorized) { - TransportSecurityOptions client_opts(root_ca.cert->to_pem(), ck.cert->to_pem(), - ck.key->private_to_pem(), std::move(authorized)); - client = create_openssl_codec(client_opts, CryptoCodec::Mode::Client); + auto ts_params = ts_builder_from(ck).authorized_peers(std::move(authorized)); + client = create_openssl_codec(TransportSecurityOptions(std::move(ts_params)), CryptoCodec::Mode::Client); } void reset_client_with_cert_opts(const CertKeyWrapper& ck, std::shared_ptr<CertificateVerificationCallback> cert_cb) { - TransportSecurityOptions client_opts(root_ca.cert->to_pem(), ck.cert->to_pem(), ck.key->private_to_pem()); - client = create_openssl_codec(client_opts, std::move(cert_cb), CryptoCodec::Mode::Client); + auto ts_params = ts_builder_from(ck).authorized_peers(AuthorizedPeers::allow_all_authenticated()); + client = create_openssl_codec(TransportSecurityOptions(std::move(ts_params)), + std::move(cert_cb), CryptoCodec::Mode::Client); } void reset_server_with_cert_opts(const CertKeyWrapper& ck, AuthorizedPeers authorized) { - TransportSecurityOptions server_opts(root_ca.cert->to_pem(), ck.cert->to_pem(), - ck.key->private_to_pem(), std::move(authorized)); - server = create_openssl_codec(server_opts, CryptoCodec::Mode::Server); + auto ts_params = ts_builder_from(ck).authorized_peers(std::move(authorized)); + server = create_openssl_codec(TransportSecurityOptions(std::move(ts_params)), CryptoCodec::Mode::Server); } void reset_server_with_cert_opts(const CertKeyWrapper& ck, std::shared_ptr<CertificateVerificationCallback> cert_cb) { - TransportSecurityOptions server_opts(root_ca.cert->to_pem(), ck.cert->to_pem(), ck.key->private_to_pem()); - server = create_openssl_codec(server_opts, std::move(cert_cb), CryptoCodec::Mode::Server); + auto ts_params = ts_builder_from(ck).authorized_peers(AuthorizedPeers::allow_all_authenticated()); + server = create_openssl_codec(TransportSecurityOptions(std::move(ts_params)), + std::move(cert_cb), CryptoCodec::Mode::Server); } void reset_server_with_cert_opts(const CertKeyWrapper& ck, std::shared_ptr<CertificateVerificationCallback> cert_cb, AuthorizationMode authz_mode) { - TransportSecurityOptions server_opts(root_ca.cert->to_pem(), ck.cert->to_pem(), ck.key->private_to_pem()); - server = create_openssl_codec_with_authz_mode(server_opts, std::move(cert_cb), CryptoCodec::Mode::Server, authz_mode); + auto ts_params = ts_builder_from(ck).authorized_peers(AuthorizedPeers::allow_all_authenticated()); + server = create_openssl_codec_with_authz_mode(TransportSecurityOptions(std::move(ts_params)), + std::move(cert_cb), CryptoCodec::Mode::Server, authz_mode); + } + + void reset_client_with_peer_spec(const CertKeyWrapper& ck, + const SocketSpec& peer_spec, + bool disable_hostname_validation = false) + { + auto ts_params = ts_builder_from(ck). + authorized_peers(AuthorizedPeers::allow_all_authenticated()). + disable_hostname_validation(disable_hostname_validation); + client = create_openssl_codec(TransportSecurityOptions(std::move(ts_params)), + CryptoCodec::Mode::Client, peer_spec); } }; @@ -537,7 +582,7 @@ TEST_F("Exception during verification callback processing breaks handshake", Cer EXPECT_FALSE(f.handshake()); } -TEST_F("certificate verification callback observes CN and DNS SANs", CertFixture) { +TEST_F("Certificate verification callback observes CN and DNS SANs", CertFixture) { auto ck = f.create_ca_issued_peer_cert( {{"rockets.wile.example.com"}}, {{"DNS:crash.wile.example.com"}, {"DNS:burn.wile.example.com"}}); @@ -556,7 +601,7 @@ TEST_F("certificate verification callback observes CN and DNS SANs", CertFixture EXPECT_EQUAL("burn.wile.example.com", creds.dns_sans[1]); } -TEST_F("last occurring CN is given to verification callback if multiple CNs are present", CertFixture) { +TEST_F("Last occurring CN is given to verification callback if multiple CNs are present", CertFixture) { auto ck = f.create_ca_issued_peer_cert( {{"foo.wile.example.com"}, {"bar.wile.example.com"}, {"baz.wile.example.com"}}, {}); @@ -646,6 +691,51 @@ TEST_F("Disabled insecure authorization mode ignores verification result", CertF EXPECT_TRUE(f.handshake()); } +void reset_peers_with_client_peer_spec(CertFixture& f, + const SocketSpec& peer_spec, + bool disable_hostname_validation = false) +{ + auto client_ck = f.create_ca_issued_peer_cert({"hello.world.example.com"}, {}); + f.reset_client_with_peer_spec(client_ck, peer_spec, disable_hostname_validation); + // Since hostname validation is enabled by default, providing a peer spec also + // means that we must have a valid server name to present back (or the handshake fails). + auto server_ck = f.create_ca_issued_peer_cert({}, {{"DNS:*.example.com"}}); + f.reset_server_with_cert_opts(server_ck, AuthorizedPeers::allow_all_authenticated()); +} + +TEST_F("Client does not send SNI extension if hostname not provided in spec", CertFixture) { + reset_peers_with_client_peer_spec(f, SocketSpec::invalid); + + ASSERT_TRUE(f.handshake()); + auto maybe_sni = f.server->client_provided_sni_extension(); + EXPECT_FALSE(maybe_sni.has_value()); +} + +TEST_F("Client sends SNI extension with hostname provided in spec", CertFixture) { + reset_peers_with_client_peer_spec(f, SocketSpec::from_host_port("sni-test.example.com", 12345)); + + ASSERT_TRUE(f.handshake()); + auto maybe_sni = f.server->client_provided_sni_extension(); + ASSERT_TRUE(maybe_sni.has_value()); + EXPECT_EQUAL("sni-test.example.com", *maybe_sni); +} + +TEST_F("Client hostname validation passes handshake if server hostname matches certificate", CertFixture) { + reset_peers_with_client_peer_spec(f, SocketSpec::from_host_port("server-must-be-under.example.com", 12345), false); + EXPECT_TRUE(f.handshake()); +} + +TEST_F("Client hostname validation fails handshake if server hostname mismatches certificate", CertFixture) { + // Wildcards only apply to a single level, so this should fail as the server only has a cert for *.example.com + reset_peers_with_client_peer_spec(f, SocketSpec::from_host_port("nested.name.example.com", 12345), false); + EXPECT_FALSE(f.handshake()); +} + +TEST_F("Mismatching server cert vs hostname does not fail if hostname validation is disabled", CertFixture) { + reset_peers_with_client_peer_spec(f, SocketSpec::from_host_port("a.very.nested.name.example.com", 12345), true); + EXPECT_TRUE(f.handshake()); +} + TEST_F("Failure statistics are incremented on authorization failures", CertFixture) { reset_peers_with_server_authz_mode(f, AuthorizationMode::Enforce); auto server_before = ConnectionStatistics::get(true).snapshot(); diff --git a/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp b/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp index a54e2f29aa1..00459a4e69c 100644 --- a/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp +++ b/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp @@ -155,6 +155,47 @@ TEST("accepted cipher list is populated if specified") { EXPECT_EQUAL("bar", ciphers[1]); } +// FIXME this is temporary until we know enabling it by default won't break the world! +TEST("hostname validation is DISABLED by default when creating options from config file") { + const char* json = R"({"files":{"private-key":"dummy_privkey.txt", + "certificates":"dummy_certs.txt", + "ca-certificates":"dummy_ca_certs.txt"}})"; + EXPECT_TRUE(read_options_from_json_string(json)->disable_hostname_validation()); +} + +TEST("TransportSecurityOptions builder does not disable hostname validation by default") { + auto ts_builder = vespalib::net::tls::TransportSecurityOptions::Params(). + ca_certs_pem("foo"). + cert_chain_pem("bar"). + private_key_pem("fantonald"); + TransportSecurityOptions ts_opts(std::move(ts_builder)); + EXPECT_FALSE(ts_opts.disable_hostname_validation()); +} + +TEST("hostname validation can be explicitly disabled") { + const char* json = R"({"files":{"private-key":"dummy_privkey.txt", + "certificates":"dummy_certs.txt", + "ca-certificates":"dummy_ca_certs.txt"}, + "disable-hostname-validation": true})"; + EXPECT_TRUE(read_options_from_json_string(json)->disable_hostname_validation()); +} + +TEST("hostname validation can be explicitly enabled") { + const char* json = R"({"files":{"private-key":"dummy_privkey.txt", + "certificates":"dummy_certs.txt", + "ca-certificates":"dummy_ca_certs.txt"}, + "disable-hostname-validation": false})"; + EXPECT_FALSE(read_options_from_json_string(json)->disable_hostname_validation()); +} + +TEST("unknown fields are ignored at parse-time") { + const char* json = R"({"files":{"private-key":"dummy_privkey.txt", + "certificates":"dummy_certs.txt", + "ca-certificates":"dummy_ca_certs.txt"}, + "flipper-the-dolphin": "*weird dolphin noises*"})"; + EXPECT_TRUE(read_options_from_json_string(json).get() != nullptr); // And no exception thrown. +} + // TODO test parsing of multiple policies TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/vespa/vespalib/net/socket_spec.cpp b/vespalib/src/vespa/vespalib/net/socket_spec.cpp index 06682086670..d1dd81a454a 100644 --- a/vespalib/src/vespa/vespalib/net/socket_spec.cpp +++ b/vespalib/src/vespa/vespalib/net/socket_spec.cpp @@ -41,7 +41,7 @@ SocketSpec::address(bool server) const return SocketAddress(); } -SocketSpec SocketSpec::invalid; +const SocketSpec SocketSpec::invalid; SocketSpec::SocketSpec(const vespalib::string &spec) : SocketSpec() diff --git a/vespalib/src/vespa/vespalib/net/socket_spec.h b/vespalib/src/vespa/vespalib/net/socket_spec.h index 01af382d638..4e3dddf6814 100644 --- a/vespalib/src/vespa/vespalib/net/socket_spec.h +++ b/vespalib/src/vespa/vespalib/net/socket_spec.h @@ -24,7 +24,7 @@ private: : _type(type), _node(node), _port(port) {} SocketAddress address(bool server) const; public: - static SocketSpec invalid; + static const SocketSpec invalid; explicit SocketSpec(const vespalib::string &spec); vespalib::string spec() const; SocketSpec replace_host(const vespalib::string &new_host) const; diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.cpp b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.cpp index c54990b3782..d3ac975d90a 100644 --- a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.cpp @@ -6,12 +6,23 @@ namespace vespalib::net::tls { -std::unique_ptr<CryptoCodec> CryptoCodec::create_default_codec( - std::shared_ptr<TlsContext> ctx, const SocketAddress& peer_address, Mode mode) +std::unique_ptr<CryptoCodec> +CryptoCodec::create_default_client_codec(std::shared_ptr<TlsContext> ctx, + const SocketSpec& peer_spec, + const SocketAddress& peer_address) { auto ctx_impl = std::dynamic_pointer_cast<impl::OpenSslTlsContextImpl>(ctx); // only takes by const ref assert(ctx_impl); - return std::make_unique<impl::OpenSslCryptoCodecImpl>(std::move(ctx_impl), peer_address, mode); + return impl::OpenSslCryptoCodecImpl::make_client_codec(std::move(ctx_impl), peer_spec, peer_address); +} + +std::unique_ptr<CryptoCodec> +CryptoCodec::create_default_server_codec(std::shared_ptr<TlsContext> ctx, + const SocketAddress& peer_address) +{ + auto ctx_impl = std::dynamic_pointer_cast<impl::OpenSslTlsContextImpl>(ctx); // only takes by const ref + assert(ctx_impl); + return impl::OpenSslCryptoCodecImpl::make_server_codec(std::move(ctx_impl), peer_address); } } diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h index 5d9684461d7..787485b47be 100644 --- a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h +++ b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h @@ -4,6 +4,8 @@ #include <vespa/vespalib/net/socket_address.h> #include <memory> +namespace vespalib { class SocketSpec; } + namespace vespalib::net::tls { struct HandshakeResult { @@ -179,9 +181,13 @@ public: * * Throws CryptoException if resources cannot be allocated for the codec. */ - static std::unique_ptr<CryptoCodec> create_default_codec(std::shared_ptr<TlsContext> ctx, - const SocketAddress& peer_address, - Mode mode); + static std::unique_ptr<CryptoCodec> + create_default_client_codec(std::shared_ptr<TlsContext> ctx, + const SocketSpec& peer_spec, + const SocketAddress& peer_address); + static std::unique_ptr<CryptoCodec> + create_default_server_codec(std::shared_ptr<TlsContext> ctx, + const SocketAddress& peer_address); }; } diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp index 5315754d53a..6a79caa8264 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp @@ -172,9 +172,11 @@ void log_ssl_error(const char* source, const SocketAddress& peer_address, int ss } // anon ns OpenSslCryptoCodecImpl::OpenSslCryptoCodecImpl(std::shared_ptr<OpenSslTlsContextImpl> ctx, + const SocketSpec& peer_spec, const SocketAddress& peer_address, Mode mode) : _ctx(std::move(ctx)), + _peer_spec(peer_spec), _peer_address(peer_address), _ssl(::SSL_new(_ctx->native_context())), _mode(mode), @@ -219,6 +221,8 @@ OpenSslCryptoCodecImpl::OpenSslCryptoCodecImpl(std::shared_ptr<OpenSslTlsContext _output_bio = tmp_output_bio.release(); if (_mode == Mode::Client) { ::SSL_set_connect_state(_ssl.get()); + enable_hostname_validation_if_requested(); + set_server_name_indication_extension(); } else { ::SSL_set_accept_state(_ssl.get()); } @@ -230,6 +234,59 @@ OpenSslCryptoCodecImpl::OpenSslCryptoCodecImpl(std::shared_ptr<OpenSslTlsContext OpenSslCryptoCodecImpl::~OpenSslCryptoCodecImpl() = default; +std::unique_ptr<OpenSslCryptoCodecImpl> +OpenSslCryptoCodecImpl::make_client_codec(std::shared_ptr<OpenSslTlsContextImpl> ctx, + const SocketSpec& peer_spec, + const SocketAddress& peer_address) +{ + // Naked new due to private ctor + return std::unique_ptr<OpenSslCryptoCodecImpl>( + new OpenSslCryptoCodecImpl(std::move(ctx), peer_spec, peer_address, Mode::Client)); +} +std::unique_ptr<OpenSslCryptoCodecImpl> +OpenSslCryptoCodecImpl::make_server_codec(std::shared_ptr<OpenSslTlsContextImpl> ctx, + const SocketAddress& peer_address) +{ + // Naked new due to private ctor + return std::unique_ptr<OpenSslCryptoCodecImpl>( + new OpenSslCryptoCodecImpl(std::move(ctx), SocketSpec::invalid, peer_address, Mode::Server)); +} + +void OpenSslCryptoCodecImpl::enable_hostname_validation_if_requested() { + if (_peer_spec.valid() && !_ctx->transport_security_options().disable_hostname_validation()) { + auto* verify_param = SSL_get0_param(_ssl.get()); // Internal ptr, no refcount bump or alloc. We must not free. + LOG_ASSERT(verify_param != nullptr); + vespalib::string host = _peer_spec.host(); + if (X509_VERIFY_PARAM_set1_host(verify_param, host.c_str(), host.size()) != 1) { + throw CryptoException("X509_VERIFY_PARAM_set1_host() failed"); + } + // TODO should we set expected IP based on peer address as well? + } +} + +void OpenSslCryptoCodecImpl::set_server_name_indication_extension() { + if (_peer_spec.valid()) { + vespalib::string host = _peer_spec.host(); + // OpenSSL tries to cast const char* to void* in a macro, even on 1.1.1. GCC is not overly impressed, + // so to satiate OpenSSL's quirks we pre-cast away the constness. + auto* host_cstr_that_trusts_openssl_not_to_mess_up = const_cast<char*>(host.c_str()); + if (SSL_set_tlsext_host_name(_ssl.get(), host_cstr_that_trusts_openssl_not_to_mess_up) != 1) { + throw CryptoException("SSL_set_tlsext_host_name() failed"); + } + } +} + +std::optional<vespalib::string> OpenSslCryptoCodecImpl::client_provided_sni_extension() const { + if ((_mode != Mode::Server) || (SSL_get_servername_type(_ssl.get()) != TLSEXT_NAMETYPE_host_name)) { + return {}; + } + const char* sni_host_raw = SSL_get_servername(_ssl.get(), TLSEXT_NAMETYPE_host_name); + if (sni_host_raw == nullptr) { + return {}; + } + return vespalib::string(sni_host_raw); +} + HandshakeResult OpenSslCryptoCodecImpl::handshake(const char* from_peer, size_t from_peer_buf_size, char* to_peer, size_t to_peer_buf_size) noexcept { @@ -428,3 +485,5 @@ EncodeResult OpenSslCryptoCodecImpl::half_close(char* ciphertext, size_t ciphert // External references: // [0] http://openssl.6102.n7.nabble.com/nonblocking-implementation-question-tp1728p1732.html // [1] https://github.com/grpc/grpc/blob/master/src/core/tsi/ssl_transport_security.cc +// [2] https://wiki.openssl.org/index.php/Hostname_validation +// [3] https://wiki.openssl.org/index.php/SSL/TLS_Client diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h index 14200de449a..ec8df853c16 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h +++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h @@ -3,6 +3,7 @@ #include "openssl_typedefs.h" #include <vespa/vespalib/net/socket_address.h> +#include <vespa/vespalib/net/socket_spec.h> #include <vespa/vespalib/net/tls/transport_security_options.h> #include <vespa/vespalib/net/tls/crypto_codec.h> #include <memory> @@ -46,17 +47,23 @@ class OpenSslCryptoCodecImpl : public CryptoCodec { // The context maintains shared verification callback state, so it must be // kept alive explictly for at least as long as any codecs. std::shared_ptr<OpenSslTlsContextImpl> _ctx; + SocketSpec _peer_spec; SocketAddress _peer_address; - SslPtr _ssl; - ::BIO* _input_bio; // Owned by _ssl - ::BIO* _output_bio; // Owned by _ssl - Mode _mode; + SslPtr _ssl; + ::BIO* _input_bio; // Owned by _ssl + ::BIO* _output_bio; // Owned by _ssl + Mode _mode; std::optional<DeferredHandshakeParams> _deferred_handshake_params; - std::optional<HandshakeResult> _deferred_handshake_result; + std::optional<HandshakeResult> _deferred_handshake_result; public: - OpenSslCryptoCodecImpl(std::shared_ptr<OpenSslTlsContextImpl> ctx, const SocketAddress& peer_address, Mode mode); ~OpenSslCryptoCodecImpl() override; + static std::unique_ptr<OpenSslCryptoCodecImpl> make_client_codec(std::shared_ptr<OpenSslTlsContextImpl> ctx, + const SocketSpec& peer_spec, + const SocketAddress& peer_address); + static std::unique_ptr<OpenSslCryptoCodecImpl> make_server_codec(std::shared_ptr<OpenSslTlsContextImpl> ctx, + const SocketAddress& peer_address); + /* * From RFC 8449 (Record Size Limit Extension for TLS), section 1: * "TLS versions 1.2 [RFC5246] and earlier permit senders to @@ -89,7 +96,20 @@ public: EncodeResult half_close(char* ciphertext, size_t ciphertext_size) noexcept override; const SocketAddress& peer_address() const noexcept { return _peer_address; } + /* + * If a client has sent a SNI extension field as part of the handshake, + * returns the raw string representation of this. It only makes sense to + * call this for codecs in server mode. + */ + std::optional<vespalib::string> client_provided_sni_extension() const; private: + OpenSslCryptoCodecImpl(std::shared_ptr<OpenSslTlsContextImpl> ctx, + const SocketSpec& peer_spec, + const SocketAddress& peer_address, + Mode mode); + + void enable_hostname_validation_if_requested(); + void set_server_name_indication_extension(); HandshakeResult do_handshake_and_consume_peer_input_bytes() noexcept; DecodeResult drain_and_produce_plaintext_from_ssl(char* plaintext, size_t plaintext_size) noexcept; // Precondition: read_result < 0 diff --git a/vespalib/src/vespa/vespalib/net/tls/peer_policies.h b/vespalib/src/vespa/vespalib/net/tls/peer_policies.h index 44cbe70fd92..c558708de8f 100644 --- a/vespalib/src/vespa/vespalib/net/tls/peer_policies.h +++ b/vespalib/src/vespa/vespalib/net/tls/peer_policies.h @@ -64,19 +64,24 @@ public: class AuthorizedPeers { // A peer will be authorized iff it matches _one or more_ policies. std::vector<PeerPolicy> _peer_policies; - bool _allow_all_if_empty = false; + bool _allow_all_if_empty; explicit AuthorizedPeers(bool allow_all_if_empty) : _peer_policies(), _allow_all_if_empty(allow_all_if_empty) {} public: - AuthorizedPeers() = default; + AuthorizedPeers() : _peer_policies(), _allow_all_if_empty(false) {} explicit AuthorizedPeers(std::vector<PeerPolicy> peer_policies_) : _peer_policies(std::move(peer_policies_)), _allow_all_if_empty(false) {} + AuthorizedPeers(const AuthorizedPeers&) = default; + AuthorizedPeers& operator=(const AuthorizedPeers&) = default; + AuthorizedPeers(AuthorizedPeers&&) noexcept = default; + AuthorizedPeers& operator=(AuthorizedPeers&&) noexcept = default; + static AuthorizedPeers allow_all_authenticated() { return AuthorizedPeers(true); } diff --git a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp index d0475f3e88d..99862e83dbf 100644 --- a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp @@ -12,18 +12,16 @@ TlsCryptoEngine::TlsCryptoEngine(net::tls::TransportSecurityOptions tls_opts, ne } std::unique_ptr<TlsCryptoSocket> -TlsCryptoEngine::create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &) +TlsCryptoEngine::create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &peer_spec) { - auto mode = net::tls::CryptoCodec::Mode::Client; - auto codec = net::tls::CryptoCodec::create_default_codec(_tls_ctx, SocketAddress::peer_address(socket.get()), mode); + auto codec = net::tls::CryptoCodec::create_default_client_codec(_tls_ctx, peer_spec, SocketAddress::peer_address(socket.get())); return std::make_unique<net::tls::CryptoCodecAdapter>(std::move(socket), std::move(codec)); } std::unique_ptr<TlsCryptoSocket> TlsCryptoEngine::create_tls_server_crypto_socket(SocketHandle socket) { - auto mode = net::tls::CryptoCodec::Mode::Server; - auto codec = net::tls::CryptoCodec::create_default_codec(_tls_ctx, SocketAddress::peer_address(socket.get()), mode); + auto codec = net::tls::CryptoCodec::create_default_server_codec(_tls_ctx, SocketAddress::peer_address(socket.get())); return std::make_unique<net::tls::CryptoCodecAdapter>(std::move(socket), std::move(codec)); } diff --git a/vespalib/src/vespa/vespalib/net/tls/transport_security_options.cpp b/vespalib/src/vespa/vespalib/net/tls/transport_security_options.cpp index c9a5fa5a6e9..47b0e1e0a43 100644 --- a/vespalib/src/vespa/vespalib/net/tls/transport_security_options.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/transport_security_options.cpp @@ -11,41 +11,57 @@ TransportSecurityOptions::TransportSecurityOptions(Params params) _cert_chain_pem(std::move(params._cert_chain_pem)), _private_key_pem(std::move(params._private_key_pem)), _authorized_peers(std::move(params._authorized_peers)), - _accepted_ciphers(std::move(params._accepted_ciphers)) + _accepted_ciphers(std::move(params._accepted_ciphers)), + _disable_hostname_validation(params._disable_hostname_validation) { } TransportSecurityOptions::TransportSecurityOptions(vespalib::string ca_certs_pem, vespalib::string cert_chain_pem, - vespalib::string private_key_pem) + vespalib::string private_key_pem, + AuthorizedPeers authorized_peers, + bool disable_hostname_validation) : _ca_certs_pem(std::move(ca_certs_pem)), _cert_chain_pem(std::move(cert_chain_pem)), _private_key_pem(std::move(private_key_pem)), - _authorized_peers(AuthorizedPeers::allow_all_authenticated()) + _authorized_peers(std::move(authorized_peers)), + _disable_hostname_validation(disable_hostname_validation) { } -TransportSecurityOptions::TransportSecurityOptions(vespalib::string ca_certs_pem, - vespalib::string cert_chain_pem, - vespalib::string private_key_pem, - AuthorizedPeers authorized_peers) - : _ca_certs_pem(std::move(ca_certs_pem)), - _cert_chain_pem(std::move(cert_chain_pem)), - _private_key_pem(std::move(private_key_pem)), - _authorized_peers(std::move(authorized_peers)) -{ +TransportSecurityOptions TransportSecurityOptions::copy_without_private_key() const { + return TransportSecurityOptions(_ca_certs_pem, _cert_chain_pem, "", + _authorized_peers, _disable_hostname_validation); } void secure_memzero(void* buf, size_t size) noexcept { OPENSSL_cleanse(buf, size); } -TransportSecurityOptions::Params::Params() = default; +TransportSecurityOptions::Params::Params() + : _ca_certs_pem(), + _cert_chain_pem(), + _private_key_pem(), + _authorized_peers(), + _accepted_ciphers(), + _disable_hostname_validation(false) +{ +} TransportSecurityOptions::Params::~Params() { secure_memzero(&_private_key_pem[0], _private_key_pem.size()); } +TransportSecurityOptions::Params::Params(const Params&) = default; + +TransportSecurityOptions::Params& +TransportSecurityOptions::Params::operator=(const TransportSecurityOptions::Params&) = default; + +TransportSecurityOptions::Params::Params(Params&&) noexcept = default; + +TransportSecurityOptions::Params& +TransportSecurityOptions::Params::operator=(TransportSecurityOptions::Params&&) noexcept = default; + TransportSecurityOptions::~TransportSecurityOptions() { secure_memzero(&_private_key_pem[0], _private_key_pem.size()); } diff --git a/vespalib/src/vespa/vespalib/net/tls/transport_security_options.h b/vespalib/src/vespa/vespalib/net/tls/transport_security_options.h index 39abb1ce4de..e0a48fc6cf5 100644 --- a/vespalib/src/vespa/vespalib/net/tls/transport_security_options.h +++ b/vespalib/src/vespa/vespalib/net/tls/transport_security_options.h @@ -14,18 +14,22 @@ class TransportSecurityOptions { vespalib::string _private_key_pem; AuthorizedPeers _authorized_peers; std::vector<vespalib::string> _accepted_ciphers; + bool _disable_hostname_validation; public: - TransportSecurityOptions() = default; - struct Params { vespalib::string _ca_certs_pem; vespalib::string _cert_chain_pem; vespalib::string _private_key_pem; AuthorizedPeers _authorized_peers; std::vector<vespalib::string> _accepted_ciphers; + bool _disable_hostname_validation; Params(); ~Params(); + Params(const Params&); + Params& operator=(const Params&); + Params(Params&&) noexcept; + Params& operator=(Params&&) noexcept; Params& ca_certs_pem(vespalib::stringref pem) { _ca_certs_pem = pem; return *this; } Params& cert_chain_pem(vespalib::stringref pem) { _cert_chain_pem = pem; return *this; } @@ -35,19 +39,14 @@ public: _accepted_ciphers = std::move(ciphers); return *this; } + Params& disable_hostname_validation(bool disable) { + _disable_hostname_validation = disable; + return *this; + } }; explicit TransportSecurityOptions(Params params); - TransportSecurityOptions(vespalib::string ca_certs_pem, - vespalib::string cert_chain_pem, - vespalib::string private_key_pem); - - TransportSecurityOptions(vespalib::string ca_certs_pem, - vespalib::string cert_chain_pem, - vespalib::string private_key_pem, - AuthorizedPeers authorized_peers); - ~TransportSecurityOptions(); const vespalib::string& ca_certs_pem() const noexcept { return _ca_certs_pem; } @@ -55,10 +54,16 @@ public: const vespalib::string& private_key_pem() const noexcept { return _private_key_pem; } const AuthorizedPeers& authorized_peers() const noexcept { return _authorized_peers; } - TransportSecurityOptions copy_without_private_key() const { - return TransportSecurityOptions(_ca_certs_pem, _cert_chain_pem, "", _authorized_peers); - } + TransportSecurityOptions copy_without_private_key() const; const std::vector<vespalib::string>& accepted_ciphers() const noexcept { return _accepted_ciphers; } + bool disable_hostname_validation() const noexcept { return _disable_hostname_validation; } + +private: + TransportSecurityOptions(vespalib::string ca_certs_pem, + vespalib::string cert_chain_pem, + vespalib::string private_key_pem, + AuthorizedPeers authorized_peers, + bool disable_hostname_validation); }; // Zeroes out `size` bytes in `buf` in a way that shall never be optimized diff --git a/vespalib/src/vespa/vespalib/net/tls/transport_security_options_reading.cpp b/vespalib/src/vespa/vespalib/net/tls/transport_security_options_reading.cpp index 6b6e65bef8a..80caa15e8b2 100644 --- a/vespalib/src/vespa/vespalib/net/tls/transport_security_options_reading.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/transport_security_options_reading.cpp @@ -123,6 +123,12 @@ std::unique_ptr<TransportSecurityOptions> load_from_input(Input& input) { auto priv_key = load_file_referenced_by_field(files, "private-key"); auto authorized_peers = parse_authorized_peers(root["authorized-peers"]); auto accepted_ciphers = parse_accepted_ciphers(root["accepted-ciphers"]); + // FIXME this is temporary until we know it won't break a bunch of things! + // It's still possible to explicitly enable hostname validation by setting this to false. + bool disable_hostname_validation = true; + if (root["disable-hostname-validation"].valid()) { + disable_hostname_validation = root["disable-hostname-validation"].asBool(); + } auto options = std::make_unique<TransportSecurityOptions>( TransportSecurityOptions::Params() @@ -130,7 +136,8 @@ std::unique_ptr<TransportSecurityOptions> load_from_input(Input& input) { .cert_chain_pem(certs) .private_key_pem(priv_key) .authorized_peers(std::move(authorized_peers)) - .accepted_ciphers(std::move(accepted_ciphers))); + .accepted_ciphers(std::move(accepted_ciphers)) + .disable_hostname_validation(disable_hostname_validation)); secure_memzero(&priv_key[0], priv_key.size()); return options; } diff --git a/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp b/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp index dcd2ced8036..b31f2830976 100644 --- a/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp +++ b/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp @@ -70,7 +70,13 @@ namespace vespalib::test { SocketSpec local_spec("tcp/localhost:123"); vespalib::net::tls::TransportSecurityOptions make_tls_options_for_testing() { - return vespalib::net::tls::TransportSecurityOptions(ca_pem, cert_pem, key_pem); + auto ts_builder = vespalib::net::tls::TransportSecurityOptions::Params(). + ca_certs_pem(ca_pem). + cert_chain_pem(cert_pem). + private_key_pem(key_pem). + authorized_peers(vespalib::net::tls::AuthorizedPeers::allow_all_authenticated()). + disable_hostname_validation(true); // FIXME this is to avoid mass breakage of TLS'd networking tests. + return vespalib::net::tls::TransportSecurityOptions(std::move(ts_builder)); } } // namespace vespalib::test diff --git a/zookeeper-server/CMakeLists.txt b/zookeeper-server/CMakeLists.txt index 2d8620f1028..6e8c82bc66e 100644 --- a/zookeeper-server/CMakeLists.txt +++ b/zookeeper-server/CMakeLists.txt @@ -1,4 +1,3 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. add_subdirectory(zookeeper-server-common) -add_subdirectory(zookeeper-server-3.4) add_subdirectory(zookeeper-server-3.5) diff --git a/zookeeper-server/pom.xml b/zookeeper-server/pom.xml index fe182645681..edfbdbad02e 100644 --- a/zookeeper-server/pom.xml +++ b/zookeeper-server/pom.xml @@ -13,7 +13,6 @@ <version>7-SNAPSHOT</version> <modules> <module>zookeeper-server-common</module> - <module>zookeeper-server-3.4</module> <module>zookeeper-server-3.5</module> </modules> <dependencies> diff --git a/zookeeper-server/zookeeper-server-3.4/CMakeLists.txt b/zookeeper-server/zookeeper-server-3.4/CMakeLists.txt deleted file mode 100644 index d80b48c9093..00000000000 --- a/zookeeper-server/zookeeper-server-3.4/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(zookeeper-server-3.4) diff --git a/zookeeper-server/zookeeper-server-3.4/pom.xml b/zookeeper-server/zookeeper-server-3.4/pom.xml deleted file mode 100644 index 756e2332c15..00000000000 --- a/zookeeper-server/zookeeper-server-3.4/pom.xml +++ /dev/null @@ -1,74 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>zookeeper-server</artifactId> - <version>7-SNAPSHOT</version> - <relativePath>../pom.xml</relativePath> - </parent> - <artifactId>zookeeper-server-3.4</artifactId> - <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> - <dependencies> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>zookeeper-server-common</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> - <version>3.4.14</version> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - <version>1.7.5</version> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <compilerArgs> - <arg>-Xlint:all</arg> - <arg>-Werror</arg> - </compilerArgs> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> - <forkMode>once</forkMode> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-install-plugin</artifactId> - <configuration> - <updateReleaseInfo>true</updateReleaseInfo> - </configuration> - </plugin> - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>bundle-plugin</artifactId> - <extensions>true</extensions> - <configuration> - <importPackage>com.sun.management</importPackage> - <bundleSymbolicName>zookeeper-server</bundleSymbolicName> - </configuration> - </plugin> - </plugins> - </build> -</project> diff --git a/zookeeper-server/zookeeper-server-3.4/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.4/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java deleted file mode 100644 index 5b4c0c11e80..00000000000 --- a/zookeeper-server/zookeeper-server-3.4/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import com.google.inject.Inject; -import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.component.AbstractComponent; -import com.yahoo.log.LogLevel; -import static com.yahoo.vespa.defaults.Defaults.getDefaults; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Writes zookeeper config and starts zookeeper server. - * - * @author Ulf Lilleengen - * @author Harald Musum - */ -public class VespaZooKeeperServerImpl extends AbstractComponent implements Runnable, VespaZooKeeperServer { - - private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(VespaZooKeeperServerImpl.class.getName()); - private static final String ZOOKEEPER_JMX_LOG4J_DISABLE = "zookeeper.jmx.log4j.disable"; - static final String ZOOKEEPER_JUTE_MAX_BUFFER = "jute.maxbuffer"; - private final Thread zkServerThread; - private final ZookeeperServerConfig zookeeperServerConfig; - - VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig, boolean startServer) { - this.zookeeperServerConfig = zookeeperServerConfig; - System.setProperty("zookeeper.jmx.log4j.disable", "true"); - System.setProperty("zookeeper.snapshot.trust.empty", Boolean.valueOf(zookeeperServerConfig.trustEmptySnapshot()).toString()); - System.setProperty(ZOOKEEPER_JUTE_MAX_BUFFER, Integer.valueOf(zookeeperServerConfig.juteMaxBuffer()).toString()); - - writeConfigToDisk(zookeeperServerConfig); - zkServerThread = new Thread(this, "zookeeper server"); - if (startServer) { - zkServerThread.start(); - } - } - - @Inject - public VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig) { - this(zookeeperServerConfig, true); - } - - private void writeConfigToDisk(ZookeeperServerConfig config) { - String configFilePath = getDefaults().underVespaHome(config.zooKeeperConfigFile()); - new File(configFilePath).getParentFile().mkdirs(); - try (FileWriter writer = new FileWriter(configFilePath)) { - writer.write(transformConfigToString(config)); - writeMyIdFile(config); - } catch (IOException e) { - throw new RuntimeException("Error writing zookeeper config", e); - } - } - - private String transformConfigToString(ZookeeperServerConfig config) { - StringBuilder sb = new StringBuilder(); - sb.append("tickTime=").append(config.tickTime()).append("\n"); - sb.append("initLimit=").append(config.initLimit()).append("\n"); - sb.append("syncLimit=").append(config.syncLimit()).append("\n"); - sb.append("maxClientCnxns=").append(config.maxClientConnections()).append("\n"); - sb.append("snapCount=").append(config.snapshotCount()).append("\n"); - sb.append("dataDir=").append(getDefaults().underVespaHome(config.dataDir())).append("\n"); - sb.append("clientPort=").append(config.clientPort()).append("\n"); - sb.append("autopurge.purgeInterval=").append(config.autopurge().purgeInterval()).append("\n"); - sb.append("autopurge.snapRetainCount=").append(config.autopurge().snapRetainCount()).append("\n"); - // See http://zookeeper.apache.org/doc/r3.4.13/zookeeperAdmin.html#sc_zkCommands - // Includes all available commands in 3.4, except 'wchc' and 'wchp' - // Mandatory when using ZooKeeper 3.5 - sb.append("4lw.commands.whitelist=conf,cons,crst,dump,envi,mntr,ruok,srst,srvr,stat,wchs").append("\n"); - ensureThisServerIsRepresented(config.myid(), config.server()); - config.server().forEach(server -> addServerToCfg(sb, server)); - return sb.toString(); - } - - private void writeMyIdFile(ZookeeperServerConfig config) throws IOException { - if (config.server().size() > 1) { - try (FileWriter writer = new FileWriter(getDefaults().underVespaHome(config.myidFile()))) { - writer.write(config.myid() + "\n"); - } - } - } - - private void ensureThisServerIsRepresented(int myid, List<ZookeeperServerConfig.Server> servers) { - boolean found = false; - for (ZookeeperServerConfig.Server server : servers) { - if (myid == server.id()) { - found = true; - break; - } - } - if (!found) { - throw new RuntimeException("No id in zookeeper server list that corresponds to my id(" + myid + ")"); - } - } - - private void addServerToCfg(StringBuilder sb, ZookeeperServerConfig.Server server) { - sb.append("server.").append(server.id()).append("=").append(server.hostname()).append(":").append(server.quorumPort()).append(":").append(server.electionPort()).append("\n"); - } - - private void shutdown() { - zkServerThread.interrupt(); - try { - zkServerThread.join(); - } catch (InterruptedException e) { - log.log(LogLevel.WARNING, "Error joining server thread on shutdown", e); - } - } - - @Override - public void run() { - System.setProperty(ZOOKEEPER_JMX_LOG4J_DISABLE, "true"); - String[] args = new String[]{getDefaults().underVespaHome(zookeeperServerConfig.zooKeeperConfigFile())}; - log.log(LogLevel.DEBUG, "Starting ZooKeeper server with config file " + args[0]); - log.log(LogLevel.INFO, "Trying to establish ZooKeeper quorum (from " + zookeeperServerHostnames(zookeeperServerConfig) + ")"); - org.apache.zookeeper.server.quorum.QuorumPeerMain.main(args); - } - - @Override - public void deconstruct() { - shutdown(); - super.deconstruct(); - } - - private static Set<String> zookeeperServerHostnames(ZookeeperServerConfig zookeeperServerConfig) { - return zookeeperServerConfig.server().stream().map(ZookeeperServerConfig.Server::hostname).collect(Collectors.toSet()); - } - -} diff --git a/zookeeper-server/zookeeper-server-3.4/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java b/zookeeper-server/zookeeper-server-3.4/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java deleted file mode 100644 index 1081c5fda61..00000000000 --- a/zookeeper-server/zookeeper-server-3.4/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.io.IOUtils; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.io.IOException; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; -import static com.yahoo.vespa.defaults.Defaults.getDefaults; - -/** - * Tests the zookeeper server. - */ -public class VespaZooKeeperServerImplTest { - - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - @Test - public void config_is_written_correctly_when_one_server() throws IOException { - File cfgFile = folder.newFile(); - File idFile = folder.newFile(); - ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder(); - builder.zooKeeperConfigFile(cfgFile.getAbsolutePath()); - builder.myidFile(idFile.getAbsolutePath()); - builder.server(newServer(0, "foo", 123, 321)); - builder.myid(0); - createServer(builder); - validateConfigFileSingleHost(cfgFile); - validateIdFile(idFile, ""); - } - - @Test - public void config_is_written_correctly_when_multiple_servers() throws IOException { - File cfgFile = folder.newFile(); - File idFile = folder.newFile(); - ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder(); - builder.zooKeeperConfigFile(cfgFile.getAbsolutePath()); - builder.server(newServer(0, "foo", 123, 321)); - builder.server(newServer(1, "bar", 234, 432)); - builder.server(newServer(2, "baz", 345, 543)); - builder.myidFile(idFile.getAbsolutePath()); - builder.myid(1); - createServer(builder); - validateConfigFileMultipleHosts(cfgFile); - validateIdFile(idFile, "1\n"); - } - - private void createServer(ZookeeperServerConfig.Builder builder) { - new VespaZooKeeperServerImpl(new ZookeeperServerConfig(builder), false); - } - - @Test(expected = RuntimeException.class) - public void require_that_this_id_must_be_present_amongst_servers() { - ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder(); - builder.server(newServer(1, "bar", 234, 432)); - builder.server(newServer(2, "baz", 345, 543)); - builder.myid(0); - createServer(builder); - } - - @Test - public void juteMaxBufferCanBeSet() throws IOException { - ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder(); - builder.myid(0); - File idFile = folder.newFile(); - File cfgFile = folder.newFile(); - - builder.server(new ZookeeperServerConfig.Server.Builder().id(0).hostname("testhost")); - builder.zooKeeperConfigFile(cfgFile.getAbsolutePath()); - builder.myidFile(idFile.getAbsolutePath()); - - createServer(builder); - assertThat(System.getProperty(VespaZooKeeperServerImpl.ZOOKEEPER_JUTE_MAX_BUFFER), is("" + new ZookeeperServerConfig(builder).juteMaxBuffer())); - - final int max_buffer = 1; - builder.juteMaxBuffer(max_buffer); - createServer(builder); - assertThat(System.getProperty(VespaZooKeeperServerImpl.ZOOKEEPER_JUTE_MAX_BUFFER), is("" + max_buffer)); - } - - private ZookeeperServerConfig.Server.Builder newServer(int id, String hostName, int electionPort, int quorumPort) { - ZookeeperServerConfig.Server.Builder builder = new ZookeeperServerConfig.Server.Builder(); - builder.id(id); - builder.hostname(hostName); - builder.electionPort(electionPort); - builder.quorumPort(quorumPort); - return builder; - } - - private void validateIdFile(File idFile, String expected) throws IOException { - String actual = IOUtils.readFile(idFile); - assertThat(actual, is(expected)); - } - - private void validateConfigFileSingleHost(File cfgFile) throws IOException { - String expected = - "tickTime=2000\n" + - "initLimit=20\n" + - "syncLimit=15\n" + - "maxClientCnxns=0\n" + - "snapCount=50000\n" + - "dataDir=" + getDefaults().underVespaHome("var/zookeeper") + "\n" + - "clientPort=2181\n" + - "autopurge.purgeInterval=1\n" + - "autopurge.snapRetainCount=15\n" + - "4lw.commands.whitelist=conf,cons,crst,dump,envi,mntr,ruok,srst,srvr,stat,wchs\n" + - "server.0=foo:321:123\n"; - validateConfigFile(cfgFile, expected); - } - - private void validateConfigFileMultipleHosts(File cfgFile) throws IOException { - String expected = - "tickTime=2000\n" + - "initLimit=20\n" + - "syncLimit=15\n" + - "maxClientCnxns=0\n" + - "snapCount=50000\n" + - "dataDir=" + getDefaults().underVespaHome("var/zookeeper") + "\n" + - "clientPort=2181\n" + - "autopurge.purgeInterval=1\n" + - "autopurge.snapRetainCount=15\n" + - "4lw.commands.whitelist=conf,cons,crst,dump,envi,mntr,ruok,srst,srvr,stat,wchs\n" + - "server.0=foo:321:123\n" + - "server.1=bar:432:234\n" + - "server.2=baz:543:345\n"; - validateConfigFile(cfgFile, expected); - } - - private void validateConfigFile(File cfgFile, String expected) throws IOException { - String actual = IOUtils.readFile(cfgFile); - assertThat(actual, is(expected)); - } -} |